github/open-cluster-management
1
0
Fork 0
mirror of https://github.com/open-cluster-management-io/ocm.git synced 2026-08-23 22:26:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add ClusterAnnotations support. (#234)

Browse Source 
Signed-off-by: xuezhaojun <zxue@redhat.com>
This commit is contained in:
xuezhaojun
2023-07-27 04:31:25 +02:00
committed by GitHub
parent 70d61123a6
commit 142fd5b247
21 changed files with 249 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
deploy/klusterlet/config/crds/0000_00_operator.open-cluster-management.io_klusterlets.crd.yaml
+7
View File
@@ -184,6 +184,13 @@ spec:
set.
format: int32
type: integer
clusterAnnotations:
additionalProperties:
type: string
description: ClusterAnnotations is annotations with the reserve
prefix "agent.open-cluster-management.io" set on ManagedCluster
when creating only, other actors can update it afterwards.
type: object
featureGates:
description: 'FeatureGates represents the list of feature gates
for registration If it is set empty, default feature gates will
deploy/klusterlet/olm-catalog/klusterlet/manifests/operator.open-cluster-management.io_klusterlets.yaml
+5
View File
@@ -116,6 +116,11 @@ spec:
description: clientCertExpirationSeconds represents the seconds of a client certificate to expire. If it is not set or 0, the default duration seconds will be set by the hub cluster. If the value is larger than the max signing duration seconds set on the hub cluster, the max signing duration seconds will be set.
format: int32
type: integer
clusterAnnotations:
additionalProperties:
type: string
description: ClusterAnnotations is annotations with the reserve prefix "agent.open-cluster-management.io" set on ManagedCluster when creating only, other actors can update it afterwards.
type: object
featureGates:
description: 'FeatureGates represents the list of feature gates for registration If it is set empty, default feature gates will be used. If it is set, featuregate/Foo is an example of one item in FeatureGates: 1. If featuregate/Foo does not exist, registration-operator will discard it 2. If featuregate/Foo exists and is false by default. It is now possible to set featuregate/Foo=[false|true] 3. If featuregate/Foo exists and is true by default. If a cluster-admin upgrading from 1 to 2 wants to continue having featuregate/Foo=false, he can set featuregate/Foo=false before upgrading. Let''s say the cluster-admin wants featuregate/Foo=false.'
items:
go.mod
+1 -1
View File
@@ -28,7 +28,7 @@ require (
k8s.io/kube-aggregator v0.27.2
k8s.io/utils v0.0.0-20230313181309-38a27ef9d749
open-cluster-management.io/addon-framework v0.7.1-0.20230705031704-6a328fa5cd63
open-cluster-management.io/api v0.11.1-0.20230720020428-0336f2374d02
open-cluster-management.io/api v0.11.1-0.20230725140722-c0c9fb59d249
sigs.k8s.io/controller-runtime v0.15.0
sigs.k8s.io/kube-storage-version-migrator v0.0.5
)
go.sum
+2 -2
View File
@@ -1158,8 +1158,8 @@ k8s.io/utils v0.0.0-20230313181309-38a27ef9d749 h1:xMMXJlJbsU8w3V5N2FLDQ8YgU8s1E
k8s.io/utils v0.0.0-20230313181309-38a27ef9d749/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
open-cluster-management.io/addon-framework v0.7.1-0.20230705031704-6a328fa5cd63 h1:GCsAD1jb6wqhXTHdUM/HcWzv5b2NbZ6FxpLZcxa/jhI=
open-cluster-management.io/addon-framework v0.7.1-0.20230705031704-6a328fa5cd63/go.mod h1:V+WUFC7GD89Lc68eXSN/FJebnCH4NjrfF44VsO0YAC8=
open-cluster-management.io/api v0.11.1-0.20230720020428-0336f2374d02 h1:YdfpLV94HvYJexLeguX2WZ9Asszks0aNt2K4Ifd3Bfg=
open-cluster-management.io/api v0.11.1-0.20230720020428-0336f2374d02/go.mod h1:WgKUCJ7+Bf40DsOmH1Gdkpyj3joco+QLzrlM6Ak39zE=
open-cluster-management.io/api v0.11.1-0.20230725140722-c0c9fb59d249 h1:Q3UCh10q8w0k/sx0YDer6svU44/puTZMmxVVFndGdUs=
open-cluster-management.io/api v0.11.1-0.20230725140722-c0c9fb59d249/go.mod h1:WgKUCJ7+Bf40DsOmH1Gdkpyj3joco+QLzrlM6Ak39zE=
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
manifests/klusterlet/management/klusterlet-agent-deployment.yaml
+3
View File
@@ -75,6 +75,9 @@ spec:
{{if gt .ClientCertExpirationSeconds 0}}
- "--client-cert-expiration-seconds={{ .ClientCertExpirationSeconds }}"
{{end}}
{{if .ClusterAnnotationsString}}
- "--cluster-annotations={{ .ClusterAnnotationsString }}"
{{end}}
securityContext:
allowPrivilegeEscalation: false
capabilities:
manifests/klusterlet/management/klusterlet-registration-deployment.yaml
+3
View File
@@ -73,6 +73,9 @@ spec:
{{if gt .ClientCertExpirationSeconds 0}}
- "--client-cert-expiration-seconds={{ .ClientCertExpirationSeconds }}"
{{end}}
{{if .ClusterAnnotationsString}}
- "--cluster-annotations={{ .ClusterAnnotationsString }}"
{{end}}
securityContext:
allowPrivilegeEscalation: false
capabilities:
pkg/common/helpers/annotations.go
+26
View File
@@ -0,0 +1,26 @@
package helpers
import (
"strings"
"k8s.io/klog/v2"
operatorv1 "open-cluster-management.io/api/operator/v1"
)
func FilterClusterAnnotations(annotations map[string]string) map[string]string {
clusterAnnotations := make(map[string]string)
if annotations == nil {
return clusterAnnotations
}
for k, v := range annotations {
if strings.HasPrefix(k, operatorv1.ClusterAnnotationsKeyPrefix) {
clusterAnnotations[k] = v
} else {
klog.Warningf("annotation %q is not prefixed with %q, it will be ignored", k, operatorv1.ClusterAnnotationsKeyPrefix)
}
}
return clusterAnnotations
}
pkg/common/helpers/annotations_test.go
+71
View File
@@ -0,0 +1,71 @@
package helpers
import (
"reflect"
"testing"
operatorv1 "open-cluster-management.io/api/operator/v1"
)
func TestFilterClusterAnnotations(t *testing.T) {
tests := []struct {
name string
annotations map[string]string
want map[string]string
}{
{
name: "empty annotations",
annotations: map[string]string{},
want: map[string]string{},
},
{
name: "no cluster annotations",
annotations: map[string]string{
"foo": "bar",
"baz": "qux",
},
want: map[string]string{},
},
{
name: "one cluster annotation",
annotations: map[string]string{
operatorv1.ClusterAnnotationsKeyPrefix + "foo": "bar",
"baz": "qux",
},
want: map[string]string{
operatorv1.ClusterAnnotationsKeyPrefix + "foo": "bar",
},
},
{
name: "multiple cluster annotations",
annotations: map[string]string{
operatorv1.ClusterAnnotationsKeyPrefix + "foo": "bar",
operatorv1.ClusterAnnotationsKeyPrefix + "baz": "qux",
"quux": "corge",
},
want: map[string]string{
operatorv1.ClusterAnnotationsKeyPrefix + "foo": "bar",
operatorv1.ClusterAnnotationsKeyPrefix + "baz": "qux",
},
},
{
name: "all annotations are cluster annotations",
annotations: map[string]string{
operatorv1.ClusterAnnotationsKeyPrefix + "foo": "bar",
operatorv1.ClusterAnnotationsKeyPrefix + "baz": "qux",
},
want: map[string]string{
operatorv1.ClusterAnnotationsKeyPrefix + "foo": "bar",
operatorv1.ClusterAnnotationsKeyPrefix + "baz": "qux",
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := FilterClusterAnnotations(tt.annotations); !reflect.DeepEqual(got, tt.want) {
t.Errorf("FilterClusterAnnotations() = %v, want %v", got, tt.want)
}
})
}
}
pkg/operator/operators/klusterlet/controllers/klusterletcontroller/klusterlet_controller.go
+9
View File
@@ -28,6 +28,7 @@ import (
ocmfeature "open-cluster-management.io/api/feature"
operatorapiv1 "open-cluster-management.io/api/operator/v1"
commonhelpers "open-cluster-management.io/ocm/pkg/common/helpers"
"open-cluster-management.io/ocm/pkg/common/patcher"
"open-cluster-management.io/ocm/pkg/common/queue"
"open-cluster-management.io/ocm/pkg/operator/helpers"
@@ -134,6 +135,7 @@ type klusterletConfig struct {
OperatorNamespace string
Replica int32
ClientCertExpirationSeconds int32
ClusterAnnotationsString string
ExternalManagedKubeConfigSecret string
ExternalManagedKubeConfigRegistrationSecret string
@@ -234,6 +236,13 @@ func (n *klusterletController) sync(ctx context.Context, controllerContext facto
if klusterlet.Spec.RegistrationConfiguration != nil {
registrationFeatureGates = klusterlet.Spec.RegistrationConfiguration.FeatureGates
config.ClientCertExpirationSeconds = klusterlet.Spec.RegistrationConfiguration.ClientCertExpirationSeconds
// construct cluster annotations string, the final format is "key1=value1,key2=value2"
var annotationsArray []string
for k, v := range commonhelpers.FilterClusterAnnotations(klusterlet.Spec.RegistrationConfiguration.ClusterAnnotations) {
annotationsArray = append(annotationsArray, fmt.Sprintf("%s=%s", k, v))
}
config.ClusterAnnotationsString = strings.Join(annotationsArray, ",")
}
config.RegistrationFeatureGates, registrationFeatureMsgs = helpers.ConvertToFeatureGateFlags("Registration",
registrationFeatureGates, ocmfeature.DefaultSpokeRegistrationFeatureGates)
pkg/registration/spoke/options.go
+3
View File
@@ -18,6 +18,7 @@ type SpokeAgentOptions struct {
ClusterHealthCheckPeriod time.Duration
MaxCustomClusterClaims int
ClientCertExpirationSeconds int32
ClusterAnnotations map[string]string
}
func NewSpokeAgentOptions() *SpokeAgentOptions {
@@ -43,6 +44,8 @@ func (o *SpokeAgentOptions) AddFlags(fs *pflag.FlagSet) {
fs.Int32Var(&o.ClientCertExpirationSeconds, "client-cert-expiration-seconds", o.ClientCertExpirationSeconds,
"The requested duration in seconds of validity of the issued client certificate. If this is not set, "+
"the value of --cluster-signing-duration command-line flag of the kube-controller-manager will be used.")
fs.StringToStringVar(&o.ClusterAnnotations, "cluster-annotations", o.ClusterAnnotations, `the annotations with the reserve
prefix "agent.open-cluster-management.io" set on ManagedCluster when creating only, other actors can update it afterwards.`)
}
// Validate verifies the inputs.
pkg/registration/spoke/registration/creating_controller.go
+8 -2
View File
@@ -14,6 +14,8 @@ import (
clientset "open-cluster-management.io/api/client/cluster/clientset/versioned"
clusterv1 "open-cluster-management.io/api/cluster/v1"
commonhelpers "open-cluster-management.io/ocm/pkg/common/helpers"
)
var (
@@ -26,19 +28,22 @@ type managedClusterCreatingController struct {
clusterName string
spokeExternalServerURLs []string
spokeCABundle []byte
clusterAnnotations map[string]string
hubClusterClient clientset.Interface
}
// NewManagedClusterCreatingController creates a new managedClusterCreatingController on the managed cluster.
func NewManagedClusterCreatingController(
clusterName string, spokeExternalServerURLs []string,
clusterName string, spokeExternalServerURLs []string, annotations map[string]string,
spokeCABundle []byte,
hubClusterClient clientset.Interface,
recorder events.Recorder) factory.Controller {
c := &managedClusterCreatingController{
clusterName: clusterName,
spokeExternalServerURLs: spokeExternalServerURLs,
spokeCABundle: spokeCABundle,
clusterAnnotations: commonhelpers.FilterClusterAnnotations(annotations),
hubClusterClient: hubClusterClient,
}
@@ -64,7 +69,8 @@ func (c *managedClusterCreatingController) sync(ctx context.Context, syncCtx fac
if errors.IsNotFound(err) {
managedCluster := &clusterv1.ManagedCluster{
ObjectMeta: metav1.ObjectMeta{
Name: c.clusterName,
Name: c.clusterName,
Annotations: c.clusterAnnotations,
},
}
pkg/registration/spoke/registration/creating_controller_test.go
+11
View File
@@ -36,6 +36,14 @@ func TestCreateSpokeCluster(t *testing.T) {
actual := actions[1].(clienttesting.CreateActionImpl).Object
actualClientConfigs := actual.(*clusterv1.ManagedCluster).Spec.ManagedClusterClientConfigs
testinghelpers.AssertManagedClusterClientConfigs(t, actualClientConfigs, expectedClientConfigs)
clusterannotations := actual.(*clusterv1.ManagedCluster).Annotations
if len(clusterannotations) != 1 {
t.Errorf("expected cluster annotations %#v but got: %#v", 1, len(clusterannotations))
}
if value, ok := clusterannotations["agent.open-cluster-management.io/test"]; !ok || value != "true" {
t.Errorf("expected cluster annotations %#v but got: %#v", "agent.open-cluster-management.io/test",
clusterannotations["agent.open-cluster-management.io/test"])
}
},
},
{
@@ -55,6 +63,9 @@ func TestCreateSpokeCluster(t *testing.T) {
spokeExternalServerURLs: []string{testSpokeExternalServerUrl},
spokeCABundle: []byte("testcabundle"),
hubClusterClient: clusterClient,
clusterAnnotations: map[string]string{
"agent.open-cluster-management.io/test": "true",
},
}
syncErr := ctrl.sync(context.TODO(), testingcommon.NewFakeSyncContext(t, ""))
pkg/registration/spoke/spokeagent.go
+1 -1
View File
@@ -173,7 +173,7 @@ func (o *SpokeAgentConfig) RunSpokeAgentWithSpokeInformers(ctx context.Context,
// start a SpokeClusterCreatingController to make sure there is a spoke cluster on hub cluster
spokeClusterCreatingController := registration.NewManagedClusterCreatingController(
o.agentOptions.SpokeClusterName, o.registrationOption.SpokeExternalServerURLs,
o.agentOptions.SpokeClusterName, o.registrationOption.SpokeExternalServerURLs, o.registrationOption.ClusterAnnotations,
spokeClusterCABundle,
bootstrapClusterClient,
recorder,
test/integration/operator/klusterlet_test.go
+9
View File
@@ -963,6 +963,7 @@ var _ = ginkgo.Describe("Klusterlet", func() {
gomega.Expect(operatorClient.OperatorV1().Klusterlets().Delete(context.Background(),
klusterlet.Name, metav1.DeleteOptions{})).To(gomega.BeNil())
})
ginkgo.It("feature gates configuration is nil or empty", func() {
klusterlet.Spec.RegistrationConfiguration = nil
klusterlet.Spec.WorkConfiguration = &operatorapiv1.WorkConfiguration{}
@@ -1083,6 +1084,10 @@ var _ = ginkgo.Describe("Klusterlet", func() {
Mode: operatorapiv1.FeatureGateModeTypeDisable,
},
},
ClusterAnnotations: map[string]string{
"foo": "bar", // should be ignored
"agent.open-cluster-management.io/foo": "bar",
},
}
klusterlet.Spec.WorkConfiguration = &operatorapiv1.WorkConfiguration{
FeatureGates: []operatorapiv1.FeatureGate{
@@ -1141,6 +1146,10 @@ var _ = ginkgo.Describe("Klusterlet", func() {
gomega.Expect(registrationDeployment.Spec.Template.Spec.Containers[0].Args).Should(
gomega.ContainElement("--feature-gates=ClusterClaim=false"))
ginkgo.By("Check the registration-agent has the expected cluster-annotations")
gomega.Expect(registrationDeployment.Spec.Template.Spec.Containers[0].Args).Should(
gomega.ContainElement("--cluster-annotations=agent.open-cluster-management.io/foo=bar"))
ginkgo.By("Check the work-agent has the expected feature gates")
workDeployment, err := kubeClient.AppsV1().Deployments(klusterletNamespace).Get(
context.Background(), workDeploymentName, metav1.GetOptions{})
test/integration/registration/clusterannotations_test.go
+59
View File
@@ -0,0 +1,59 @@
package registration_test
import (
"fmt"
"path"
"time"
"github.com/onsi/ginkgo/v2"
"github.com/onsi/gomega"
commonoptions "open-cluster-management.io/ocm/pkg/common/options"
"open-cluster-management.io/ocm/pkg/registration/spoke"
"open-cluster-management.io/ocm/test/integration/util"
)
var _ = ginkgo.Describe("Cluster Annotations", func() {
ginkgo.It("Cluster Annotations should be created on the managed cluster", func() {
managedClusterName := "clusterannotations-spokecluster"
//#nosec G101
hubKubeconfigSecret := "clusterannotations-hub-kubeconfig-secret"
hubKubeconfigDir := path.Join(util.TestDir, "clusterannotations", "hub-kubeconfig")
agentOptions := &spoke.SpokeAgentOptions{
BootstrapKubeconfig: bootstrapKubeConfigFile,
HubKubeconfigSecret: hubKubeconfigSecret,
ClusterHealthCheckPeriod: 1 * time.Minute,
ClusterAnnotations: map[string]string{
"agent.open-cluster-management.io/foo": "bar",
"foo": "bar", // this annotation should be filtered out
},
}
commOptions := commonoptions.NewAgentOptions()
commOptions.HubKubeconfigDir = hubKubeconfigDir
commOptions.SpokeClusterName = managedClusterName
// run registration agent
cancel := runAgent("rotationtest", agentOptions, commOptions, spokeCfg)
defer cancel()
// after bootstrap the spokecluster and csr should be created
gomega.Eventually(func() error {
mc, err := util.GetManagedCluster(clusterClient, managedClusterName)
if err != nil {
return err
}
if len(mc.Annotations) != 1 {
return fmt.Errorf("expected 1 annotation, got %d", len(mc.Annotations))
}
if mc.Annotations["agent.open-cluster-management.io/foo"] != "bar" {
return fmt.Errorf("expected annotation agent.open-cluster-management.io/foo to be bar, got %s", mc.Annotations["agent.open-cluster-management.io/foo"])
}
return nil
}, eventuallyTimeout, eventuallyInterval).Should(gomega.Succeed())
})
})
vendor/modules.txt
Vendored
+1 -1
View File
@@ -1426,7 +1426,7 @@ open-cluster-management.io/addon-framework/pkg/index
open-cluster-management.io/addon-framework/pkg/manager/controllers/addonconfiguration
open-cluster-management.io/addon-framework/pkg/manager/controllers/addonowner
open-cluster-management.io/addon-framework/pkg/utils
# open-cluster-management.io/api v0.11.1-0.20230720020428-0336f2374d02
# open-cluster-management.io/api v0.11.1-0.20230725140722-c0c9fb59d249
## explicit; go 1.19
open-cluster-management.io/api/addon/v1alpha1
open-cluster-management.io/api/client/addon/clientset/versioned
vendor/open-cluster-management.io/api/crdsv1beta1/0001_00_operator.open-cluster-management.io_klusterlets.crd.yaml
Generated Vendored
+5
View File
@@ -119,6 +119,11 @@ spec:
description: clientCertExpirationSeconds represents the seconds of a client certificate to expire. If it is not set or 0, the default duration seconds will be set by the hub cluster. If the value is larger than the max signing duration seconds set on the hub cluster, the max signing duration seconds will be set.
type: integer
format: int32
clusterAnnotations:
description: ClusterAnnotations is annotations with the reserve prefix "agent.open-cluster-management.io" set on ManagedCluster when creating only, other actors can update it afterwards.
type: object
additionalProperties:
type: string
featureGates:
description: "FeatureGates represents the list of feature gates for registration If it is set empty, default feature gates will be used. If it is set, featuregate/Foo is an example of one item in FeatureGates: 1. If featuregate/Foo does not exist, registration-operator will discard it 2. If featuregate/Foo exists and is false by default. It is now possible to set featuregate/Foo=[false|true] 3. If featuregate/Foo exists and is true by default. If a cluster-admin upgrading from 1 to 2 wants to continue having featuregate/Foo=false, \the can set featuregate/Foo=false before upgrading. Let's say the cluster-admin wants featuregate/Foo=false."
type: array
vendor/open-cluster-management.io/api/operator/v1/0000_00_operator.open-cluster-management.io_klusterlets.crd.yaml
Generated Vendored
+7
View File
@@ -184,6 +184,13 @@ spec:
set.
format: int32
type: integer
clusterAnnotations:
additionalProperties:
type: string
description: ClusterAnnotations is annotations with the reserve
prefix "agent.open-cluster-management.io" set on ManagedCluster
when creating only, other actors can update it afterwards.
type: object
featureGates:
description: 'FeatureGates represents the list of feature gates
for registration If it is set empty, default feature gates will
vendor/open-cluster-management.io/api/operator/v1/types.go
Generated Vendored
+10
View File
@@ -107,8 +107,18 @@ type RegistrationConfiguration struct {
// he can set featuregate/Foo=false before upgrading. Let's say the cluster-admin wants featuregate/Foo=false.
// +optional
FeatureGates []FeatureGate `json:"featureGates,omitempty"`
// ClusterAnnotations is annotations with the reserve prefix "agent.open-cluster-management.io" set on
// ManagedCluster when creating only, other actors can update it afterwards.
// +optional
ClusterAnnotations map[string]string `json:"clusterAnnotations,omitempty"`
}
const (
// ClusterAnnotationsKeyPrefix is the prefix of annotations set on ManagedCluster when creating only.
ClusterAnnotationsKeyPrefix = "agent.open-cluster-management.io"
)
type WorkConfiguration struct {
// FeatureGates represents the list of feature gates for work
// If it is set empty, default feature gates will be used.
vendor/open-cluster-management.io/api/operator/v1/zz_generated.deepcopy.go
Generated Vendored
+7
View File
@@ -434,6 +434,13 @@ func (in *RegistrationConfiguration) DeepCopyInto(out *RegistrationConfiguration
*out = make([]FeatureGate, len(*in))
copy(*out, *in)
}
if in.ClusterAnnotations != nil {
in, out := &in.ClusterAnnotations, &out.ClusterAnnotations
*out = make(map[string]string, len(*in))
for key, val := range *in {
(*out)[key] = val
}
}
return
}
vendor/open-cluster-management.io/api/operator/v1/zz_generated.swagger_doc_generated.go
Generated Vendored
+1
View File
@@ -194,6 +194,7 @@ func (NodePlacement) SwaggerDoc() map[string]string {
var map_RegistrationConfiguration = map[string]string{
"clientCertExpirationSeconds": "clientCertExpirationSeconds represents the seconds of a client certificate to expire. If it is not set or 0, the default duration seconds will be set by the hub cluster. If the value is larger than the max signing duration seconds set on the hub cluster, the max signing duration seconds will be set.",
"featureGates": "FeatureGates represents the list of feature gates for registration If it is set empty, default feature gates will be used. If it is set, featuregate/Foo is an example of one item in FeatureGates:\n 1. If featuregate/Foo does not exist, registration-operator will discard it\n 2. If featuregate/Foo exists and is false by default. It is now possible to set featuregate/Foo=[false|true]\n 3. If featuregate/Foo exists and is true by default. If a cluster-admin upgrading from 1 to 2 wants to continue having featuregate/Foo=false,\n \the can set featuregate/Foo=false before upgrading. Let's say the cluster-admin wants featuregate/Foo=false.",
"clusterAnnotations": "ClusterAnnotations is annotations with the reserve prefix \"agent.open-cluster-management.io\" set on ManagedCluster when creating only, other actors can update it afterwards.",
}
func (RegistrationConfiguration) SwaggerDoc() map[string]string {