diff --git a/manifests/cluster-manager/management/cluster-manager-addon-manager-deployment.yaml b/manifests/cluster-manager/management/cluster-manager-addon-manager-deployment.yaml index d9c6d419f..df21cc2fd 100644 --- a/manifests/cluster-manager/management/cluster-manager-addon-manager-deployment.yaml +++ b/manifests/cluster-manager/management/cluster-manager-addon-manager-deployment.yaml @@ -5,6 +5,7 @@ metadata: namespace: {{ .ClusterManagerNamespace }} labels: app: clustermanager-controller + createdByClusterManager: {{ .ClusterManagerName }} spec: replicas: {{ .Replica }} selector: diff --git a/manifests/cluster-manager/management/cluster-manager-manifestworkreplicaset-deployment.yaml b/manifests/cluster-manager/management/cluster-manager-manifestworkreplicaset-deployment.yaml index 4ef2ce49f..c18bbb064 100644 --- a/manifests/cluster-manager/management/cluster-manager-manifestworkreplicaset-deployment.yaml +++ b/manifests/cluster-manager/management/cluster-manager-manifestworkreplicaset-deployment.yaml @@ -5,6 +5,7 @@ metadata: namespace: {{ .ClusterManagerNamespace }} labels: app: {{ .ClusterManagerName }}-work-controller + createdByClusterManager: {{ .ClusterManagerName }} spec: replicas: {{ .Replica }} selector: diff --git a/manifests/cluster-manager/management/cluster-manager-placement-deployment.yaml b/manifests/cluster-manager/management/cluster-manager-placement-deployment.yaml index 62f15ab76..37f18333d 100644 --- a/manifests/cluster-manager/management/cluster-manager-placement-deployment.yaml +++ b/manifests/cluster-manager/management/cluster-manager-placement-deployment.yaml @@ -5,6 +5,7 @@ metadata: namespace: {{ .ClusterManagerNamespace }} labels: app: clustermanager-controller + createdByClusterManager: {{ .ClusterManagerName }} spec: replicas: {{ .Replica }} selector: diff --git a/manifests/cluster-manager/management/cluster-manager-registration-deployment.yaml b/manifests/cluster-manager/management/cluster-manager-registration-deployment.yaml index 9a0a6c235..d932854b8 100644 --- a/manifests/cluster-manager/management/cluster-manager-registration-deployment.yaml +++ b/manifests/cluster-manager/management/cluster-manager-registration-deployment.yaml @@ -5,6 +5,7 @@ metadata: namespace: {{ .ClusterManagerNamespace }} labels: app: clustermanager-controller + createdByClusterManager: {{ .ClusterManagerName }} spec: replicas: {{ .Replica }} selector: diff --git a/manifests/cluster-manager/management/cluster-manager-registration-webhook-deployment.yaml b/manifests/cluster-manager/management/cluster-manager-registration-webhook-deployment.yaml index a435c728e..33eaab467 100644 --- a/manifests/cluster-manager/management/cluster-manager-registration-webhook-deployment.yaml +++ b/manifests/cluster-manager/management/cluster-manager-registration-webhook-deployment.yaml @@ -5,6 +5,7 @@ metadata: namespace: {{ .ClusterManagerNamespace }} labels: app: {{ .ClusterManagerName }}-registration-webhook + createdByClusterManager: {{ .ClusterManagerName }} spec: replicas: {{ .Replica }} selector: diff --git a/manifests/cluster-manager/management/cluster-manager-work-webhook-deployment.yaml b/manifests/cluster-manager/management/cluster-manager-work-webhook-deployment.yaml index 096be3ff1..21bf8e64f 100644 --- a/manifests/cluster-manager/management/cluster-manager-work-webhook-deployment.yaml +++ b/manifests/cluster-manager/management/cluster-manager-work-webhook-deployment.yaml @@ -5,6 +5,7 @@ metadata: namespace: {{ .ClusterManagerNamespace }} labels: app: {{ .ClusterManagerName }}-work-webhook + createdByClusterManager: {{ .ClusterManagerName }} spec: replicas: {{ .Replica }} selector: diff --git a/pkg/operator/helpers/helpers.go b/pkg/operator/helpers/helpers.go index 3bc3a4929..b2fccc004 100644 --- a/pkg/operator/helpers/helpers.go +++ b/pkg/operator/helpers/helpers.go @@ -52,6 +52,9 @@ const ( // DefaultAddonNamespace is the default namespace for agent addon DefaultAddonNamespace = "open-cluster-management-agent-addon" + // HubLabelKey is used to filter resources in informers + HubLabelKey = "createdByClusterManager" + // AgentLabelKey is used to filter resources in informers AgentLabelKey = "createdByKlusterlet" ) @@ -819,10 +822,12 @@ func GetOperatorNamespace() string { return operatorNamespace } -func GetKlusterletAgentLabels(klusterlet *operatorapiv1.Klusterlet) map[string]string { - labels := klusterlet.GetLabels() - if labels == nil { - labels = map[string]string{} +func GetKlusterletAgentLabels(klusterlet *operatorapiv1.Klusterlet, enableSyncLabels bool) map[string]string { + labels := map[string]string{} + if enableSyncLabels { + for k, v := range klusterlet.GetLabels() { + labels[k] = v + } } // This label key is used to filter resources in deployment informer diff --git a/pkg/operator/helpers/helpers_test.go b/pkg/operator/helpers/helpers_test.go index b9fab3b31..f5fe1b530 100644 --- a/pkg/operator/helpers/helpers_test.go +++ b/pkg/operator/helpers/helpers_test.go @@ -1657,3 +1657,61 @@ func TestFeatureGateEnabled(t *testing.T) { }) } } + +func TestGetKlusterletAgentLabels(t *testing.T) { + cases := []struct { + name string + klusterlet *operatorapiv1.Klusterlet + enableSyncLabels bool + desiredLabels map[string]string + }{ + { + name: "enableSyncLabels is false", + klusterlet: &operatorapiv1.Klusterlet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster-manager", + }, + }, + desiredLabels: map[string]string{ + AgentLabelKey: "cluster-manager", + }, + }, + { + name: "enableSyncLabels is true", + klusterlet: &operatorapiv1.Klusterlet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster-manager", + }, + }, + enableSyncLabels: true, + desiredLabels: map[string]string{ + AgentLabelKey: "cluster-manager", + }, + }, + { + name: "with klusterlet labels", + klusterlet: &operatorapiv1.Klusterlet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster-manager", + Labels: map[string]string{ + "testKey": "testValue", + }, + }, + }, + enableSyncLabels: true, + desiredLabels: map[string]string{ + AgentLabelKey: "cluster-manager", + "testKey": "testValue", + }, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + actual := GetKlusterletAgentLabels(tc.klusterlet, tc.enableSyncLabels) + if !reflect.DeepEqual(actual, tc.desiredLabels) { + t.Errorf("Name: %s, expect labels are %v, but got %v", tc.name, tc.desiredLabels, actual) + } + }) + } +} diff --git a/pkg/operator/helpers/queuekey.go b/pkg/operator/helpers/queuekey.go index 7c59f9b00..6cdc41bda 100644 --- a/pkg/operator/helpers/queuekey.go +++ b/pkg/operator/helpers/queuekey.go @@ -85,7 +85,7 @@ func KlusterletDeploymentQueueKeyFunc(klusterletLister operatorlister.Klusterlet namespace := accessor.GetNamespace() name := accessor.GetName() interestedObjectFound := false - if strings.HasSuffix(name, "registration-agent") || strings.HasSuffix(name, "work-agent") { + if strings.HasSuffix(name, "-agent") { interestedObjectFound = true } if !interestedObjectFound { @@ -115,6 +115,8 @@ func ClusterManagerDeploymentQueueKeyFunc(clusterManagerLister operatorlister.Cl if strings.HasSuffix(name, "registration-controller") || strings.HasSuffix(name, "registration-webhook") || strings.HasSuffix(name, "work-webhook") || + strings.HasSuffix(name, "addon-manager-controller") || + strings.HasSuffix(name, "work-controller") || strings.HasSuffix(name, "placement-controller") { interestedObjectFound = true } diff --git a/pkg/operator/operators/clustermanager/options.go b/pkg/operator/operators/clustermanager/options.go index 6e54cb6c5..6f5c5b533 100644 --- a/pkg/operator/operators/clustermanager/options.go +++ b/pkg/operator/operators/clustermanager/options.go @@ -36,24 +36,30 @@ func (o *Options) RunClusterManagerOperator(ctx context.Context, controllerConte return err } - // kubeInformer is for 3 usages: configmapInformer, secretInformer, deploynmentInformer - // After we introduced hosted mode, the hub components could be installed in a customized - // namespace.(Before that, it only inform from "open-cluster-management-hub" namespace) - // It requires us to add filter for each Informer respectively. - // TODO: Watch all namespace may cause performance issue. - kubeInformer := informers.NewSharedInformerFactoryWithOptions(kubeClient, 5*time.Minute) - - newOnTermInformer := func(name string) informers.SharedInformerFactory { + newOneTermInformer := func(name string) informers.SharedInformerFactory { return informers.NewSharedInformerFactoryWithOptions(kubeClient, 5*time.Minute, informers.WithTweakListOptions(func(options *metav1.ListOptions) { options.FieldSelector = fields.OneTermEqualSelector("metadata.name", name).String() })) } - signerSecretInformer := newOnTermInformer(helpers.SignerSecret) - registrationSecretInformer := newOnTermInformer(helpers.RegistrationWebhookSecret) - workSecretInformer := newOnTermInformer(helpers.WorkWebhookSecret) - configmapInformer := newOnTermInformer(helpers.CaBundleConfigmap) + signerSecretInformer := newOneTermInformer(helpers.SignerSecret) + registrationSecretInformer := newOneTermInformer(helpers.RegistrationWebhookSecret) + workSecretInformer := newOneTermInformer(helpers.WorkWebhookSecret) + configmapInformer := newOneTermInformer(helpers.CaBundleConfigmap) + + deploymentInformer := informers.NewSharedInformerFactoryWithOptions(kubeClient, 5*time.Minute, + informers.WithTweakListOptions(func(options *metav1.ListOptions) { + selector := &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: helpers.HubLabelKey, + Operator: metav1.LabelSelectorOpExists, + }, + }, + } + options.LabelSelector = metav1.FormatLabelSelector(selector) + })) secretInformers := map[string]corev1informers.SecretInformer{ helpers.SignerSecret: signerSecretInformer.Core().V1().Secrets(), @@ -73,8 +79,8 @@ func (o *Options) RunClusterManagerOperator(ctx context.Context, controllerConte controllerContext.KubeConfig, operatorClient.OperatorV1().ClusterManagers(), operatorInformer.Operator().V1().ClusterManagers(), - kubeInformer.Apps().V1().Deployments(), - kubeInformer.Core().V1().ConfigMaps(), + deploymentInformer.Apps().V1().Deployments(), + configmapInformer.Core().V1().ConfigMaps(), controllerContext.EventRecorder, o.SkipRemoveCRDs, o.ControlPlaneNodeLabelSelector, @@ -85,7 +91,7 @@ func (o *Options) RunClusterManagerOperator(ctx context.Context, controllerConte statusController := clustermanagerstatuscontroller.NewClusterManagerStatusController( operatorClient.OperatorV1().ClusterManagers(), operatorInformer.Operator().V1().ClusterManagers(), - kubeInformer.Apps().V1().Deployments(), + deploymentInformer.Apps().V1().Deployments(), controllerContext.EventRecorder) certRotationController := certrotationcontroller.NewCertRotationController( @@ -109,7 +115,7 @@ func (o *Options) RunClusterManagerOperator(ctx context.Context, controllerConte controllerContext.EventRecorder) go operatorInformer.Start(ctx.Done()) - go kubeInformer.Start(ctx.Done()) + go deploymentInformer.Start(ctx.Done()) go signerSecretInformer.Start(ctx.Done()) go registrationSecretInformer.Start(ctx.Done()) go workSecretInformer.Start(ctx.Done()) diff --git a/pkg/operator/operators/klusterlet/controllers/klusterletcontroller/klusterlet_controller.go b/pkg/operator/operators/klusterlet/controllers/klusterletcontroller/klusterlet_controller.go index 4fb2a12c7..2d1399ca8 100644 --- a/pkg/operator/operators/klusterlet/controllers/klusterletcontroller/klusterlet_controller.go +++ b/pkg/operator/operators/klusterlet/controllers/klusterletcontroller/klusterlet_controller.go @@ -290,9 +290,7 @@ func (n *klusterletController) sync(ctx context.Context, controllerContext facto config.populateBootstrap(klusterlet) - if n.enableSyncLabels { - config.Labels = helpers.GetKlusterletAgentLabels(klusterlet) - } + config.Labels = helpers.GetKlusterletAgentLabels(klusterlet, n.enableSyncLabels) managedClusterClients, err := n.managedClusterClientsBuilder. withMode(config.InstallMode). diff --git a/pkg/operator/operators/klusterlet/controllers/klusterletcontroller/klusterlet_controller_test.go b/pkg/operator/operators/klusterlet/controllers/klusterletcontroller/klusterlet_controller_test.go index 3cf5d27bf..da6771740 100644 --- a/pkg/operator/operators/klusterlet/controllers/klusterletcontroller/klusterlet_controller_test.go +++ b/pkg/operator/operators/klusterlet/controllers/klusterletcontroller/klusterlet_controller_test.go @@ -553,7 +553,7 @@ func ensureObject(t *testing.T, object runtime.Object, klusterlet *operatorapiv1 return } - if enableSyncLabels && !helpers.MapCompare(helpers.GetKlusterletAgentLabels(klusterlet), access.GetLabels()) { + if enableSyncLabels && !helpers.MapCompare(helpers.GetKlusterletAgentLabels(klusterlet, enableSyncLabels), access.GetLabels()) { t.Errorf("the labels of klusterlet are not synced to %v", access.GetName()) return } diff --git a/pkg/operator/operators/klusterlet/controllers/klusterletcontroller/klusterlet_managed_reconcile.go b/pkg/operator/operators/klusterlet/controllers/klusterletcontroller/klusterlet_managed_reconcile.go index ddebb1a98..8df6105d6 100644 --- a/pkg/operator/operators/klusterlet/controllers/klusterletcontroller/klusterlet_managed_reconcile.go +++ b/pkg/operator/operators/klusterlet/controllers/klusterletcontroller/klusterlet_managed_reconcile.go @@ -55,10 +55,7 @@ type managedReconcile struct { func (r *managedReconcile) reconcile(ctx context.Context, klusterlet *operatorapiv1.Klusterlet, config klusterletConfig) (*operatorapiv1.Klusterlet, reconcileState, error) { - labels := map[string]string{} - if r.enableSyncLabels { - labels = helpers.GetKlusterletAgentLabels(klusterlet) - } + labels := helpers.GetKlusterletAgentLabels(klusterlet, r.enableSyncLabels) if !config.DisableAddonNamespace { // For now, whether in Default or Hosted mode, the addons will be deployed on the managed cluster. @@ -153,9 +150,7 @@ func (r *managedReconcile) createAggregationRule(ctx context.Context, klusterlet }, Rules: []rbacv1.PolicyRule{}, } - if r.enableSyncLabels { - aggregateClusterRole.SetLabels(helpers.GetKlusterletAgentLabels(klusterlet)) - } + aggregateClusterRole.SetLabels(helpers.GetKlusterletAgentLabels(klusterlet, r.enableSyncLabels)) _, createErr := r.managedClusterClients.kubeClient.RbacV1().ClusterRoles().Create(ctx, aggregateClusterRole, metav1.CreateOptions{}) return createErr } diff --git a/pkg/operator/operators/klusterlet/controllers/klusterletcontroller/klusterlet_management_recocile.go b/pkg/operator/operators/klusterlet/controllers/klusterletcontroller/klusterlet_management_recocile.go index f491379fe..2e5950488 100644 --- a/pkg/operator/operators/klusterlet/controllers/klusterletcontroller/klusterlet_management_recocile.go +++ b/pkg/operator/operators/klusterlet/controllers/klusterletcontroller/klusterlet_management_recocile.go @@ -49,10 +49,7 @@ type managementReconcile struct { func (r *managementReconcile) reconcile(ctx context.Context, klusterlet *operatorapiv1.Klusterlet, config klusterletConfig) (*operatorapiv1.Klusterlet, reconcileState, error) { - labels := map[string]string{} - if r.enableSyncLabels { - labels = helpers.GetKlusterletAgentLabels(klusterlet) - } + labels := helpers.GetKlusterletAgentLabels(klusterlet, r.enableSyncLabels) err := ensureNamespace(ctx, r.kubeClient, klusterlet, config.AgentNamespace, labels, r.recorder) if err != nil { diff --git a/pkg/operator/operators/klusterlet/controllers/klusterletcontroller/klusterlet_runtime_reconcile.go b/pkg/operator/operators/klusterlet/controllers/klusterletcontroller/klusterlet_runtime_reconcile.go index 391fd2e98..a1eb49f91 100644 --- a/pkg/operator/operators/klusterlet/controllers/klusterletcontroller/klusterlet_runtime_reconcile.go +++ b/pkg/operator/operators/klusterlet/controllers/klusterletcontroller/klusterlet_runtime_reconcile.go @@ -199,10 +199,7 @@ func (r *runtimeReconcile) createManagedClusterKubeconfig( klusterlet *operatorapiv1.Klusterlet, klusterletNamespace, agentNamespace, saName, secretName string, recorder events.Recorder) error { - labels := map[string]string{} - if r.enableSyncLabels { - labels = helpers.GetKlusterletAgentLabels(klusterlet) - } + labels := helpers.GetKlusterletAgentLabels(klusterlet, r.enableSyncLabels) tokenGetter := helpers.SATokenGetter(ctx, saName, klusterletNamespace, r.managedClusterClients.kubeClient) err := helpers.SyncKubeConfigSecret(ctx, secretName, agentNamespace, "/spoke/config/kubeconfig", diff --git a/test/integration/operator/klusterlet_aws_test.go b/test/integration/operator/klusterlet_aws_test.go index 977ddb022..9b1b37645 100644 --- a/test/integration/operator/klusterlet_aws_test.go +++ b/test/integration/operator/klusterlet_aws_test.go @@ -68,7 +68,7 @@ var _ = ginkgo.Describe("Klusterlet using aws auth", func() { } agentLabelSelector = metav1.FormatLabelSelector(&metav1.LabelSelector{ - MatchLabels: helpers.GetKlusterletAgentLabels(klusterlet), + MatchLabels: helpers.GetKlusterletAgentLabels(klusterlet, false), }) hubKubeConfigSecret = &corev1.Secret{ diff --git a/test/integration/operator/klusterlet_test.go b/test/integration/operator/klusterlet_test.go index 5e217f883..adfffc19c 100644 --- a/test/integration/operator/klusterlet_test.go +++ b/test/integration/operator/klusterlet_test.go @@ -81,7 +81,7 @@ var _ = ginkgo.Describe("Klusterlet", func() { } agentLabelSelector = metav1.FormatLabelSelector(&metav1.LabelSelector{ - MatchLabels: helpers.GetKlusterletAgentLabels(klusterlet), + MatchLabels: helpers.GetKlusterletAgentLabels(klusterlet, false), }) hubKubeConfigSecret = &corev1.Secret{