🐛 watch filtered configmaps & deployments to reduce memory usage of cluster-manager (#1030)

Signed-off-by: Yang Le <yangle@redhat.com>

manifests/cluster-manager/management/cluster-manager-addon-manager-deployment.yaml

@@ -5,6 +5,7 @@ metadata:
   namespace: {{ .ClusterManagerNamespace }}
   labels:
     app: clustermanager-controller
+    createdByClusterManager: {{ .ClusterManagerName }}
 spec:
   replicas: {{ .Replica }}
   selector:

manifests/cluster-manager/management/cluster-manager-manifestworkreplicaset-deployment.yaml

@@ -5,6 +5,7 @@ metadata:
   namespace: {{ .ClusterManagerNamespace }}
   labels:
     app: {{ .ClusterManagerName }}-work-controller
+    createdByClusterManager: {{ .ClusterManagerName }}
 spec:
   replicas:  {{ .Replica }}
   selector:

manifests/cluster-manager/management/cluster-manager-placement-deployment.yaml

@@ -5,6 +5,7 @@ metadata:
   namespace: {{ .ClusterManagerNamespace }}
   labels:
     app: clustermanager-controller
+    createdByClusterManager: {{ .ClusterManagerName }}
 spec:
   replicas: {{ .Replica }}
   selector:

manifests/cluster-manager/management/cluster-manager-registration-deployment.yaml

@@ -5,6 +5,7 @@ metadata:
   namespace: {{ .ClusterManagerNamespace }}
   labels:
     app: clustermanager-controller
+    createdByClusterManager: {{ .ClusterManagerName }}
 spec:
   replicas: {{ .Replica }}
   selector:

manifests/cluster-manager/management/cluster-manager-registration-webhook-deployment.yaml

@@ -5,6 +5,7 @@ metadata:
   namespace: {{ .ClusterManagerNamespace }}
   labels:
     app: {{ .ClusterManagerName }}-registration-webhook
+    createdByClusterManager: {{ .ClusterManagerName }}
 spec:
   replicas: {{ .Replica }}
   selector:

manifests/cluster-manager/management/cluster-manager-work-webhook-deployment.yaml

@@ -5,6 +5,7 @@ metadata:
   namespace: {{ .ClusterManagerNamespace }}
   labels:
     app: {{ .ClusterManagerName }}-work-webhook
+    createdByClusterManager: {{ .ClusterManagerName }}
 spec:
   replicas: {{ .Replica }}
   selector:

pkg/operator/helpers/helpers.go

@@ -52,6 +52,9 @@ const (
 	// DefaultAddonNamespace is the default namespace for agent addon
 	DefaultAddonNamespace = "open-cluster-management-agent-addon"
 
+	// HubLabelKey is used to filter resources in informers
+	HubLabelKey = "createdByClusterManager"
+
 	// AgentLabelKey is used to filter resources in informers
 	AgentLabelKey = "createdByKlusterlet"
 )
@@ -819,10 +822,12 @@ func GetOperatorNamespace() string {
 	return operatorNamespace
 }
 
-func GetKlusterletAgentLabels(klusterlet *operatorapiv1.Klusterlet) map[string]string {
-	labels := klusterlet.GetLabels()
-	if labels == nil {
-		labels = map[string]string{}
+func GetKlusterletAgentLabels(klusterlet *operatorapiv1.Klusterlet, enableSyncLabels bool) map[string]string {
+	labels := map[string]string{}
+	if enableSyncLabels {
+		for k, v := range klusterlet.GetLabels() {
+			labels[k] = v
+		}
 	}
 
 	// This label key is used to filter resources in deployment informer

pkg/operator/helpers/helpers_test.go

@@ -1657,3 +1657,61 @@ func TestFeatureGateEnabled(t *testing.T) {
 		})
 	}
 }
+
+func TestGetKlusterletAgentLabels(t *testing.T) {
+	cases := []struct {
+		name             string
+		klusterlet       *operatorapiv1.Klusterlet
+		enableSyncLabels bool
+		desiredLabels    map[string]string
+	}{
+		{
+			name: "enableSyncLabels is false",
+			klusterlet: &operatorapiv1.Klusterlet{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "cluster-manager",
+				},
+			},
+			desiredLabels: map[string]string{
+				AgentLabelKey: "cluster-manager",
+			},
+		},
+		{
+			name: "enableSyncLabels is true",
+			klusterlet: &operatorapiv1.Klusterlet{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "cluster-manager",
+				},
+			},
+			enableSyncLabels: true,
+			desiredLabels: map[string]string{
+				AgentLabelKey: "cluster-manager",
+			},
+		},
+		{
+			name: "with klusterlet labels",
+			klusterlet: &operatorapiv1.Klusterlet{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "cluster-manager",
+					Labels: map[string]string{
+						"testKey": "testValue",
+					},
+				},
+			},
+			enableSyncLabels: true,
+			desiredLabels: map[string]string{
+				AgentLabelKey: "cluster-manager",
+				"testKey":     "testValue",
+			},
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			actual := GetKlusterletAgentLabels(tc.klusterlet, tc.enableSyncLabels)
+			if !reflect.DeepEqual(actual, tc.desiredLabels) {
+				t.Errorf("Name: %s, expect labels are %v, but got %v", tc.name, tc.desiredLabels, actual)
+			}
+		})
+	}
+}

pkg/operator/helpers/queuekey.go

@@ -85,7 +85,7 @@ func KlusterletDeploymentQueueKeyFunc(klusterletLister operatorlister.Klusterlet
 		namespace := accessor.GetNamespace()
 		name := accessor.GetName()
 		interestedObjectFound := false
-		if strings.HasSuffix(name, "registration-agent") || strings.HasSuffix(name, "work-agent") {
+		if strings.HasSuffix(name, "-agent") {
 			interestedObjectFound = true
 		}
 		if !interestedObjectFound {
@@ -115,6 +115,8 @@ func ClusterManagerDeploymentQueueKeyFunc(clusterManagerLister operatorlister.Cl
 		if strings.HasSuffix(name, "registration-controller") ||
 			strings.HasSuffix(name, "registration-webhook") ||
 			strings.HasSuffix(name, "work-webhook") ||
+			strings.HasSuffix(name, "addon-manager-controller") ||
+			strings.HasSuffix(name, "work-controller") ||
 			strings.HasSuffix(name, "placement-controller") {
 			interestedObjectFound = true
 		}

pkg/operator/operators/clustermanager/options.go

@@ -36,24 +36,30 @@ func (o *Options) RunClusterManagerOperator(ctx context.Context, controllerConte
 		return err
 	}
 
-	// kubeInformer is for 3 usages: configmapInformer, secretInformer, deploynmentInformer
-	// After we introduced hosted mode, the hub components could be installed in a customized
-	// namespace.(Before that, it only inform from "open-cluster-management-hub" namespace)
-	// It requires us to add filter for each Informer respectively.
-	// TODO: Watch all namespace may cause performance issue.
-	kubeInformer := informers.NewSharedInformerFactoryWithOptions(kubeClient, 5*time.Minute)
-
-	newOnTermInformer := func(name string) informers.SharedInformerFactory {
+	newOneTermInformer := func(name string) informers.SharedInformerFactory {
 		return informers.NewSharedInformerFactoryWithOptions(kubeClient, 5*time.Minute,
 			informers.WithTweakListOptions(func(options *metav1.ListOptions) {
 				options.FieldSelector = fields.OneTermEqualSelector("metadata.name", name).String()
 			}))
 	}
 
-	signerSecretInformer := newOnTermInformer(helpers.SignerSecret)
-	registrationSecretInformer := newOnTermInformer(helpers.RegistrationWebhookSecret)
-	workSecretInformer := newOnTermInformer(helpers.WorkWebhookSecret)
-	configmapInformer := newOnTermInformer(helpers.CaBundleConfigmap)
+	signerSecretInformer := newOneTermInformer(helpers.SignerSecret)
+	registrationSecretInformer := newOneTermInformer(helpers.RegistrationWebhookSecret)
+	workSecretInformer := newOneTermInformer(helpers.WorkWebhookSecret)
+	configmapInformer := newOneTermInformer(helpers.CaBundleConfigmap)
+
+	deploymentInformer := informers.NewSharedInformerFactoryWithOptions(kubeClient, 5*time.Minute,
+		informers.WithTweakListOptions(func(options *metav1.ListOptions) {
+			selector := &metav1.LabelSelector{
+				MatchExpressions: []metav1.LabelSelectorRequirement{
+					{
+						Key:      helpers.HubLabelKey,
+						Operator: metav1.LabelSelectorOpExists,
+					},
+				},
+			}
+			options.LabelSelector = metav1.FormatLabelSelector(selector)
+		}))
 
 	secretInformers := map[string]corev1informers.SecretInformer{
 		helpers.SignerSecret:              signerSecretInformer.Core().V1().Secrets(),
@@ -73,8 +79,8 @@ func (o *Options) RunClusterManagerOperator(ctx context.Context, controllerConte
 		controllerContext.KubeConfig,
 		operatorClient.OperatorV1().ClusterManagers(),
 		operatorInformer.Operator().V1().ClusterManagers(),
-		kubeInformer.Apps().V1().Deployments(),
-		kubeInformer.Core().V1().ConfigMaps(),
+		deploymentInformer.Apps().V1().Deployments(),
+		configmapInformer.Core().V1().ConfigMaps(),
 		controllerContext.EventRecorder,
 		o.SkipRemoveCRDs,
 		o.ControlPlaneNodeLabelSelector,
@@ -85,7 +91,7 @@ func (o *Options) RunClusterManagerOperator(ctx context.Context, controllerConte
 	statusController := clustermanagerstatuscontroller.NewClusterManagerStatusController(
 		operatorClient.OperatorV1().ClusterManagers(),
 		operatorInformer.Operator().V1().ClusterManagers(),
-		kubeInformer.Apps().V1().Deployments(),
+		deploymentInformer.Apps().V1().Deployments(),
 		controllerContext.EventRecorder)
 
 	certRotationController := certrotationcontroller.NewCertRotationController(
@@ -109,7 +115,7 @@ func (o *Options) RunClusterManagerOperator(ctx context.Context, controllerConte
 		controllerContext.EventRecorder)
 
 	go operatorInformer.Start(ctx.Done())
-	go kubeInformer.Start(ctx.Done())
+	go deploymentInformer.Start(ctx.Done())
 	go signerSecretInformer.Start(ctx.Done())
 	go registrationSecretInformer.Start(ctx.Done())
 	go workSecretInformer.Start(ctx.Done())

pkg/operator/operators/klusterlet/controllers/klusterletcontroller/klusterlet_controller.go

@@ -290,9 +290,7 @@ func (n *klusterletController) sync(ctx context.Context, controllerContext facto
 
 	config.populateBootstrap(klusterlet)
 
-	if n.enableSyncLabels {
-		config.Labels = helpers.GetKlusterletAgentLabels(klusterlet)
-	}
+	config.Labels = helpers.GetKlusterletAgentLabels(klusterlet, n.enableSyncLabels)
 
 	managedClusterClients, err := n.managedClusterClientsBuilder.
 		withMode(config.InstallMode).

pkg/operator/operators/klusterlet/controllers/klusterletcontroller/klusterlet_controller_test.go

@@ -553,7 +553,7 @@ func ensureObject(t *testing.T, object runtime.Object, klusterlet *operatorapiv1
 		return
 	}
 
-	if enableSyncLabels && !helpers.MapCompare(helpers.GetKlusterletAgentLabels(klusterlet), access.GetLabels()) {
+	if enableSyncLabels && !helpers.MapCompare(helpers.GetKlusterletAgentLabels(klusterlet, enableSyncLabels), access.GetLabels()) {
 		t.Errorf("the labels of klusterlet are not synced to %v", access.GetName())
 		return
 	}

pkg/operator/operators/klusterlet/controllers/klusterletcontroller/klusterlet_managed_reconcile.go

@@ -55,10 +55,7 @@ type managedReconcile struct {
 
 func (r *managedReconcile) reconcile(ctx context.Context, klusterlet *operatorapiv1.Klusterlet,
 	config klusterletConfig) (*operatorapiv1.Klusterlet, reconcileState, error) {
-	labels := map[string]string{}
-	if r.enableSyncLabels {
-		labels = helpers.GetKlusterletAgentLabels(klusterlet)
-	}
+	labels := helpers.GetKlusterletAgentLabels(klusterlet, r.enableSyncLabels)
 
 	if !config.DisableAddonNamespace {
 		// For now, whether in Default or Hosted mode, the addons will be deployed on the managed cluster.
@@ -153,9 +150,7 @@ func (r *managedReconcile) createAggregationRule(ctx context.Context, klusterlet
 			},
 			Rules: []rbacv1.PolicyRule{},
 		}
-		if r.enableSyncLabels {
-			aggregateClusterRole.SetLabels(helpers.GetKlusterletAgentLabels(klusterlet))
-		}
+		aggregateClusterRole.SetLabels(helpers.GetKlusterletAgentLabels(klusterlet, r.enableSyncLabels))
 		_, createErr := r.managedClusterClients.kubeClient.RbacV1().ClusterRoles().Create(ctx, aggregateClusterRole, metav1.CreateOptions{})
 		return createErr
 	}

pkg/operator/operators/klusterlet/controllers/klusterletcontroller/klusterlet_management_recocile.go

@@ -49,10 +49,7 @@ type managementReconcile struct {
 
 func (r *managementReconcile) reconcile(ctx context.Context, klusterlet *operatorapiv1.Klusterlet,
 	config klusterletConfig) (*operatorapiv1.Klusterlet, reconcileState, error) {
-	labels := map[string]string{}
-	if r.enableSyncLabels {
-		labels = helpers.GetKlusterletAgentLabels(klusterlet)
-	}
+	labels := helpers.GetKlusterletAgentLabels(klusterlet, r.enableSyncLabels)
 
 	err := ensureNamespace(ctx, r.kubeClient, klusterlet, config.AgentNamespace, labels, r.recorder)
 	if err != nil {

pkg/operator/operators/klusterlet/controllers/klusterletcontroller/klusterlet_runtime_reconcile.go

@@ -199,10 +199,7 @@ func (r *runtimeReconcile) createManagedClusterKubeconfig(
 	klusterlet *operatorapiv1.Klusterlet,
 	klusterletNamespace, agentNamespace, saName, secretName string,
 	recorder events.Recorder) error {
-	labels := map[string]string{}
-	if r.enableSyncLabels {
-		labels = helpers.GetKlusterletAgentLabels(klusterlet)
-	}
+	labels := helpers.GetKlusterletAgentLabels(klusterlet, r.enableSyncLabels)
 
 	tokenGetter := helpers.SATokenGetter(ctx, saName, klusterletNamespace, r.managedClusterClients.kubeClient)
 	err := helpers.SyncKubeConfigSecret(ctx, secretName, agentNamespace, "/spoke/config/kubeconfig",

test/integration/operator/klusterlet_aws_test.go

@@ -68,7 +68,7 @@ var _ = ginkgo.Describe("Klusterlet using aws auth", func() {
 		}
 
 		agentLabelSelector = metav1.FormatLabelSelector(&metav1.LabelSelector{
-			MatchLabels: helpers.GetKlusterletAgentLabels(klusterlet),
+			MatchLabels: helpers.GetKlusterletAgentLabels(klusterlet, false),
 		})
 
 		hubKubeConfigSecret = &corev1.Secret{

test/integration/operator/klusterlet_test.go

@@ -81,7 +81,7 @@ var _ = ginkgo.Describe("Klusterlet", func() {
 		}
 
 		agentLabelSelector = metav1.FormatLabelSelector(&metav1.LabelSelector{
-			MatchLabels: helpers.GetKlusterletAgentLabels(klusterlet),
+			MatchLabels: helpers.GetKlusterletAgentLabels(klusterlet, false),
 		})
 
 		hubKubeConfigSecret = &corev1.Secret{