From c2cedbe2ff077fdbed296b0dfa78605330cf4c9e Mon Sep 17 00:00:00 2001 From: Mohamed ElSerngawy Date: Wed, 5 Apr 2023 22:09:18 -0400 Subject: [PATCH] Fix ManifestWorkReplicaset featureGate (#340) Signed-off-by: melserngawy --- manifests/config.go | 1 + .../clustermanager_controller.go | 3 +- .../clustermanager_controller_test.go | 8 +++++ .../clustermanager_hub_reconcile.go | 15 ++++++++ .../clustermanager_runtime_reconcile.go | 16 ++++++++- test/e2e/common.go | 35 ++++++++++++------- test/integration/integration_suite_test.go | 8 +++++ 7 files changed, 71 insertions(+), 15 deletions(-) diff --git a/manifests/config.go b/manifests/config.go index f7da7fad1..df350a501 100644 --- a/manifests/config.go +++ b/manifests/config.go @@ -16,6 +16,7 @@ type HubConfig struct { WorkFeatureGates []string AddOnManagerImage string AddOnManagerEnabled bool + MWReplicaSetEnabled bool } type Webhook struct { diff --git a/pkg/operators/clustermanager/controllers/clustermanagercontroller/clustermanager_controller.go b/pkg/operators/clustermanager/controllers/clustermanagercontroller/clustermanager_controller.go index 96e6df765..9fbe63b24 100644 --- a/pkg/operators/clustermanager/controllers/clustermanagercontroller/clustermanager_controller.go +++ b/pkg/operators/clustermanager/controllers/clustermanagercontroller/clustermanager_controller.go @@ -3,7 +3,6 @@ package clustermanagercontroller import ( "context" "encoding/base64" - ocmfeature "open-cluster-management.io/api/feature" utilerrors "k8s.io/apimachinery/pkg/util/errors" @@ -30,6 +29,7 @@ import ( operatorv1client "open-cluster-management.io/api/client/operator/clientset/versioned/typed/operator/v1" operatorinformer "open-cluster-management.io/api/client/operator/informers/externalversions/operator/v1" operatorlister "open-cluster-management.io/api/client/operator/listers/operator/v1" + ocmfeature "open-cluster-management.io/api/feature" operatorapiv1 "open-cluster-management.io/api/operator/v1" "open-cluster-management.io/registration-operator/manifests" "open-cluster-management.io/registration-operator/pkg/helpers" @@ -165,6 +165,7 @@ func (n *clusterManagerController) sync(ctx context.Context, controllerContext f workFeatureGates = clusterManager.Spec.WorkConfiguration.FeatureGates } config.WorkFeatureGates, workFeatureMsgs = helpers.ConvertToFeatureGateFlags("Work", workFeatureGates, ocmfeature.DefaultHubWorkFeatureGates) + config.MWReplicaSetEnabled = helpers.FeatureGateEnabled(workFeatureGates, ocmfeature.DefaultHubWorkFeatureGates, ocmfeature.ManifestWorkReplicaSet) addonFeatureGates := []operatorapiv1.FeatureGate{} if clusterManager.Spec.AddOnManagerConfiguration != nil { diff --git a/pkg/operators/clustermanager/controllers/clustermanagercontroller/clustermanager_controller_test.go b/pkg/operators/clustermanager/controllers/clustermanagercontroller/clustermanager_controller_test.go index 7e9c01980..ca16ab5a0 100644 --- a/pkg/operators/clustermanager/controllers/clustermanagercontroller/clustermanager_controller_test.go +++ b/pkg/operators/clustermanager/controllers/clustermanagercontroller/clustermanager_controller_test.go @@ -46,6 +46,11 @@ type testController struct { } func newClusterManager(name string) *operatorapiv1.ClusterManager { + featureGate := operatorapiv1.FeatureGate{ + Feature: "ManifestWorkReplicaSet", + Mode: operatorapiv1.FeatureGateModeTypeEnable, + } + return &operatorapiv1.ClusterManager{ ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -61,6 +66,9 @@ func newClusterManager(name string) *operatorapiv1.ClusterManager { {Feature: "AddonManagement", Mode: operatorapiv1.FeatureGateModeTypeEnable}, }, }, + WorkConfiguration: &operatorapiv1.WorkConfiguration{ + FeatureGates: []operatorapiv1.FeatureGate{featureGate}, + }, }, } } diff --git a/pkg/operators/clustermanager/controllers/clustermanagercontroller/clustermanager_hub_reconcile.go b/pkg/operators/clustermanager/controllers/clustermanagercontroller/clustermanager_hub_reconcile.go index d13b012ca..5f3e4f98f 100644 --- a/pkg/operators/clustermanager/controllers/clustermanagercontroller/clustermanager_hub_reconcile.go +++ b/pkg/operators/clustermanager/controllers/clustermanagercontroller/clustermanager_hub_reconcile.go @@ -41,6 +41,9 @@ var ( "cluster-manager/hub/cluster-manager-placement-clusterrole.yaml", "cluster-manager/hub/cluster-manager-placement-clusterrolebinding.yaml", "cluster-manager/hub/cluster-manager-placement-serviceaccount.yaml", + } + + mwReplicaSetResourceFiles = []string{ // manifestworkreplicaset "cluster-manager/hub/cluster-manager-manifestworkreplicaset-clusterrole.yaml", "cluster-manager/hub/cluster-manager-manifestworkreplicaset-clusterrolebinding.yaml", @@ -84,6 +87,14 @@ func (c *hubReoncile) reconcile(ctx context.Context, cm *operatorapiv1.ClusterMa } } + // Remove ManifestWokReplicaSet deployment if feature not enabled + if !config.MWReplicaSetEnabled { + _, _, err := cleanResources(ctx, c.hubKubeClient, cm, config, mwReplicaSetResourceFiles...) + if err != nil { + return cm, reconcileStop, err + } + } + hubResources := getHubResources(cm.Spec.DeployOption.Mode, config) var appliedErrs []error @@ -134,6 +145,10 @@ func getHubResources(mode operatorapiv1.InstallMode, config manifests.HubConfig) if config.AddOnManagerEnabled { hubResources = append(hubResources, hubAddOnManagerRbacResourceFiles...) } + + if config.MWReplicaSetEnabled { + hubResources = append(hubResources, mwReplicaSetResourceFiles...) + } // the hubHostedWebhookServiceFiles are only used in hosted mode if mode == operatorapiv1.InstallModeHosted { hubResources = append(hubResources, hubHostedWebhookServiceFiles...) diff --git a/pkg/operators/clustermanager/controllers/clustermanagercontroller/clustermanager_runtime_reconcile.go b/pkg/operators/clustermanager/controllers/clustermanagercontroller/clustermanager_runtime_reconcile.go index e1b00ada3..8d9ee1ff8 100644 --- a/pkg/operators/clustermanager/controllers/clustermanagercontroller/clustermanager_runtime_reconcile.go +++ b/pkg/operators/clustermanager/controllers/clustermanagercontroller/clustermanager_runtime_reconcile.go @@ -29,12 +29,15 @@ var ( "cluster-manager/management/cluster-manager-registration-webhook-deployment.yaml", "cluster-manager/management/cluster-manager-work-webhook-deployment.yaml", "cluster-manager/management/cluster-manager-placement-deployment.yaml", - "cluster-manager/management/cluster-manager-manifestworkreplicaset-deployment.yaml", } addOnManagerDeploymentFiles = []string{ "cluster-manager/management/cluster-manager-addon-manager-deployment.yaml", } + + mwReplicaSetDeploymentFiles = []string{ + "cluster-manager/management/cluster-manager-manifestworkreplicaset-deployment.yaml", + } ) type runtimeReconcile struct { @@ -57,6 +60,14 @@ func (c *runtimeReconcile) reconcile(ctx context.Context, cm *operatorapiv1.Clus } } + // Remove ManifestWokReplicaSet deployment if feature not enabled + if !config.MWReplicaSetEnabled { + _, _, err := cleanResources(ctx, c.kubeClient, cm, config, mwReplicaSetDeploymentFiles...) + if err != nil { + return cm, reconcileStop, err + } + } + // In the Hosted mode, ensure the rbac kubeconfig secrets is existed for deployments to mount. // In this step, we get serviceaccount token from the hub cluster to form a kubeconfig and set it as a secret on the management cluster. // Before this step, the serviceaccounts in the hub cluster and the namespace in the management cluster should be applied first. @@ -107,6 +118,9 @@ func (c *runtimeReconcile) reconcile(ctx context.Context, cm *operatorapiv1.Clus if config.AddOnManagerEnabled { deployResources = append(deployResources, addOnManagerDeploymentFiles...) } + if config.MWReplicaSetEnabled { + deployResources = append(deployResources, mwReplicaSetDeploymentFiles...) + } for _, file := range deployResources { updatedDeployment, currentGeneration, err := helpers.ApplyDeployment( ctx, diff --git a/test/e2e/common.go b/test/e2e/common.go index 60874ee5f..be826f899 100644 --- a/test/e2e/common.go +++ b/test/e2e/common.go @@ -31,6 +31,7 @@ import ( operatorclient "open-cluster-management.io/api/client/operator/clientset/versioned" workv1client "open-cluster-management.io/api/client/work/clientset/versioned" clusterv1 "open-cluster-management.io/api/cluster/v1" + ocmfeature "open-cluster-management.io/api/feature" operatorapiv1 "open-cluster-management.io/api/operator/v1" workapiv1 "open-cluster-management.io/api/work/v1" "open-cluster-management.io/registration-operator/pkg/helpers" @@ -57,6 +58,7 @@ type Tester struct { hubPlacementDeployment string operatorNamespace string klusterletOperator string + hubWorkControllerEnabled bool } // kubeconfigPath is the path of kubeconfig file, will be get from env "KUBECONFIG" by default. @@ -76,6 +78,7 @@ func NewTester(kubeconfigPath string) *Tester { hubPlacementDeployment: "cluster-manager-placement-controller", operatorNamespace: "open-cluster-management", klusterletOperator: "klusterlet", + hubWorkControllerEnabled: false, } return &tester @@ -535,19 +538,21 @@ func (t *Tester) CheckHubReady() error { return nil }, t.EventuallyTimeout*5, t.EventuallyInterval*5).Should(gomega.BeNil()) - gomega.Eventually(func() error { - workHubControllerDeployment, err := t.KubeClient.AppsV1().Deployments(t.clusterManagerNamespace). - Get(context.TODO(), t.hubWorkControllerDeployment, metav1.GetOptions{}) - if err != nil { - return err - } - replicas := *workHubControllerDeployment.Spec.Replicas - readyReplicas := workHubControllerDeployment.Status.ReadyReplicas - if readyReplicas != replicas { - return fmt.Errorf("deployment %s should have %d but got %d ready replicas", t.hubWorkControllerDeployment, replicas, readyReplicas) - } - return nil - }, t.EventuallyTimeout*5, t.EventuallyInterval*5).Should(gomega.BeNil()) + if t.hubWorkControllerEnabled { + gomega.Eventually(func() error { + workHubControllerDeployment, err := t.KubeClient.AppsV1().Deployments(t.clusterManagerNamespace). + Get(context.TODO(), t.hubWorkControllerDeployment, metav1.GetOptions{}) + if err != nil { + return err + } + replicas := *workHubControllerDeployment.Spec.Replicas + readyReplicas := workHubControllerDeployment.Status.ReadyReplicas + if readyReplicas != replicas { + return fmt.Errorf("deployment %s should have %d but got %d ready replicas", t.hubWorkControllerDeployment, replicas, readyReplicas) + } + return nil + }, t.EventuallyTimeout*5, t.EventuallyInterval*5).Should(gomega.BeNil()) + } if _, err := t.KubeClient.AppsV1().Deployments(t.clusterManagerNamespace). Get(context.TODO(), t.hubPlacementDeployment, metav1.GetOptions{}); err != nil { @@ -574,6 +579,10 @@ func (t *Tester) CheckClusterManagerStatus() error { if !meta.IsStatusConditionFalse(cm.Status.Conditions, "Progressing") { return fmt.Errorf("ClusterManager is still progressing") } + if cm.Spec.WorkConfiguration != nil { + t.hubWorkControllerEnabled = helpers.FeatureGateEnabled(cm.Spec.WorkConfiguration.FeatureGates, ocmfeature.DefaultHubWorkFeatureGates, ocmfeature.ManifestWorkReplicaSet) + } + return nil } diff --git a/test/integration/integration_suite_test.go b/test/integration/integration_suite_test.go index 39c2496ec..75ef700bd 100644 --- a/test/integration/integration_suite_test.go +++ b/test/integration/integration_suite_test.go @@ -174,6 +174,14 @@ var _ = ginkgo.BeforeSuite(func() { }, }, }, + WorkConfiguration: &operatorapiv1.WorkConfiguration{ + FeatureGates: []operatorapiv1.FeatureGate{ + { + Feature: "ManifestWorkReplicaSet", + Mode: "Enable", + }, + }, + }, }, }, metav1.CreateOptions{}) gomega.Expect(err).ToNot(gomega.HaveOccurred())