From 3401c8e4bdb09cd72b8bc628e7fed1b837b28d5b Mon Sep 17 00:00:00 2001 From: ivanscai Date: Wed, 22 Jun 2022 14:47:06 +0800 Subject: [PATCH] support hubRegistrationFeatureGates and spokeRegistrationFeatureGates (#230) * support RegistrationFeatureGates in ClusterManager and Klusterlet Signed-off-by: ivan-cai * update to use DefaultHubRegistrationFeatureGates&DefaultSpokeRegistrationFeatureGates in api repo to check registration variable if valid Signed-off-by: ivan-cai * enable addonmanagement and defaultClusterset feature if RegistrationConfiguration is nil Signed-off-by: ivan-cai * fix some errors in unit test Signed-off-by: ivan-cai * add RegistrationFeatureGates to registration webhook Signed-off-by: ivan-cai Co-authored-by: caijing.cai --- ...cluster-management_clustermanagers.cr.yaml | 4 ++ ...pen-cluster-management_klusterlets.cr.yaml | 4 ++ go.mod | 2 +- go.sum | 4 +- ...uster-management.io_manifestworks.crd.yaml | 29 ++++++++- ...uster-manager-registration-deployment.yaml | 6 ++ ...nager-registration-webhook-deployment.yaml | 9 ++- manifests/config.go | 1 + .../klusterlet-registration-deployment.yaml | 6 ++ pkg/helpers/helpers.go | 49 ++++++++++++--- pkg/helpers/helpers_test.go | 59 +++++++++++++++++ .../clustermanager_controller.go | 28 ++++++++- .../klusterlet_controller.go | 24 +++++++ .../klusterlet_controller_test.go | 49 ++++++++++----- test/integration/klusterlet_test.go | 6 ++ vendor/modules.txt | 3 +- .../api/feature/feature.go | 58 +++++++++++++++++ ...uster-management.io_manifestworks.crd.yaml | 29 ++++++++- .../api/work/v1/types.go | 63 +++++++++++++++++-- .../api/work/v1/zz_generated.deepcopy.go | 42 +++++++++++++ .../v1/zz_generated.swagger_doc_generated.go | 22 ++++++- 21 files changed, 460 insertions(+), 37 deletions(-) create mode 100644 vendor/open-cluster-management.io/api/feature/feature.go diff --git a/deploy/cluster-manager/config/samples/operator_open-cluster-management_clustermanagers.cr.yaml b/deploy/cluster-manager/config/samples/operator_open-cluster-management_clustermanagers.cr.yaml index a12cf76f8..d76da8658 100644 --- a/deploy/cluster-manager/config/samples/operator_open-cluster-management_clustermanagers.cr.yaml +++ b/deploy/cluster-manager/config/samples/operator_open-cluster-management_clustermanagers.cr.yaml @@ -8,3 +8,7 @@ spec: placementImagePullSpec: quay.io/open-cluster-management/placement deployOption: mode: Default + registrationConfiguration: + featureGates: + - feature: DefaultClusterSet + mode: Enable \ No newline at end of file diff --git a/deploy/klusterlet/config/samples/operator_open-cluster-management_klusterlets.cr.yaml b/deploy/klusterlet/config/samples/operator_open-cluster-management_klusterlets.cr.yaml index 3d01ab035..80b2831b6 100644 --- a/deploy/klusterlet/config/samples/operator_open-cluster-management_klusterlets.cr.yaml +++ b/deploy/klusterlet/config/samples/operator_open-cluster-management_klusterlets.cr.yaml @@ -11,3 +11,7 @@ spec: namespace: open-cluster-management-agent externalServerURLs: - url: https://localhost + registrationConfiguration: + featureGates: + - feature: AddonManagement + mode: Enable \ No newline at end of file diff --git a/go.mod b/go.mod index f5bad9f02..8716f617a 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( k8s.io/component-base v0.24.0 k8s.io/klog/v2 v2.60.1 k8s.io/kube-aggregator v0.24.0 - open-cluster-management.io/api v0.7.1-0.20220530034043-2b50efdde1de + open-cluster-management.io/api v0.7.1-0.20220609033924-5cc58e815c1a sigs.k8s.io/controller-runtime v0.11.1 sigs.k8s.io/kube-storage-version-migrator v0.0.5 ) diff --git a/go.sum b/go.sum index a5c23a9ed..9d3d265f6 100644 --- a/go.sum +++ b/go.sum @@ -1227,8 +1227,8 @@ k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/ k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 h1:HNSDgDCrr/6Ly3WEGKZftiE7IY19Vz2GdbOCyI4qqhc= k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -open-cluster-management.io/api v0.7.1-0.20220530034043-2b50efdde1de h1:xxlXg3eM5DXa7ezj7J++CggxyaHjYuC67o28w6cFMiM= -open-cluster-management.io/api v0.7.1-0.20220530034043-2b50efdde1de/go.mod h1:QKW4hRonyzoXavBX8XK/Rljo4PYEKKt/IOShuLv49XI= +open-cluster-management.io/api v0.7.1-0.20220609033924-5cc58e815c1a h1:WasoJMZTgGkQLWfpSU3qtP885oa+o/Uqo2qtGWvSleY= +open-cluster-management.io/api v0.7.1-0.20220609033924-5cc58e815c1a/go.mod h1:+OEARSAl2jIhuLItUcS30UgLA3khmA9ihygLVxzEn+U= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/manifests/cluster-manager/hub/0000_00_work.open-cluster-management.io_manifestworks.crd.yaml b/manifests/cluster-manager/hub/0000_00_work.open-cluster-management.io_manifestworks.crd.yaml index 1182080b4..4100bf2ac 100644 --- a/manifests/cluster-manager/hub/0000_00_work.open-cluster-management.io_manifestworks.crd.yaml +++ b/manifests/cluster-manager/hub/0000_00_work.open-cluster-management.io_manifestworks.crd.yaml @@ -109,11 +109,10 @@ spec: description: ManifestConfigOption represents the configurations of a manifest defined in workload field. type: object required: - - feedbackRules - resourceIdentifier properties: feedbackRules: - description: FeedbackRules defines what resource status field should be returned. + description: FeedbackRules defines what resource status field should be returned. If it is not set or empty, no feedback rules will be honored. type: array items: type: object @@ -163,6 +162,32 @@ spec: resource: description: Resource is the resource name of the Kubernetes resource. type: string + updateStrategy: + description: UpdateStrategy defines the strategy to update this manifest. UpdateStrategy is Update if it is not set, optional + type: object + required: + - type + properties: + serverSideApply: + description: serverSideApply defines the configuration for server side apply. It is honored only when type of updateStrategy is ServerSideApply + type: object + properties: + fieldManager: + description: FieldManager is the manager to apply the resource. It is work-agent by default, but can be other name with work-agent as the prefix. + type: string + default: work-agent + pattern: ^work-agent + force: + description: Force represents to force apply the manifest. + type: boolean + type: + description: type defines the strategy to update this manifest, default value is Update. Update type means to update resource by an update call. CreateOnly type means do not update resource based on current manifest. ServerSideApply type means to update resource using server side apply with work-controller as the field manager. If there is conflict, the related Applied condition of manifest will be in the status of False with the reason of ApplyConflict. + type: string + default: Update + enum: + - Update + - CreateOnly + - ServerSideApply workload: description: Workload represents the manifest workload to be deployed on a managed cluster. type: object diff --git a/manifests/cluster-manager/management/cluster-manager-registration-deployment.yaml b/manifests/cluster-manager/management/cluster-manager-registration-deployment.yaml index 76e643772..3f43f9402 100644 --- a/manifests/cluster-manager/management/cluster-manager-registration-deployment.yaml +++ b/manifests/cluster-manager/management/cluster-manager-registration-deployment.yaml @@ -45,7 +45,13 @@ spec: args: - "/registration" - "controller" + {{ if gt (len .RegistrationFeatureGates) 0 }} + {{range .RegistrationFeatureGates}} + - {{ . }} + {{ end }} + {{ else }} - "--feature-gates=DefaultClusterSet=true" + {{ end }} {{ if .HostedMode }} - "--kubeconfig=/var/run/secrets/hub/kubeconfig" {{ end }} diff --git a/manifests/cluster-manager/management/cluster-manager-registration-webhook-deployment.yaml b/manifests/cluster-manager/management/cluster-manager-registration-webhook-deployment.yaml index b5d340797..0cb055c23 100644 --- a/manifests/cluster-manager/management/cluster-manager-registration-webhook-deployment.yaml +++ b/manifests/cluster-manager/management/cluster-manager-registration-webhook-deployment.yaml @@ -48,7 +48,14 @@ spec: - "--secure-port=6443" - "--tls-cert-file=/serving-cert/tls.crt" - "--tls-private-key-file=/serving-cert/tls.key" - - "--feature-gates=DefaultClusterSet=true,APIPriorityAndFairness=false" + {{ if gt (len .RegistrationFeatureGates) 0 }} + {{range .RegistrationFeatureGates}} + - {{ . }} + {{ end }} + {{ else }} + - "--feature-gates=DefaultClusterSet=true" + {{ end }} + - "--feature-gates=APIPriorityAndFairness=false" {{ if .HostedMode }} - "--kubeconfig=/var/run/secrets/hub/kubeconfig" - "--authentication-kubeconfig=/var/run/secrets/hub/kubeconfig" diff --git a/manifests/config.go b/manifests/config.go index ea0d7d437..4f07e9d97 100644 --- a/manifests/config.go +++ b/manifests/config.go @@ -12,6 +12,7 @@ type HubConfig struct { HostedMode bool RegistrationWebhook Webhook WorkWebhook Webhook + RegistrationFeatureGates []string } type Webhook struct { diff --git a/manifests/klusterlet/management/klusterlet-registration-deployment.yaml b/manifests/klusterlet/management/klusterlet-registration-deployment.yaml index 120b5ba1d..7ce7bca93 100644 --- a/manifests/klusterlet/management/klusterlet-registration-deployment.yaml +++ b/manifests/klusterlet/management/klusterlet-registration-deployment.yaml @@ -47,7 +47,13 @@ spec: - "agent" - "--cluster-name={{ .ClusterName }}" - "--bootstrap-kubeconfig=/spoke/bootstrap/kubeconfig" + {{ if gt (len .RegistrationFeatureGates) 0 }} + {{range .RegistrationFeatureGates}} + - {{ . }} + {{end}} + {{ else }} - "--feature-gates=AddonManagement=true" + {{ end }} {{if .ExternalServerURL}} - "--spoke-external-server-urls={{ .ExternalServerURL }}" {{end}} diff --git a/pkg/helpers/helpers.go b/pkg/helpers/helpers.go index 98f35418e..69de18ddc 100644 --- a/pkg/helpers/helpers.go +++ b/pkg/helpers/helpers.go @@ -4,8 +4,10 @@ import ( "context" "crypto/sha256" "fmt" - "reflect" - + "github.com/openshift/api" + "github.com/openshift/library-go/pkg/operator/events" + "github.com/openshift/library-go/pkg/operator/resource/resourceapply" + "github.com/openshift/library-go/pkg/operator/resource/resourcemerge" admissionv1 "k8s.io/api/admissionregistration/v1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -31,27 +33,33 @@ import ( "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" "k8s.io/client-go/util/retry" + "k8s.io/component-base/featuregate" "k8s.io/klog/v2" apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" apiregistrationclient "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/typed/apiregistration/v1" operatorv1client "open-cluster-management.io/api/client/operator/clientset/versioned/typed/operator/v1" + ocmfeature "open-cluster-management.io/api/feature" operatorapiv1 "open-cluster-management.io/api/operator/v1" - - "github.com/openshift/api" - "github.com/openshift/library-go/pkg/operator/events" - "github.com/openshift/library-go/pkg/operator/resource/resourceapply" - "github.com/openshift/library-go/pkg/operator/resource/resourcemerge" + "reflect" ) const ( defaultReplica = 3 singleReplica = 1 + + ComponentHubKey = "hub" + ComponentSpokeKey = "spoke" ) var ( genericScheme = runtime.NewScheme() genericCodecs = serializer.NewCodecFactory(genericScheme) genericCodec = genericCodecs.UniversalDeserializer() + + RegistrationFeatureGatesMap = map[string]map[featuregate.Feature]featuregate.FeatureSpec{ + ComponentHubKey: ocmfeature.DefaultHubRegistrationFeatureGates, + ComponentSpokeKey: ocmfeature.DefaultSpokeRegistrationFeatureGates, + } ) func init() { @@ -847,3 +855,30 @@ func GetHubKubeconfig(ctx context.Context, return operatorKubeconfig, nil } } + +// FeatureGatesArgs is used to generate feature gates args +func FeatureGatesArgs(featureGates []operatorapiv1.FeatureGate, component string) (featureGatesArgs []string, invalidFeatureGates []string) { + for _, featureGate := range featureGates { + if !isValidRegistrationFeatureGate(featureGate.Feature, component) { + invalidFeatureGates = append(invalidFeatureGates, featureGate.Feature) + } else { + value := false + if featureGate.Mode == operatorapiv1.FeatureGateModeTypeEnable { + value = true + } + featureGatesArgs = append(featureGatesArgs, fmt.Sprintf("--feature-gates=%s=%t", featureGate.Feature, value)) + } + } + + return featureGatesArgs, invalidFeatureGates +} + +func isValidRegistrationFeatureGate(feature string, component string) bool { + if featureGates, ok := RegistrationFeatureGatesMap[component]; ok { + if _, ok := featureGates[featuregate.Feature(feature)]; ok { + return true + } + } + + return false +} diff --git a/pkg/helpers/helpers_test.go b/pkg/helpers/helpers_test.go index 39a30fe50..2edf09fea 100644 --- a/pkg/helpers/helpers_test.go +++ b/pkg/helpers/helpers_test.go @@ -1649,3 +1649,62 @@ func TestGetHubKubeconfig(t *testing.T) { }) } } + +func TestFeatureGatesArgs(t *testing.T) { + tt := []struct { + name string + component string + inputFeatureGates []operatorapiv1.FeatureGate + expect1 []string + expect2 []string + }{ + { + name: "empty input feature gates", + component: ComponentHubKey, + inputFeatureGates: []operatorapiv1.FeatureGate{}, + expect1: []string{}, + expect2: []string{}, + }, + { + name: "valid input feature gates", + component: ComponentSpokeKey, + inputFeatureGates: []operatorapiv1.FeatureGate{ + { + Feature: "AddonManagement", + Mode: operatorapiv1.FeatureGateModeTypeEnable, + }, + { + Feature: "V1beta1CSRAPICompatibility", + Mode: operatorapiv1.FeatureGateModeTypeEnable, + }, + }, + expect1: []string{"--feature-gates=AddonManagement=true", "--feature-gates=V1beta1CSRAPICompatibility=true"}, + expect2: []string{}, + }, + { + name: "invalid input feature gates", + component: ComponentSpokeKey, + inputFeatureGates: []operatorapiv1.FeatureGate{ + { + Feature: "AddonManagementInvalid", + Mode: operatorapiv1.FeatureGateModeTypeEnable, + }, + }, + expect1: []string{}, + expect2: []string{"AddonManagementInvalid"}, + }, + } + + for _, tc := range tt { + t.Run(tc.name, func(t *testing.T) { + output1, output2 := FeatureGatesArgs(tc.inputFeatureGates, tc.component) + if len(output1) == len(tc.expect1) && len(output2) == len(tc.expect2) { + return + } + + if !reflect.DeepEqual(output1, tc.expect1) || !reflect.DeepEqual(output2, tc.expect2) { + t.Errorf("Expect to get %v,%v, but got %v,%v", tc.expect1, tc.expect2, output1, output2) + } + }) + } +} diff --git a/pkg/operators/clustermanager/controllers/clustermanagercontroller/clustermanager_controller.go b/pkg/operators/clustermanager/controllers/clustermanagercontroller/clustermanager_controller.go index c1b378c45..d8f15b283 100644 --- a/pkg/operators/clustermanager/controllers/clustermanagercontroller/clustermanager_controller.go +++ b/pkg/operators/clustermanager/controllers/clustermanagercontroller/clustermanager_controller.go @@ -119,6 +119,8 @@ const ( clusterManagerFinalizer = "operator.open-cluster-management.io/cluster-manager-cleanup" clusterManagerApplied = "Applied" caBundleConfigmap = "ca-bundle-configmap" + + hubRegistrationFeatureGatesValid = "ValidRegistrationFeatureGates" ) type clusterManagerController struct { @@ -202,6 +204,31 @@ func (n *clusterManagerController) sync(ctx context.Context, controllerContext f Replica: helpers.DetermineReplica(ctx, n.operatorKubeClient, clusterManagerMode, nil), HostedMode: clusterManager.Spec.DeployOption.Mode == operatorapiv1.InstallModeHosted, } + + conditions := &clusterManager.Status.Conditions + + // If there are some invalid feature gates of registration, will output condition `InvalidRegistrationFeatureGates` in ClusterManager. + if clusterManager.Spec.RegistrationConfiguration != nil && len(clusterManager.Spec.RegistrationConfiguration.FeatureGates) > 0 { + featureGateArgs, invalidFeatureGates := helpers.FeatureGatesArgs( + clusterManager.Spec.RegistrationConfiguration.FeatureGates, helpers.ComponentHubKey) + if len(invalidFeatureGates) == 0 { + config.RegistrationFeatureGates = featureGateArgs + meta.SetStatusCondition(conditions, metav1.Condition{ + Type: hubRegistrationFeatureGatesValid, + Status: metav1.ConditionTrue, + Reason: "FeatureGatesAllValid", + Message: "Registration feature gates of cluster manager are all valid", + }) + } else { + invalidFGErr := fmt.Errorf("There are some invalid feature gates of registration: %v ", invalidFeatureGates) + _, _, updateError := helpers.UpdateClusterManagerStatus(ctx, n.clusterManagerClient, clusterManagerName, helpers.UpdateClusterManagerConditionFn(metav1.Condition{ + Type: hubRegistrationFeatureGatesValid, Status: metav1.ConditionFalse, Reason: "InvalidFeatureGatesExisting", + Message: invalidFGErr.Error(), + })) + return updateError + } + } + // If we are deploying in the hosted mode, it requires us to create webhook in a different way with the default mode. // In the hosted mode, the webhook servers is running in the management cluster but the users are accessing the hub cluster. // So we need to add configuration to make the apiserver of the hub cluster could access the webhook servers on the management cluster. @@ -278,7 +305,6 @@ func (n *clusterManagerController) sync(ctx context.Context, controllerContext f // Update status errs := append(hubAppliedErrs, managementAppliedErrs...) - conditions := &clusterManager.Status.Conditions observedGeneration := clusterManager.Status.ObservedGeneration if len(errs) == 0 { meta.SetStatusCondition(conditions, metav1.Condition{ diff --git a/pkg/operators/klusterlet/controllers/klusterletcontroller/klusterlet_controller.go b/pkg/operators/klusterlet/controllers/klusterletcontroller/klusterlet_controller.go index 92d7ed57c..eadb6358c 100644 --- a/pkg/operators/klusterlet/controllers/klusterletcontroller/klusterlet_controller.go +++ b/pkg/operators/klusterlet/controllers/klusterletcontroller/klusterlet_controller.go @@ -47,6 +47,8 @@ const ( hubConnectionDegraded = "HubConnectionDegraded" hubKubeConfigSecretMissing = "HubKubeConfigSecretMissing" appliedManifestWorkFinalizer = "cluster.open-cluster-management.io/applied-manifest-work-cleanup" + + spokeRegistrationFeatureGatesInvalid = "InvalidRegistrationFeatureGates" ) var ( @@ -170,6 +172,8 @@ type klusterletConfig struct { ExternalManagedKubeConfigRegistrationSecret string ExternalManagedKubeConfigWorkSecret string InstallMode operatorapiv1.InstallMode + + RegistrationFeatureGates []string } // managedClusterClients holds variety of kube client for managed cluster @@ -222,6 +226,26 @@ func (n *klusterletController) sync(ctx context.Context, controllerContext facto appliedManifestWorkClient: n.appliedManifestWorkClient, } + // If there are some invalid feature gates of registration, will output condition `InvalidRegistrationFeatureGates` in Klusterlet. + if klusterlet.Spec.RegistrationConfiguration != nil && len(klusterlet.Spec.RegistrationConfiguration.FeatureGates) > 0 { + featureGateArgs, invalidFeatureGates := helpers.FeatureGatesArgs( + klusterlet.Spec.RegistrationConfiguration.FeatureGates, helpers.ComponentSpokeKey) + if len(invalidFeatureGates) == 0 { + config.RegistrationFeatureGates = featureGateArgs + _, _, _ = helpers.UpdateKlusterletStatus(ctx, n.klusterletClient, klusterletName, helpers.UpdateKlusterletConditionFn(metav1.Condition{ + Type: spokeRegistrationFeatureGatesInvalid, Status: metav1.ConditionTrue, Reason: "FeatureGatesAllValid", + Message: fmt.Sprintf("Registration feature gates of klusterlet are all valid"), + })) + } else { + invalidFGErr := fmt.Errorf("There are some invalid feature gates of registration: %v ", invalidFeatureGates) + _, _, _ = helpers.UpdateKlusterletStatus(ctx, n.klusterletClient, klusterletName, helpers.UpdateKlusterletConditionFn(metav1.Condition{ + Type: spokeRegistrationFeatureGatesInvalid, Status: metav1.ConditionFalse, Reason: "InvalidFeatureGatesExisting", + Message: invalidFGErr.Error(), + })) + return invalidFGErr + } + } + // TODO: remove this when detached mode is not used in klusterlet if config.InstallMode == operatorapiv1.InstallModeDetached { config.InstallMode = operatorapiv1.InstallModeHosted diff --git a/pkg/operators/klusterlet/controllers/klusterletcontroller/klusterlet_controller_test.go b/pkg/operators/klusterlet/controllers/klusterletcontroller/klusterlet_controller_test.go index 927419610..186a59939 100644 --- a/pkg/operators/klusterlet/controllers/klusterletcontroller/klusterlet_controller_test.go +++ b/pkg/operators/klusterlet/controllers/klusterletcontroller/klusterlet_controller_test.go @@ -81,6 +81,14 @@ func newKlusterlet(name, namespace, clustername string) *opratorapiv1.Klusterlet ClusterName: clustername, Namespace: namespace, ExternalServerURLs: []opratorapiv1.ServerURL{}, + RegistrationConfiguration: &opratorapiv1.RegistrationConfiguration{ + FeatureGates: []opratorapiv1.FeatureGate{ + { + Feature: "AddonManagement", + Mode: "Enable", + }, + }, + }, }, } } @@ -444,15 +452,18 @@ func TestSyncDeploy(t *testing.T) { } operatorAction := controller.operatorClient.Actions() - if len(operatorAction) != 2 { - t.Errorf("Expect 2 actions in the sync loop, actual %#v", operatorAction) + if len(operatorAction) != 4 { + t.Errorf("Expect 4 actions in the sync loop, actual %#v", operatorAction) } testinghelper.AssertGet(t, operatorAction[0], "operator.open-cluster-management.io", "v1", "klusterlets") testinghelper.AssertAction(t, operatorAction[1], "update") + testinghelper.AssertGet(t, operatorAction[2], "operator.open-cluster-management.io", "v1", "klusterlets") + testinghelper.AssertAction(t, operatorAction[3], "update") testinghelper.AssertOnlyConditions( - t, operatorAction[1].(clienttesting.UpdateActionImpl).Object, - testinghelper.NamedCondition(klusterletApplied, "KlusterletApplied", metav1.ConditionTrue)) + t, operatorAction[3].(clienttesting.UpdateActionImpl).Object, + testinghelper.NamedCondition(klusterletApplied, "KlusterletApplied", metav1.ConditionTrue), + testinghelper.NamedCondition(spokeRegistrationFeatureGatesInvalid, "FeatureGatesAllValid", metav1.ConditionTrue)) } // TestSyncDeployHosted test deployment of klusterlet components in hosted mode @@ -538,21 +549,24 @@ func TestSyncDeployHosted(t *testing.T) { klog.Infof("operator actions, verb:%v \t resource:%v \t namespace:%v", action.GetVerb(), action.GetResource(), action.GetNamespace()) } - if len(operatorAction) != 4 { - t.Errorf("Expect 4 actions in the sync loop, actual %#v", len(operatorAction)) + if len(operatorAction) != 6 { + t.Errorf("Expect 6 actions in the sync loop, actual %#v", len(operatorAction)) } testinghelper.AssertGet(t, operatorAction[0], "operator.open-cluster-management.io", "v1", "klusterlets") testinghelper.AssertAction(t, operatorAction[1], "update") testinghelper.AssertGet(t, operatorAction[2], "operator.open-cluster-management.io", "v1", "klusterlets") testinghelper.AssertAction(t, operatorAction[3], "update") + testinghelper.AssertGet(t, operatorAction[4], "operator.open-cluster-management.io", "v1", "klusterlets") + testinghelper.AssertAction(t, operatorAction[5], "update") conditionReady := testinghelper.NamedCondition(klusterletReadyToApply, "KlusterletPrepared", metav1.ConditionTrue) conditionApplied := testinghelper.NamedCondition(klusterletApplied, "KlusterletApplied", metav1.ConditionTrue) + conditionFeaturesValid := testinghelper.NamedCondition(spokeRegistrationFeatureGatesInvalid, "FeatureGatesAllValid", metav1.ConditionTrue) testinghelper.AssertOnlyConditions( - t, operatorAction[1].(clienttesting.UpdateActionImpl).Object, conditionReady) + t, operatorAction[3].(clienttesting.UpdateActionImpl).Object, conditionReady, conditionFeaturesValid) testinghelper.AssertOnlyConditions( - t, operatorAction[3].(clienttesting.UpdateActionImpl).Object, conditionReady, conditionApplied) + t, operatorAction[5].(clienttesting.UpdateActionImpl).Object, conditionReady, conditionApplied, conditionFeaturesValid) } // TestSyncDelete test cleanup hub deploy @@ -833,13 +847,15 @@ func TestClusterNameChange(t *testing.T) { assertRegistrationDeployment(t, controller.kubeClient.Actions(), "create", "", "cluster1", 1) operatorAction := controller.operatorClient.Actions() - if len(operatorAction) != 2 { - t.Errorf("Expect 2 actions in the sync loop, actual %#v", operatorAction) + if len(operatorAction) != 4 { + t.Errorf("Expect 4 actions in the sync loop, actual %#v", operatorAction) } testinghelper.AssertGet(t, operatorAction[0], "operator.open-cluster-management.io", "v1", "klusterlets") testinghelper.AssertAction(t, operatorAction[1], "update") - updatedKlusterlet := operatorAction[1].(clienttesting.UpdateActionImpl).Object.(*opratorapiv1.Klusterlet) + testinghelper.AssertGet(t, operatorAction[2], "operator.open-cluster-management.io", "v1", "klusterlets") + testinghelper.AssertAction(t, operatorAction[3], "update") + updatedKlusterlet := operatorAction[3].(clienttesting.UpdateActionImpl).Object.(*opratorapiv1.Klusterlet) testinghelper.AssertOnlyGenerationStatuses( t, updatedKlusterlet, testinghelper.NamedDeploymentGenerationStatus("klusterlet-registration-agent", "testns", 0), @@ -979,15 +995,18 @@ func TestDeployOnKube111(t *testing.T) { } operatorAction := controller.operatorClient.Actions() - if len(operatorAction) != 2 { - t.Errorf("Expect 2 actions in the sync loop, actual %#v", operatorAction) + if len(operatorAction) != 4 { + t.Errorf("Expect 4 actions in the sync loop, actual %#v", operatorAction) } testinghelper.AssertGet(t, operatorAction[0], "operator.open-cluster-management.io", "v1", "klusterlets") testinghelper.AssertAction(t, operatorAction[1], "update") + testinghelper.AssertGet(t, operatorAction[2], "operator.open-cluster-management.io", "v1", "klusterlets") + testinghelper.AssertAction(t, operatorAction[3], "update") testinghelper.AssertOnlyConditions( - t, operatorAction[1].(clienttesting.UpdateActionImpl).Object, - testinghelper.NamedCondition(klusterletApplied, "KlusterletApplied", metav1.ConditionTrue)) + t, operatorAction[3].(clienttesting.UpdateActionImpl).Object, + testinghelper.NamedCondition(klusterletApplied, "KlusterletApplied", metav1.ConditionTrue), + testinghelper.NamedCondition(spokeRegistrationFeatureGatesInvalid, "FeatureGatesAllValid", metav1.ConditionTrue)) // Delete the klusterlet now := metav1.Now() diff --git a/test/integration/klusterlet_test.go b/test/integration/klusterlet_test.go index 35b845c58..42488f1be 100644 --- a/test/integration/klusterlet_test.go +++ b/test/integration/klusterlet_test.go @@ -70,6 +70,12 @@ var _ = ginkgo.Describe("Klusterlet", func() { }, ClusterName: "testcluster", Namespace: klusterletNamespace, + RegistrationConfiguration: &operatorapiv1.RegistrationConfiguration{FeatureGates: []operatorapiv1.FeatureGate{ + { + Feature: "AddonManagement", + Mode: "Enable", + }, + }}, }, } diff --git a/vendor/modules.txt b/vendor/modules.txt index 1abf1e863..a18886d77 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1190,7 +1190,7 @@ k8s.io/utils/path k8s.io/utils/pointer k8s.io/utils/strings/slices k8s.io/utils/trace -# open-cluster-management.io/api v0.7.1-0.20220530034043-2b50efdde1de +# open-cluster-management.io/api v0.7.1-0.20220609033924-5cc58e815c1a ## explicit; go 1.18 open-cluster-management.io/api/addon/v1alpha1 open-cluster-management.io/api/client/addon/clientset/versioned @@ -1219,6 +1219,7 @@ open-cluster-management.io/api/client/work/clientset/versioned/typed/work/v1/fak open-cluster-management.io/api/cluster/v1 open-cluster-management.io/api/cluster/v1alpha1 open-cluster-management.io/api/cluster/v1beta1 +open-cluster-management.io/api/feature open-cluster-management.io/api/operator/v1 open-cluster-management.io/api/work/v1 # sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.30 diff --git a/vendor/open-cluster-management.io/api/feature/feature.go b/vendor/open-cluster-management.io/api/feature/feature.go new file mode 100644 index 000000000..9975797a5 --- /dev/null +++ b/vendor/open-cluster-management.io/api/feature/feature.go @@ -0,0 +1,58 @@ +package feature + +import ( + "k8s.io/component-base/featuregate" +) + +const ( + // Every feature gate should add method here following this template: + // + // // owner: @username + // // alpha: v1.X + // MyFeature featuregate.Feature = "MyFeature" + + // ClusterClaim will start a new controller in the spoke-agent to manage the cluster-claim + // resources in the managed cluster. + // + // The cluster-claim controller is majorly for collecting claims and updating claims field + // in managedcluster status. When it exceeds the limit specified by "--max-custom-cluster-claims", + // the extra claims will be truncated. + // + // If it is disabled, the user will see empty claims field in managedcluster status. The + // deployer who disable the feature may need to update claim field in managed cluster status + // itself to avoid impact to users. + ClusterClaim featuregate.Feature = "ClusterClaim" + + // AddonManagement will start new controllers in the spoke-agent to manage the managed cluster addons + // registration and maintains the status of managed cluster addons through watching their leases. + AddonManagement featuregate.Feature = "AddonManagement" + + // DefaultClusterSet will make registration hub controller to maintain a default cluster set. All clusters + // without clusterset label will be automatically added into the default cluster set by adding a label + // "cluster.open-cluster-management.io/clusterset=default" to the clusters. + DefaultClusterSet featuregate.Feature = "DefaultClusterSet" + + // V1beta1CSRAPICompatibility will make the spoke registration agent to issue CSR requests + // via V1beta1 api, so that registration agent can still manage the certificate rotation for the + // ManagedCluster and ManagedClusterAddon. + // Note that kubernetes release [1.12, 1.18)'s beta CSR api doesn't have the "signerName" field which + // means that all the approved CSR objects will be signed by the built-in CSR controller in + // kube-controller-manager. + V1beta1CSRAPICompatibility featuregate.Feature = "V1beta1CSRAPICompatibility" +) + +// DefaultSpokeRegistrationFeatureGates consists of all known ocm-registration +// feature keys for registration agent. To add a new feature, define a key for it above and +// add it here. +var DefaultSpokeRegistrationFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{ + ClusterClaim: {Default: true, PreRelease: featuregate.Beta}, + AddonManagement: {Default: false, PreRelease: featuregate.Alpha}, + V1beta1CSRAPICompatibility: {Default: false, PreRelease: featuregate.Alpha}, +} + +// DefaultHubRegistrationFeatureGates consists of all known ocm-registration +// feature keys for registration hub controller. To add a new feature, define a key for it above and +// add it here. +var DefaultHubRegistrationFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{ + DefaultClusterSet: {Default: false, PreRelease: featuregate.Alpha}, +} diff --git a/vendor/open-cluster-management.io/api/work/v1/0000_00_work.open-cluster-management.io_manifestworks.crd.yaml b/vendor/open-cluster-management.io/api/work/v1/0000_00_work.open-cluster-management.io_manifestworks.crd.yaml index 1182080b4..4100bf2ac 100644 --- a/vendor/open-cluster-management.io/api/work/v1/0000_00_work.open-cluster-management.io_manifestworks.crd.yaml +++ b/vendor/open-cluster-management.io/api/work/v1/0000_00_work.open-cluster-management.io_manifestworks.crd.yaml @@ -109,11 +109,10 @@ spec: description: ManifestConfigOption represents the configurations of a manifest defined in workload field. type: object required: - - feedbackRules - resourceIdentifier properties: feedbackRules: - description: FeedbackRules defines what resource status field should be returned. + description: FeedbackRules defines what resource status field should be returned. If it is not set or empty, no feedback rules will be honored. type: array items: type: object @@ -163,6 +162,32 @@ spec: resource: description: Resource is the resource name of the Kubernetes resource. type: string + updateStrategy: + description: UpdateStrategy defines the strategy to update this manifest. UpdateStrategy is Update if it is not set, optional + type: object + required: + - type + properties: + serverSideApply: + description: serverSideApply defines the configuration for server side apply. It is honored only when type of updateStrategy is ServerSideApply + type: object + properties: + fieldManager: + description: FieldManager is the manager to apply the resource. It is work-agent by default, but can be other name with work-agent as the prefix. + type: string + default: work-agent + pattern: ^work-agent + force: + description: Force represents to force apply the manifest. + type: boolean + type: + description: type defines the strategy to update this manifest, default value is Update. Update type means to update resource by an update call. CreateOnly type means do not update resource based on current manifest. ServerSideApply type means to update resource using server side apply with work-controller as the field manager. If there is conflict, the related Applied condition of manifest will be in the status of False with the reason of ApplyConflict. + type: string + default: Update + enum: + - Update + - CreateOnly + - ServerSideApply workload: description: Workload represents the manifest workload to be deployed on a managed cluster. type: object diff --git a/vendor/open-cluster-management.io/api/work/v1/types.go b/vendor/open-cluster-management.io/api/work/v1/types.go index cb29c2499..9216d8bbd 100644 --- a/vendor/open-cluster-management.io/api/work/v1/types.go +++ b/vendor/open-cluster-management.io/api/work/v1/types.go @@ -86,10 +86,15 @@ type ManifestConfigOption struct { // +required ResourceIdentifier ResourceIdentifier `json:"resourceIdentifier"` - // FeedbackRules defines what resource status field should be returned. - // +kubebuilder:validation:Required - // +required - FeedbackRules []FeedbackRule `json:"feedbackRules"` + // FeedbackRules defines what resource status field should be returned. If it is not set or empty, + // no feedback rules will be honored. + // +optional + FeedbackRules []FeedbackRule `json:"feedbackRules,omitempty"` + + // UpdateStrategy defines the strategy to update this manifest. UpdateStrategy is Update + // if it is not set, + // optional + UpdateStrategy *UpdateStrategy `json:"updateStrategy"` } // ManifestWorkExecutor is the executor that applies the resources to the managed cluster. i.e. the @@ -143,6 +148,56 @@ type ManifestWorkSubjectServiceAccount struct { Name string `json:"name"` } +// UpdateStrategy defines the strategy to update this manifest +type UpdateStrategy struct { + // type defines the strategy to update this manifest, default value is Update. + // Update type means to update resource by an update call. + // CreateOnly type means do not update resource based on current manifest. + // ServerSideApply type means to update resource using server side apply with work-controller as the field manager. + // If there is conflict, the related Applied condition of manifest will be in the status of False with the + // reason of ApplyConflict. + // +kubebuilder:default=Update + // +kubebuilder:validation:Enum=Update;CreateOnly;ServerSideApply + // +kubebuilder:validation:Required + // +required + Type UpdateStrategyType `json:"type,omitempty"` + + // serverSideApply defines the configuration for server side apply. It is honored only when + // type of updateStrategy is ServerSideApply + // +optional + ServerSideApply *ServerSideApplyConfig `json:"serverSideApply,omitempty"` +} + +type UpdateStrategyType string + +const ( + // Update type means to update resource by an update call. + UpdateStrategyTypeUpdate UpdateStrategyType = "Update" + + // CreateOnly type means do not update resource based on current manifest. This should be used only when + // ServerSideApply type is not support on the spoke, and the user on hub would like some other controller + // on the spoke to own the control of the resource. + UpdateStrategyTypeCreateOnly UpdateStrategyType = "CreateOnly" + + // ServerSideApply type means to update resource using server side apply with work-controller as the field manager. + // If there is conflict, the related Applied condition of manifest will be in the status of False with the + // reason of ApplyConflict. This type allows another controller on the spoke to control certain field of the resource. + UpdateStrategyTypeServerSideApply UpdateStrategyType = "ServerSideApply" +) + +type ServerSideApplyConfig struct { + // Force represents to force apply the manifest. + // +optional + Force bool `json:"force"` + + // FieldManager is the manager to apply the resource. It is work-agent by default, but can be other name with work-agent + // as the prefix. + // +kubebuilder:default=work-agent + // +kubebuilder:validation:Pattern=`^work-agent` + // +optional + FieldManager string `json:"fieldManager,omitempty"` +} + type FeedbackRule struct { // Type defines the option of how status can be returned. // It can be jsonPaths or wellKnownStatus. diff --git a/vendor/open-cluster-management.io/api/work/v1/zz_generated.deepcopy.go b/vendor/open-cluster-management.io/api/work/v1/zz_generated.deepcopy.go index 084f39e20..3fb3f47e4 100644 --- a/vendor/open-cluster-management.io/api/work/v1/zz_generated.deepcopy.go +++ b/vendor/open-cluster-management.io/api/work/v1/zz_generated.deepcopy.go @@ -284,6 +284,11 @@ func (in *ManifestConfigOption) DeepCopyInto(out *ManifestConfigOption) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.UpdateStrategy != nil { + in, out := &in.UpdateStrategy, &out.UpdateStrategy + *out = new(UpdateStrategy) + (*in).DeepCopyInto(*out) + } return } @@ -585,6 +590,22 @@ func (in *SelectivelyOrphan) DeepCopy() *SelectivelyOrphan { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ServerSideApplyConfig) DeepCopyInto(out *ServerSideApplyConfig) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ServerSideApplyConfig. +func (in *ServerSideApplyConfig) DeepCopy() *ServerSideApplyConfig { + if in == nil { + return nil + } + out := new(ServerSideApplyConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *StatusFeedbackResult) DeepCopyInto(out *StatusFeedbackResult) { *out = *in @@ -607,3 +628,24 @@ func (in *StatusFeedbackResult) DeepCopy() *StatusFeedbackResult { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *UpdateStrategy) DeepCopyInto(out *UpdateStrategy) { + *out = *in + if in.ServerSideApply != nil { + in, out := &in.ServerSideApply, &out.ServerSideApply + *out = new(ServerSideApplyConfig) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UpdateStrategy. +func (in *UpdateStrategy) DeepCopy() *UpdateStrategy { + if in == nil { + return nil + } + out := new(UpdateStrategy) + in.DeepCopyInto(out) + return out +} diff --git a/vendor/open-cluster-management.io/api/work/v1/zz_generated.swagger_doc_generated.go b/vendor/open-cluster-management.io/api/work/v1/zz_generated.swagger_doc_generated.go index e9b2bd566..24f8a4cbe 100644 --- a/vendor/open-cluster-management.io/api/work/v1/zz_generated.swagger_doc_generated.go +++ b/vendor/open-cluster-management.io/api/work/v1/zz_generated.swagger_doc_generated.go @@ -131,7 +131,8 @@ func (ManifestCondition) SwaggerDoc() map[string]string { var map_ManifestConfigOption = map[string]string{ "": "ManifestConfigOption represents the configurations of a manifest defined in workload field.", "resourceIdentifier": "ResourceIdentifier represents the group, resource, name and namespace of a resoure. iff this refers to a resource not created by this manifest work, the related rules will not be executed.", - "feedbackRules": "FeedbackRules defines what resource status field should be returned.", + "feedbackRules": "FeedbackRules defines what resource status field should be returned. If it is not set or empty, no feedback rules will be honored.", + "updateStrategy": "UpdateStrategy defines the strategy to update this manifest. UpdateStrategy is Update if it is not set, optional", } func (ManifestConfigOption) SwaggerDoc() map[string]string { @@ -263,6 +264,15 @@ func (SelectivelyOrphan) SwaggerDoc() map[string]string { return map_SelectivelyOrphan } +var map_ServerSideApplyConfig = map[string]string{ + "force": "Force represents to force apply the manifest.", + "fieldManager": "FieldManager is the manager to apply the resource. It is work-agent by default, but can be other name with work-agent as the prefix.", +} + +func (ServerSideApplyConfig) SwaggerDoc() map[string]string { + return map_ServerSideApplyConfig +} + var map_StatusFeedbackResult = map[string]string{ "": "StatusFeedbackResult represents the values of the feild synced back defined in statusFeedbacks", "values": "Values represents the synced value of the interested field.", @@ -272,4 +282,14 @@ func (StatusFeedbackResult) SwaggerDoc() map[string]string { return map_StatusFeedbackResult } +var map_UpdateStrategy = map[string]string{ + "": "UpdateStrategy defines the strategy to update this manifest", + "type": "type defines the strategy to update this manifest, default value is Update. Update type means to update resource by an update call. CreateOnly type means do not update resource based on current manifest. ServerSideApply type means to update resource using server side apply with work-controller as the field manager. If there is conflict, the related Applied condition of manifest will be in the status of False with the reason of ApplyConflict.", + "serverSideApply": "serverSideApply defines the configuration for server side apply. It is honored only when type of updateStrategy is ServerSideApply", +} + +func (UpdateStrategy) SwaggerDoc() map[string]string { + return map_UpdateStrategy +} + // AUTO-GENERATED FUNCTIONS END HERE