diff --git a/README.md b/README.md index ce03d03dd..9053a2634 100644 --- a/README.md +++ b/README.md @@ -160,8 +160,7 @@ We mainly provide deployment in two scenarios: selector: app: cluster-manager-registration-webhook ports: - - port: 443 - targetPort: 6443 + - port: 9443 nodePort: 30443 --- apiVersion: v1 @@ -174,8 +173,7 @@ We mainly provide deployment in two scenarios: selector: app: cluster-manager-work-webhook ports: - - port: 443 - targetPort: 6443 + - port: 9443 nodePort: 31443 EOF ``` diff --git a/hack/patch/0000_00_clusters.open-cluster-management.io_managedclustersets.crd.yaml.yaml-patch b/hack/patch/0000_00_clusters.open-cluster-management.io_managedclustersets.crd.yaml.yaml-patch index 9661f8f6d..3eda033f0 100644 --- a/hack/patch/0000_00_clusters.open-cluster-management.io_managedclustersets.crd.yaml.yaml-patch +++ b/hack/patch/0000_00_clusters.open-cluster-management.io_managedclustersets.crd.yaml.yaml-patch @@ -9,7 +9,7 @@ namespace: {{ .ClusterManagerNamespace }} name: cluster-manager-registration-webhook path: /convert - port: 9443 + port: {{.RegistrationWebhook.Port}} caBundle: {{ .RegistrationAPIServiceCABundle }} conversionReviewVersions: - v1beta1 diff --git a/manifests/cluster-manager/hub/0000_00_clusters.open-cluster-management.io_managedclustersets.crd.yaml b/manifests/cluster-manager/hub/0000_00_clusters.open-cluster-management.io_managedclustersets.crd.yaml index 893ec0639..cad76cd6c 100644 --- a/manifests/cluster-manager/hub/0000_00_clusters.open-cluster-management.io_managedclustersets.crd.yaml +++ b/manifests/cluster-manager/hub/0000_00_clusters.open-cluster-management.io_managedclustersets.crd.yaml @@ -12,7 +12,7 @@ spec: name: cluster-manager-registration-webhook namespace: {{ .ClusterManagerNamespace }} path: /convert - port: 9443 + port: {{.RegistrationWebhook.Port}} conversionReviewVersions: - v1beta1 - v1beta2 diff --git a/manifests/cluster-manager/hub/cluster-manager-registration-webhook-apiservice.yaml b/manifests/cluster-manager/hub/cluster-manager-registration-webhook-apiservice.yaml index 9dfc3ce1f..72f1391b8 100644 --- a/manifests/cluster-manager/hub/cluster-manager-registration-webhook-apiservice.yaml +++ b/manifests/cluster-manager/hub/cluster-manager-registration-webhook-apiservice.yaml @@ -8,11 +8,6 @@ spec: service: name: cluster-manager-registration-webhook namespace: {{ .ClusterManagerNamespace }} - {{if eq .RegistrationWebhook.Port 0}} - port: 443 - {{else}} port: {{.RegistrationWebhook.Port}} - {{end}} - caBundle: {{ .RegistrationAPIServiceCABundle }} groupPriorityMinimum: 10000 versionPriority: 20 diff --git a/manifests/cluster-manager/hub/cluster-manager-registration-webhook-clustersetbinding-validatingconfiguration-v1beta1.yaml b/manifests/cluster-manager/hub/cluster-manager-registration-webhook-clustersetbinding-validatingconfiguration-v1beta1.yaml new file mode 100644 index 000000000..f6c533d03 --- /dev/null +++ b/manifests/cluster-manager/hub/cluster-manager-registration-webhook-clustersetbinding-validatingconfiguration-v1beta1.yaml @@ -0,0 +1,27 @@ +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: managedclustersetbindingv1beta1validators.admission.cluster.open-cluster-management.io +webhooks: +- name: managedclustersetbindingv1beta1validators.admission.cluster.open-cluster-management.io + failurePolicy: Fail + clientConfig: + service: + namespace: {{ .ClusterManagerNamespace }} + name: cluster-manager-registration-webhook + path: /validate-cluster-open-cluster-management-io-v1beta1-managedclustersetbinding + port: {{.RegistrationWebhook.Port}} + caBundle: {{ .RegistrationAPIServiceCABundle }} + rules: + - operations: + - CREATE + - UPDATE + apiGroups: + - cluster.open-cluster-management.io + apiVersions: + - v1beta1 + resources: + - managedclustersetbindings + admissionReviewVersions: ["v1beta1","v1"] + sideEffects: None + timeoutSeconds: 10 diff --git a/manifests/cluster-manager/hub/cluster-manager-registration-webhook-clustersetbinding-validatingconfiguration.yaml b/manifests/cluster-manager/hub/cluster-manager-registration-webhook-clustersetbinding-validatingconfiguration.yaml index 46d981753..52090d8b7 100644 --- a/manifests/cluster-manager/hub/cluster-manager-registration-webhook-clustersetbinding-validatingconfiguration.yaml +++ b/manifests/cluster-manager/hub/cluster-manager-registration-webhook-clustersetbinding-validatingconfiguration.yaml @@ -7,10 +7,11 @@ webhooks: failurePolicy: Fail clientConfig: service: - # reach the webhook via the registered aggregated API - namespace: default - name: kubernetes - path: /apis/admission.cluster.open-cluster-management.io/v1/managedclustersetbindingvalidators + namespace: {{ .ClusterManagerNamespace }} + name: cluster-manager-registration-webhook + path: /validate-cluster-open-cluster-management-io-v1beta2-managedclustersetbinding + port: {{.RegistrationWebhook.Port}} + caBundle: {{ .RegistrationAPIServiceCABundle }} rules: - operations: - CREATE @@ -18,9 +19,9 @@ webhooks: apiGroups: - cluster.open-cluster-management.io apiVersions: - - "*" + - v1beta2 resources: - managedclustersetbindings - admissionReviewVersions: ["v1beta1"] + admissionReviewVersions: ["v1beta1","v1"] sideEffects: None timeoutSeconds: 10 diff --git a/manifests/cluster-manager/hub/cluster-manager-registration-webhook-mutatingconfiguration.yaml b/manifests/cluster-manager/hub/cluster-manager-registration-webhook-mutatingconfiguration.yaml index d9a942bb9..1524f4c96 100644 --- a/manifests/cluster-manager/hub/cluster-manager-registration-webhook-mutatingconfiguration.yaml +++ b/manifests/cluster-manager/hub/cluster-manager-registration-webhook-mutatingconfiguration.yaml @@ -7,10 +7,11 @@ webhooks: failurePolicy: Fail clientConfig: service: - # reach the webhook via the registered aggregated API - namespace: default - name: kubernetes - path: /apis/admission.cluster.open-cluster-management.io/v1/managedclustermutators + namespace: {{ .ClusterManagerNamespace }} + name: cluster-manager-registration-webhook + path: /mutate-cluster-open-cluster-management-io-v1-managedcluster + port: {{.RegistrationWebhook.Port}} + caBundle: {{ .RegistrationAPIServiceCABundle }} rules: - operations: - CREATE @@ -21,6 +22,6 @@ webhooks: - "*" resources: - managedclusters - admissionReviewVersions: ["v1beta1"] + admissionReviewVersions: ["v1beta1","v1"] sideEffects: None timeoutSeconds: 10 diff --git a/manifests/cluster-manager/hub/cluster-manager-registration-webhook-service.yaml b/manifests/cluster-manager/hub/cluster-manager-registration-webhook-service.yaml index fc7b22470..19bf0b4ee 100644 --- a/manifests/cluster-manager/hub/cluster-manager-registration-webhook-service.yaml +++ b/manifests/cluster-manager/hub/cluster-manager-registration-webhook-service.yaml @@ -10,6 +10,6 @@ spec: - name: webhook-server port: 443 targetPort: 6443 - - name: conversion-webhook + - name: webhook port: 9443 targetPort: 9443 diff --git a/manifests/cluster-manager/hub/cluster-manager-registration-webhook-validatingconfiguration.yaml b/manifests/cluster-manager/hub/cluster-manager-registration-webhook-validatingconfiguration.yaml index f5ce3f4d1..7e4f8869a 100644 --- a/manifests/cluster-manager/hub/cluster-manager-registration-webhook-validatingconfiguration.yaml +++ b/manifests/cluster-manager/hub/cluster-manager-registration-webhook-validatingconfiguration.yaml @@ -7,10 +7,11 @@ webhooks: failurePolicy: Fail clientConfig: service: - # reach the webhook via the registered aggregated API - namespace: default - name: kubernetes - path: /apis/admission.cluster.open-cluster-management.io/v1/managedclustervalidators + namespace: {{ .ClusterManagerNamespace }} + name: cluster-manager-registration-webhook + path: /validate-cluster-open-cluster-management-io-v1-managedcluster + port: {{.RegistrationWebhook.Port}} + caBundle: {{ .RegistrationAPIServiceCABundle }} rules: - operations: - CREATE @@ -21,6 +22,6 @@ webhooks: - "*" resources: - managedclusters - admissionReviewVersions: ["v1beta1"] + admissionReviewVersions: ["v1beta1","v1"] sideEffects: None timeoutSeconds: 10 diff --git a/manifests/cluster-manager/hub/cluster-manager-work-webhook-apiservice.yaml b/manifests/cluster-manager/hub/cluster-manager-work-webhook-apiservice.yaml index b6ac60038..9e8f2b4c5 100644 --- a/manifests/cluster-manager/hub/cluster-manager-work-webhook-apiservice.yaml +++ b/manifests/cluster-manager/hub/cluster-manager-work-webhook-apiservice.yaml @@ -8,11 +8,6 @@ spec: service: name: cluster-manager-work-webhook namespace: {{ .ClusterManagerNamespace }} - {{if eq .WorkWebhook.Port 0}} - port: 443 - {{else}} port: {{.WorkWebhook.Port}} - {{end}} - caBundle: {{ .WorkAPIServiceCABundle }} groupPriorityMinimum: 10000 versionPriority: 20 diff --git a/manifests/cluster-manager/hub/cluster-manager-work-webhook-service.yaml b/manifests/cluster-manager/hub/cluster-manager-work-webhook-service.yaml index eeb450047..f45b06ea8 100644 --- a/manifests/cluster-manager/hub/cluster-manager-work-webhook-service.yaml +++ b/manifests/cluster-manager/hub/cluster-manager-work-webhook-service.yaml @@ -7,5 +7,9 @@ spec: selector: app: {{ .ClusterManagerName }}-work-webhook ports: - - port: 443 + - name: webhook-server + port: 443 targetPort: 6443 + - name: webhook + port: 9443 + targetPort: 9443 diff --git a/manifests/cluster-manager/hub/cluster-manager-work-webhook-validatingconfiguration.yaml b/manifests/cluster-manager/hub/cluster-manager-work-webhook-validatingconfiguration.yaml index 293ea30f0..9645ea384 100644 --- a/manifests/cluster-manager/hub/cluster-manager-work-webhook-validatingconfiguration.yaml +++ b/manifests/cluster-manager/hub/cluster-manager-work-webhook-validatingconfiguration.yaml @@ -7,10 +7,11 @@ webhooks: failurePolicy: Fail clientConfig: service: - # reach the webhook via the registered aggregated API - namespace: default - name: kubernetes - path: /apis/admission.work.open-cluster-management.io/v1/manifestworkvalidators + namespace: {{ .ClusterManagerNamespace }} + name: cluster-manager-work-webhook + path: /validate-work-open-cluster-management-io-v1-manifestwork + port: {{.RegistrationWebhook.Port}} + caBundle: {{ .RegistrationAPIServiceCABundle }} rules: - operations: - CREATE @@ -21,6 +22,6 @@ webhooks: - "*" resources: - manifestworks - admissionReviewVersions: ["v1beta1"] + admissionReviewVersions: ["v1beta1","v1"] sideEffects: None timeoutSeconds: 10 diff --git a/manifests/cluster-manager/management/cluster-manager-registration-webhook-deployment.yaml b/manifests/cluster-manager/management/cluster-manager-registration-webhook-deployment.yaml index 761b97a0d..708a3a419 100644 --- a/manifests/cluster-manager/management/cluster-manager-registration-webhook-deployment.yaml +++ b/manifests/cluster-manager/management/cluster-manager-registration-webhook-deployment.yaml @@ -94,13 +94,22 @@ spec: name: kubeconfig readOnly: true {{ end }} - {{ if not .HostedMode }} - - name: {{ .ClusterManagerName }}-registration-conversion-webhook + - name: {{ .ClusterManagerName }}-webhook image: {{ .RegistrationImage }} args: - /registration - "webhook-server" - "port=9443" + {{ if gt (len .RegistrationFeatureGates) 0 }} + {{range .RegistrationFeatureGates}} + - {{ . }} + {{ end }} + {{ else }} + - "--feature-gates=DefaultClusterSet=true" + {{ end }} + {{ if .HostedMode }} + - "--kubeconfig=/var/run/secrets/hub/kubeconfig" + {{ end }} imagePullPolicy: Always resources: requests: @@ -133,7 +142,11 @@ spec: - mountPath: /tmp/k8s-webhook-server/serving-certs name: webhook-secret readOnly: true - {{ end }} + {{ if .HostedMode }} + - mountPath: /var/run/secrets/hub + name: kubeconfig + readOnly: true + {{ end }} volumes: - name: webhook-secret secret: diff --git a/manifests/cluster-manager/management/cluster-manager-work-webhook-deployment.yaml b/manifests/cluster-manager/management/cluster-manager-work-webhook-deployment.yaml index c6e414c49..dba630767 100644 --- a/manifests/cluster-manager/management/cluster-manager-work-webhook-deployment.yaml +++ b/manifests/cluster-manager/management/cluster-manager-work-webhook-deployment.yaml @@ -40,10 +40,60 @@ spec: serviceAccountName: {{ .ClusterManagerName }}-work-webhook-sa {{ end }} containers: + - name: {{ .ClusterManagerName }}-webhook + image: {{ .WorkImage }} + args: + - /work + - "webhook-server" + - "port=9443" + {{ if gt (len .WorkFeatureGates) 0 }} + {{range .WorkFeatureGates}} + - {{ . }} + {{ end }} + {{ end }} + {{ if .HostedMode }} + - "--kubeconfig=/var/run/secrets/hub/kubeconfig" + {{ end }} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + privileged: false + runAsNonRoot: true + livenessProbe: + httpGet: + path: /healthz + scheme: HTTP + port: 8000 + initialDelaySeconds: 2 + periodSeconds: 10 + readinessProbe: + httpGet: + path: /healthz + scheme: HTTP + port: 8000 + initialDelaySeconds: 2 + resources: + requests: + cpu: 2m + memory: 16Mi + ports: + - containerPort: 9443 + protocol: TCP + volumeMounts: + - mountPath: /tmp/k8s-webhook-server/serving-certs + name: webhook-secret + readOnly: true + {{ if .HostedMode }} + - mountPath: /var/run/secrets/hub + name: kubeconfig + readOnly: true + {{ end }} - name: {{ .ClusterManagerName }}-work-webhook-sa image: {{ .WorkImage }} args: - - "/work" + - /work - "webhook" {{ if gt (len .WorkFeatureGates) 0 }} {{range .WorkFeatureGates}} @@ -83,6 +133,9 @@ spec: requests: cpu: 2m memory: 16Mi + ports: + - containerPort: 9443 + protocol: TCP volumeMounts: - name: webhook-secret mountPath: "/serving-cert" @@ -101,4 +154,3 @@ spec: secret: secretName: {{ .ClusterManagerName }}-work-webhook-sa-kubeconfig {{ end }} - diff --git a/pkg/helpers/error.go b/pkg/helpers/error.go new file mode 100644 index 000000000..d2cdd2f40 --- /dev/null +++ b/pkg/helpers/error.go @@ -0,0 +1,22 @@ +package helpers + +import ( + "fmt" + "time" +) + +type RequeueError struct { + RequeueTime time.Duration + Message string +} + +func (r RequeueError) Error() string { + return fmt.Sprintf(r.Message+", Requeue time: %v", r.RequeueTime) +} + +func NewRequeueError(msg string, requeueTime time.Duration) RequeueError { + return RequeueError{ + RequeueTime: requeueTime, + Message: msg, + } +} diff --git a/pkg/helpers/helpers.go b/pkg/helpers/helpers.go index 0cd2f02ee..b8be56076 100644 --- a/pkg/helpers/helpers.go +++ b/pkg/helpers/helpers.go @@ -273,8 +273,7 @@ func ApplyValidatingWebhookConfiguration( if !*modified { return existing, false, nil } - - actual, err := client.ValidatingWebhookConfigurations().Update(context.TODO(), existingCopy, metav1.UpdateOptions{}) + actual, err := client.ValidatingWebhookConfigurations().Update(context.TODO(), existing, metav1.UpdateOptions{}) return actual, true, err } @@ -301,7 +300,7 @@ func ApplyMutatingWebhookConfiguration( return existing, false, nil } - actual, err := client.MutatingWebhookConfigurations().Update(context.TODO(), existingCopy, metav1.UpdateOptions{}) + actual, err := client.MutatingWebhookConfigurations().Update(context.TODO(), existing, metav1.UpdateOptions{}) return actual, true, err } diff --git a/pkg/operators/clustermanager/controllers/clustermanagercontroller/clustermanager_controller.go b/pkg/operators/clustermanager/controllers/clustermanagercontroller/clustermanager_controller.go index ebbbcf3a1..9503a0d03 100644 --- a/pkg/operators/clustermanager/controllers/clustermanagercontroller/clustermanager_controller.go +++ b/pkg/operators/clustermanager/controllers/clustermanagercontroller/clustermanager_controller.go @@ -3,6 +3,8 @@ package clustermanagercontroller import ( "context" "encoding/base64" + + errorhelpers "errors" "fmt" "reflect" "strings" @@ -48,14 +50,13 @@ var ( "managedclusters.cluster.open-cluster-management.io", } - namespaceResource = "cluster-manager/cluster-manager-namespace.yaml" - hostedClusterSetCrdFile = "cluster-manager/hub/0000_00_clusters.open-cluster-management.io_managedclustersets.crd-hosted.yaml" - clusterSetCrdFile = "cluster-manager/hub/0000_00_clusters.open-cluster-management.io_managedclustersets.crd.yaml" + namespaceResource = "cluster-manager/cluster-manager-namespace.yaml" // crdResourceFiles should be deployed in the hub cluster hubCRDResourceFiles = []string{ "cluster-manager/hub/0000_00_addon.open-cluster-management.io_clustermanagementaddons.crd.yaml", "cluster-manager/hub/0000_00_clusters.open-cluster-management.io_managedclusters.crd.yaml", + "cluster-manager/hub/0000_00_clusters.open-cluster-management.io_managedclustersets.crd.yaml", "cluster-manager/hub/0000_00_work.open-cluster-management.io_manifestworks.crd.yaml", "cluster-manager/hub/0000_01_addon.open-cluster-management.io_managedclusteraddons.crd.yaml", "cluster-manager/hub/0000_01_clusters.open-cluster-management.io_managedclustersetbindings.crd.yaml", @@ -75,17 +76,17 @@ var ( // The hubWebhookResourceFiles should be deployed in the hub cluster // The service should may point to a external url which represent the webhook-server's address. - hubWebhookResourceFiles = []string{ - // registration-webhook + hubRegistrationWebhookResourceFiles = []string{ "cluster-manager/hub/cluster-manager-registration-webhook-validatingconfiguration.yaml", "cluster-manager/hub/cluster-manager-registration-webhook-mutatingconfiguration.yaml", "cluster-manager/hub/cluster-manager-registration-webhook-clustersetbinding-validatingconfiguration.yaml", - // work-webhook + "cluster-manager/hub/cluster-manager-registration-webhook-clustersetbinding-validatingconfiguration-v1beta1.yaml", + } + hubWorkWebhookResourceFiles = []string{ "cluster-manager/hub/cluster-manager-work-webhook-validatingconfiguration.yaml", } - // The apiservice resources should be applied after CABundle created. - // And also should be deployed in the hub cluster. + // The apiservice resources should be deleted hubApiserviceFiles = []string{ "cluster-manager/hub/cluster-manager-work-webhook-apiservice.yaml", "cluster-manager/hub/cluster-manager-registration-webhook-apiservice.yaml", @@ -136,10 +137,13 @@ var ( const ( clusterManagerFinalizer = "operator.open-cluster-management.io/cluster-manager-cleanup" clusterManagerApplied = "Applied" - caBundleConfigmap = "ca-bundle-configmap" + + caBundleConfigmap = "ca-bundle-configmap" hubRegistrationFeatureGatesValid = "ValidRegistrationFeatureGates" hubWorkFeatureGatesValid = "ValidWorkFeatureGates" + defaultWebhookPort = int32(9443) + clusterManagerReSyncTime = 5 * time.Second ) type clusterManagerController struct { @@ -292,6 +296,10 @@ func (n *clusterManagerController) sync(ctx context.Context, controllerContext f meta.SetStatusCondition(conditions, condition) } + //Set default port + config.RegistrationWebhook.Port = defaultWebhookPort + config.WorkWebhook.Port = defaultWebhookPort + // If we are deploying in the hosted mode, it requires us to create webhook in a different way with the default mode. // In the hosted mode, the webhook servers is running in the management cluster but the users are accessing the hub cluster. // So we need to add configuration to make the apiserver of the hub cluster could access the webhook servers on the management cluster. @@ -345,6 +353,23 @@ func (n *clusterManagerController) sync(ctx context.Context, controllerContext f return err } + //get caBundle + caBundle := "placeholder" + configmap, err := n.configMapLister.ConfigMaps(clusterManagerNamespace).Get(caBundleConfigmap) + switch { + case errors.IsNotFound(err): + // do nothing + case err != nil: + return err + default: + if cb := configmap.Data["ca-bundle.crt"]; len(cb) > 0 { + caBundle = cb + } + } + encodedCaBundle := base64.StdEncoding.EncodeToString([]byte(caBundle)) + config.RegistrationAPIServiceCABundle = encodedCaBundle + config.WorkAPIServiceCABundle = encodedCaBundle + // Apply resources on the hub cluster hubAppliedErrs, err := applyHubResources( ctx, @@ -353,7 +378,7 @@ func (n *clusterManagerController) sync(ctx context.Context, controllerContext f config, &relatedResources, hubClient, hubApiExtensionClient, hubApiRegistrationClient, - n.configMapLister, n.recorder, n.cache) + n.recorder, n.cache) if err != nil { return err } @@ -370,9 +395,34 @@ func (n *clusterManagerController) sync(ctx context.Context, controllerContext f if err != nil { return err } + errs := append(hubAppliedErrs, managementAppliedErrs...) + + //Check registration and work webhook pod running, then apply webhook config files + registrationWebhookName := clusterManagerName + "-registration-webhook" + workWebhookName := clusterManagerName + "-work-webhook" + rqe := helpers.RequeueError{} + + err = applyWebhookConfig(ctx, registrationWebhookName, hubRegistrationWebhookResourceFiles, &relatedResources, config, hubClient, managementClient, n.recorder, n.cache) + if err != nil { + if errorhelpers.As(err, &rqe) { + klog.Warning("Apply webhook config %v fail. Error: %v", registrationWebhookName, err) + controllerContext.Queue().AddAfter(clusterManagerName, err.(helpers.RequeueError).RequeueTime) + } else { + errs = append(errs, err) + } + } + + err = applyWebhookConfig(ctx, workWebhookName, hubWorkWebhookResourceFiles, &relatedResources, config, hubClient, managementClient, n.recorder, n.cache) + if err != nil { + if errorhelpers.As(err, &rqe) { + klog.Warning("Apply webhook config %v fail. Error: %v", registrationWebhookName, err) + controllerContext.Queue().AddAfter(clusterManagerName, err.(helpers.RequeueError).RequeueTime) + } else { + errs = append(errs, err) + } + } // Update status - errs := append(hubAppliedErrs, managementAppliedErrs...) observedGeneration := clusterManager.Status.ObservedGeneration if len(errs) == 0 { meta.SetStatusCondition(conditions, metav1.Condition{ @@ -416,31 +466,14 @@ func applyHubResources( relatedResources *[]operatorapiv1.RelatedResourceMeta, // hub clients hubClient kubernetes.Interface, hubApiExtensionClient apiextensionsclient.Interface, hubApiRegistrationClient apiregistrationclient.APIServicesGetter, - configMapLister corev1listers.ConfigMapLister, recorder events.Recorder, cache resourceapply.ResourceCache, ) (appliedErrs []error, err error) { // Try to load ca bundle from configmap // If the configmap is found, populate it into configmap. // If the configmap not found yet, skip this and apply other resources first. - caBundle := "placeholder" - configmap, err := configMapLister.ConfigMaps(clusterManagerNamespace).Get(caBundleConfigmap) - switch { - case errors.IsNotFound(err): - // do nothing - case err != nil: - return appliedErrs, err - default: - if cb := configmap.Data["ca-bundle.crt"]; len(cb) > 0 { - caBundle = cb - } - } - encodedCaBundle := base64.StdEncoding.EncodeToString([]byte(caBundle)) - manifestsConfig.RegistrationAPIServiceCABundle = encodedCaBundle - manifestsConfig.WorkAPIServiceCABundle = encodedCaBundle // Apply hub cluster resources - // ClusterSet crd need caBundle, so we need to apply the hubresources after get caBundle hubResources := getHubResources(clusterManagerMode, manifestsConfig.RegistrationWebhook.IsIPFormat, manifestsConfig.WorkWebhook.IsIPFormat, false) resourceResults := helpers.ApplyDirectly( ctx, @@ -467,14 +500,34 @@ func applyHubResources( } } - // Apply Apiservice files to hub cluster. - // The reason why apply Apiservice after apply other staticfiles(including namespace) is because Apiservices requires the CABundleConfigmap. - // And it will return an error(uncatchable with NotFound type) if the namespace is not created. - apiserviceResults := helpers.ApplyDirectly( + return appliedErrs, nil +} + +func applyWebhookConfig( + ctx context.Context, + webhookDeploymentName string, + webhookResourceFiles []string, + relatedResources *[]operatorapiv1.RelatedResourceMeta, + manifestsConfig manifests.HubConfig, + // hub clients + hubClient kubernetes.Interface, + managementKubeClient kubernetes.Interface, + recorder events.Recorder, + cache resourceapply.ResourceCache, +) error { + var appliedErrs []error + //Check registration webhook running + err := checkWebhookPodRunning(manifestsConfig.ClusterManagerNamespace, managementKubeClient, webhookDeploymentName) + if err != nil { + return err + } + + // If all webhook pod running , then apply webhook config files + resourceResults := helpers.ApplyDirectly( ctx, hubClient, - hubApiExtensionClient, - hubApiRegistrationClient, + nil, + nil, nil, recorder, cache, @@ -487,15 +540,32 @@ func applyHubResources( helpers.SetRelatedResourcesStatusesWithObj(relatedResources, objData) return objData, nil }, - hubApiserviceFiles..., + webhookResourceFiles..., ) - for _, result := range apiserviceResults { + for _, result := range resourceResults { if result.Error != nil { appliedErrs = append(appliedErrs, fmt.Errorf("%q (%T): %v", result.File, result.Type, result.Error)) } } + return operatorhelpers.NewMultiLineAggregate(appliedErrs) +} - return appliedErrs, nil +func checkWebhookPodRunning(clusterManagerNamespace string, managementKubeClient kubernetes.Interface, webhookName string) error { + webhookDeployment, err := managementKubeClient.AppsV1().Deployments(clusterManagerNamespace).Get(context.Background(), + webhookName, metav1.GetOptions{}) + if err != nil { + return err + } + if webhookDeployment.Generation != webhookDeployment.Status.ObservedGeneration { + return helpers.NewRequeueError(fmt.Sprintf("New %v webhook deployment not observed. Expect generation:%v, Observed Generation:%v", webhookName, webhookDeployment.Generation, webhookDeployment.Status.ObservedGeneration), clusterManagerReSyncTime) + } + replicas := *webhookDeployment.Spec.Replicas + readyReplicas := webhookDeployment.Status.ReadyReplicas + + if readyReplicas != replicas { + return helpers.NewRequeueError(fmt.Sprintf("Deployment %s not ready. Replica:%v, ReadyReplicas:%v", webhookName, replicas, readyReplicas), clusterManagerReSyncTime) + } + return nil } func applyManagementResources( @@ -679,7 +749,12 @@ func cleanUpHub(ctx context.Context, controllerContext factory.SyncContext, } // Remove All Static files + // API service should be deleted in 0.10.0 + // TODO: should remove apiservice files future hubResources := append(getHubResources(mode, config.RegistrationWebhook.IsIPFormat, config.WorkWebhook.IsIPFormat, skipRemoveCRDs), hubApiserviceFiles...) + hubResources = append(hubResources, hubRegistrationWebhookResourceFiles...) + hubResources = append(hubResources, hubWorkWebhookResourceFiles...) + for _, file := range hubResources { err := helpers.CleanUpStaticObject( ctx, @@ -781,16 +856,9 @@ func getSAs(clusterManagerName string) []string { func getHubResources(mode operatorapiv1.InstallMode, isRegistrationIPFormat, isWorkIPFormat, skipAddCRDs bool) []string { hubResources := []string{namespaceResource} if !skipAddCRDs { - //TODO: Currently, in hosted mode, clusterset only support v1beta1 version, will fix it in future releases - if mode == operatorapiv1.InstallModeHosted { - hubResources = append(hubResources, hostedClusterSetCrdFile) - } else { - hubResources = append(hubResources, clusterSetCrdFile) - } hubResources = append(hubResources, hubCRDResourceFiles...) } - hubResources = append(hubResources, hubWebhookResourceFiles...) hubResources = append(hubResources, hubRbacResourceFiles...) // the hubHostedWebhookServiceFiles are only used in hosted mode diff --git a/pkg/operators/clustermanager/controllers/clustermanagercontroller/clustermanager_controller_test.go b/pkg/operators/clustermanager/controllers/clustermanagercontroller/clustermanager_controller_test.go index 2a36aeaa6..04d0ba208 100644 --- a/pkg/operators/clustermanager/controllers/clustermanagercontroller/clustermanager_controller_test.go +++ b/pkg/operators/clustermanager/controllers/clustermanagercontroller/clustermanager_controller_test.go @@ -87,9 +87,62 @@ func newTestController(t *testing.T, clustermanager *operatorapiv1.ClusterManage } } -func setup(t *testing.T, tc *testController, crds ...runtime.Object) { +func setWebhookDeployment(clusterManagerName, clusterManagerNamespace string) []runtime.Object { + var replicas = int32(1) + + return []runtime.Object{ + &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterManagerName + "-registration-webhook", + Namespace: clusterManagerNamespace, + Generation: 1, + }, + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: clusterManagerName + "-webhook", + }, + }, + }, + }, + Replicas: &replicas, + }, + Status: appsv1.DeploymentStatus{ + ReadyReplicas: replicas, + ObservedGeneration: 1, + }, + }, + &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterManagerName + "-work-webhook", + Namespace: clusterManagerNamespace, + Generation: 1, + }, + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: clusterManagerName + "-webhook", + }, + }, + }, + }, + Replicas: &replicas, + }, + Status: appsv1.DeploymentStatus{ + ReadyReplicas: replicas, + ObservedGeneration: 1, + }, + }, + } +} + +func setup(t *testing.T, tc *testController, cd []runtime.Object, crds ...runtime.Object) { fakeHubKubeClient := fakekube.NewSimpleClientset() - fakeManagementKubeClient := fakekube.NewSimpleClientset() + fakeManagementKubeClient := fakekube.NewSimpleClientset(cd...) fakeAPIExtensionClient := fakeapiextensions.NewSimpleClientset(crds...) fakeAPIRegistrationClient := fakeapiregistration.NewSimpleClientset() fakeMigrationClient := fakemigrationclient.NewSimpleClientset() @@ -134,7 +187,9 @@ func ensureObject(t *testing.T, object runtime.Object, hubCore *operatorapiv1.Cl func TestSyncDeploy(t *testing.T) { clusterManager := newClusterManager("testhub") tc := newTestController(t, clusterManager) - setup(t, tc) + clusterManagerNamespace := helpers.ClusterManagerNamespace(clusterManager.Name, clusterManager.Spec.DeployOption.Mode) + cd := setWebhookDeployment(clusterManager.Name, clusterManagerNamespace) + setup(t, tc, cd) syncContext := testinghelper.NewFakeSyncContext(t, "testhub") @@ -154,7 +209,7 @@ func TestSyncDeploy(t *testing.T) { // Check if resources are created as expected // We expect creat the namespace twice respectively in the management cluster and the hub cluster. - testinghelper.AssertEqualNumber(t, len(createKubeObjects), 24) + testinghelper.AssertEqualNumber(t, len(createKubeObjects), 23) for _, object := range createKubeObjects { ensureObject(t, object, clusterManager) } @@ -169,24 +224,46 @@ func TestSyncDeploy(t *testing.T) { } // Check if resources are created as expected testinghelper.AssertEqualNumber(t, len(createCRDObjects), 10) +} - createAPIServiceObjects := []runtime.Object{} - apiServiceActions := tc.apiRegistrationClient.Actions() - for _, action := range apiServiceActions { +func TestSyncDeployNoWebhook(t *testing.T) { + clusterManager := newClusterManager("testhub") + tc := newTestController(t, clusterManager) + setup(t, tc, nil) + + syncContext := testinghelper.NewFakeSyncContext(t, "testhub") + + err := tc.clusterManagerController.sync(ctx, syncContext) + if err != nil { + t.Fatalf("Expected no error when sync, %v", err) + } + + createKubeObjects := []runtime.Object{} + kubeActions := append(tc.hubKubeClient.Actions(), tc.managementKubeClient.Actions()...) // record objects from both hub and management cluster + for _, action := range kubeActions { if action.GetVerb() == "create" { object := action.(clienttesting.CreateActionImpl).Object - createAPIServiceObjects = append(createAPIServiceObjects, object) + createKubeObjects = append(createKubeObjects, object) + } + } + + // Check if resources are created as expected + // We expect creat the namespace twice respectively in the management cluster and the hub cluster. + testinghelper.AssertEqualNumber(t, len(createKubeObjects), 20) + for _, object := range createKubeObjects { + ensureObject(t, object, clusterManager) + } + + createCRDObjects := []runtime.Object{} + crdActions := tc.apiExtensionClient.Actions() + for _, action := range crdActions { + if action.GetVerb() == "create" { + object := action.(clienttesting.CreateActionImpl).Object + createCRDObjects = append(createCRDObjects, object) } } // Check if resources are created as expected - testinghelper.AssertEqualNumber(t, len(createAPIServiceObjects), 2) - - clusterManagerAction := tc.operatorClient.Actions() - testinghelper.AssertEqualNumber(t, len(clusterManagerAction), 2) - testinghelper.AssertAction(t, clusterManagerAction[1], "update") - testinghelper.AssertOnlyConditions( - t, clusterManagerAction[1].(clienttesting.UpdateActionImpl).Object, - testinghelper.NamedCondition(clusterManagerApplied, "ClusterManagerApplied", metav1.ConditionTrue)) + testinghelper.AssertEqualNumber(t, len(createCRDObjects), 10) } // TestSyncDelete test cleanup hub deploy @@ -196,7 +273,7 @@ func TestSyncDelete(t *testing.T) { clusterManager.ObjectMeta.SetDeletionTimestamp(&now) tc := newTestController(t, clusterManager) - setup(t, tc) + setup(t, tc, nil) syncContext := testinghelper.NewFakeSyncContext(t, "testhub") clusterManagerNamespace := helpers.ClusterManagerNamespace(clusterManager.Name, clusterManager.Spec.DeployOption.Mode) @@ -214,7 +291,7 @@ func TestSyncDelete(t *testing.T) { deleteKubeActions = append(deleteKubeActions, deleteKubeAction) } } - testinghelper.AssertEqualNumber(t, len(deleteKubeActions), 20) // delete namespace both from the hub cluster and the mangement cluster + testinghelper.AssertEqualNumber(t, len(deleteKubeActions), 21) // delete namespace both from the hub cluster and the mangement cluster deleteCRDActions := []clienttesting.DeleteActionImpl{} crdActions := tc.apiExtensionClient.Actions() @@ -258,7 +335,7 @@ func TestDeleteCRD(t *testing.T) { } tc := newTestController(t, clusterManager) - setup(t, tc, crd) + setup(t, tc, nil, crd) // Return crd with the first get, and return not found with the 2nd get getCount := 0 diff --git a/test/e2e/common.go b/test/e2e/common.go index db9974d8d..6cae9a78a 100644 --- a/test/e2e/common.go +++ b/test/e2e/common.go @@ -492,16 +492,33 @@ func (t *Tester) CheckHubReady() error { Get(context.TODO(), t.hubRegistrationDeployment, metav1.GetOptions{}); err != nil { return err } + gomega.Eventually(func() error { + registrationWebhookDeployment, err := t.KubeClient.AppsV1().Deployments(t.clusterManagerNamespace). + Get(context.TODO(), t.hubRegistrationWebhookDeployment, metav1.GetOptions{}) + if err != nil { + return err + } + replicas := *registrationWebhookDeployment.Spec.Replicas + readyReplicas := registrationWebhookDeployment.Status.ReadyReplicas + if readyReplicas != replicas { + return fmt.Errorf("deployment %s should have %d but got %d ready replicas", t.hubRegistrationWebhookDeployment, replicas, readyReplicas) + } + return nil + }, t.EventuallyTimeout*5, t.EventuallyInterval*5).Should(gomega.BeNil()) - if _, err := t.KubeClient.AppsV1().Deployments(t.clusterManagerNamespace). - Get(context.TODO(), t.hubRegistrationWebhookDeployment, metav1.GetOptions{}); err != nil { - return err - } - - if _, err := t.KubeClient.AppsV1().Deployments(t.clusterManagerNamespace). - Get(context.TODO(), t.hubWorkWebhookDeployment, metav1.GetOptions{}); err != nil { - return err - } + gomega.Eventually(func() error { + workWebhookDeployment, err := t.KubeClient.AppsV1().Deployments(t.clusterManagerNamespace). + Get(context.TODO(), t.hubWorkWebhookDeployment, metav1.GetOptions{}) + if err != nil { + return err + } + replicas := *workWebhookDeployment.Spec.Replicas + readyReplicas := workWebhookDeployment.Status.ReadyReplicas + if readyReplicas != replicas { + return fmt.Errorf("deployment %s should have %d but got %d ready replicas", t.hubWorkWebhookDeployment, replicas, readyReplicas) + } + return nil + }, t.EventuallyTimeout*5, t.EventuallyInterval*5).Should(gomega.BeNil()) if _, err := t.KubeClient.AppsV1().Deployments(t.clusterManagerNamespace). Get(context.TODO(), t.hubPlacementDeployment, metav1.GetOptions{}); err != nil { diff --git a/test/integration/clustermanager_hosted_test.go b/test/integration/clustermanager_hosted_test.go index 8b53e76ca..5a17816ea 100644 --- a/test/integration/clustermanager_hosted_test.go +++ b/test/integration/clustermanager_hosted_test.go @@ -7,23 +7,34 @@ import ( "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" - corev1 "k8s.io/api/core/v1" - v1 "open-cluster-management.io/api/operator/v1" - - appsv1 "k8s.io/api/apps/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/util/cert" - "github.com/openshift/library-go/pkg/operator/resource/resourceapply" - + v1 "open-cluster-management.io/api/operator/v1" "open-cluster-management.io/registration-operator/pkg/helpers" "open-cluster-management.io/registration-operator/test/integration/util" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/util/cert" ) +func updateDeploymentStatus(kubeClient kubernetes.Interface, namespace, deploymentName string) { + deployment, err := kubeClient.AppsV1().Deployments(namespace).Get(context.Background(), deploymentName, metav1.GetOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + deployment.Status.Replicas = *deployment.Spec.Replicas + deployment.Status.ReadyReplicas = *deployment.Spec.Replicas + deployment.Status.ObservedGeneration = *&deployment.Generation + _, err = kubeClient.AppsV1().Deployments(namespace).UpdateStatus(context.Background(), deployment, metav1.UpdateOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) +} + var _ = ginkgo.Describe("ClusterManager Hosted Mode", func() { var hostedCtx context.Context var hostedCancel context.CancelFunc var hubRegistrationDeployment = fmt.Sprintf("%s-registration-controller", clusterManagerName) + var hubRegistrationWebhookDeployment = fmt.Sprintf("%s-registration-webhook", clusterManagerName) + var hubWorkWebhookDeployment = fmt.Sprintf("%s-work-webhook", clusterManagerName) ginkgo.BeforeEach(func() { hostedCtx, hostedCancel = context.WithCancel(context.Background()) @@ -143,7 +154,6 @@ var _ = ginkgo.Describe("ClusterManager Hosted Mode", func() { return nil }, eventuallyTimeout, eventuallyInterval).Should(gomega.BeNil()) - hubRegistrationWebhookDeployment := fmt.Sprintf("%s-registration-webhook", clusterManagerName) gomega.Eventually(func() error { if _, err := hostedKubeClient.AppsV1().Deployments(hubNamespaceHosted).Get(hostedCtx, hubRegistrationWebhookDeployment, metav1.GetOptions{}); err != nil { return err @@ -151,7 +161,6 @@ var _ = ginkgo.Describe("ClusterManager Hosted Mode", func() { return nil }, eventuallyTimeout, eventuallyInterval).Should(gomega.BeNil()) - hubWorkWebhookDeployment := fmt.Sprintf("%s-work-webhook", clusterManagerName) gomega.Eventually(func() error { if _, err := hostedKubeClient.AppsV1().Deployments(hubNamespaceHosted).Get(hostedCtx, hubWorkWebhookDeployment, metav1.GetOptions{}); err != nil { return err @@ -209,6 +218,13 @@ var _ = ginkgo.Describe("ClusterManager Hosted Mode", func() { // Check validating webhook registrationValidtingWebhook := "managedclustervalidators.admission.cluster.open-cluster-management.io" + + //Should not apply the webhook config if the replica and observed is not set + _, err := hostedKubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Get(hostedCtx, registrationValidtingWebhook, metav1.GetOptions{}) + gomega.Expect(err).To(gomega.HaveOccurred()) + + updateDeploymentStatus(hostedKubeClient, hubNamespaceHosted, hubRegistrationWebhookDeployment) + gomega.Eventually(func() error { if _, err := hostedKubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Get(hostedCtx, registrationValidtingWebhook, metav1.GetOptions{}); err != nil { return err @@ -217,6 +233,13 @@ var _ = ginkgo.Describe("ClusterManager Hosted Mode", func() { }, eventuallyTimeout, eventuallyInterval).Should(gomega.BeNil()) workValidtingWebhook := "manifestworkvalidators.admission.work.open-cluster-management.io" + //Should not apply the webhook config if the replica and observed is not set + _, err = hostedKubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Get(hostedCtx, workValidtingWebhook, metav1.GetOptions{}) + + gomega.Expect(err).To(gomega.HaveOccurred()) + + updateDeploymentStatus(hostedKubeClient, hubNamespaceHosted, hubWorkWebhookDeployment) + gomega.Eventually(func() error { if _, err := hostedKubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Get(hostedCtx, workValidtingWebhook, metav1.GetOptions{}); err != nil { return err @@ -267,6 +290,8 @@ var _ = ginkgo.Describe("ClusterManager Hosted Mode", func() { return nil }, eventuallyTimeout, eventuallyInterval).Should(gomega.BeNil()) + updateDeploymentStatus(hostedKubeClient, hubNamespaceHosted, hubRegistrationWebhookDeployment) + // Check if generations are correct gomega.Eventually(func() error { actual, err := hostedOperatorClient.OperatorV1().ClusterManagers().Get(hostedCtx, clusterManagerName, metav1.GetOptions{}) @@ -287,12 +312,11 @@ var _ = ginkgo.Describe("ClusterManager Hosted Mode", func() { if err != nil { return err } - if len(actual.Status.RelatedResources) != 35 { - return fmt.Errorf("should get 35 relatedResources, actual got %v", len(actual.Status.RelatedResources)) + if len(actual.Status.RelatedResources) != 34 { + return fmt.Errorf("should get 34 relatedResources, actual got %v", len(actual.Status.RelatedResources)) } return nil }, eventuallyTimeout, eventuallyInterval).ShouldNot(gomega.HaveOccurred()) - }) ginkgo.It("Deployment should be added nodeSelector and toleration when add nodePlacement into clustermanager", func() { @@ -334,6 +358,10 @@ var _ = ginkgo.Describe("ClusterManager Hosted Mode", func() { return fmt.Errorf("no key equals to node-role.kubernetes.io/infra") }, eventuallyTimeout, eventuallyInterval).Should(gomega.BeNil()) + + updateDeploymentStatus(hostedKubeClient, hubNamespaceHosted, hubRegistrationWebhookDeployment) + updateDeploymentStatus(hostedKubeClient, hubNamespaceHosted, hubWorkWebhookDeployment) + }) ginkgo.It("Deployment should be reconciled when manually updated", func() { gomega.Eventually(func() error { diff --git a/test/integration/clustermanager_test.go b/test/integration/clustermanager_test.go index d6e664744..cfb3d26d3 100644 --- a/test/integration/clustermanager_test.go +++ b/test/integration/clustermanager_test.go @@ -47,6 +47,8 @@ func startHubOperator(ctx context.Context, mode v1.InstallMode) { var _ = ginkgo.Describe("ClusterManager Default Mode", func() { var cancel context.CancelFunc var hubRegistrationDeployment = fmt.Sprintf("%s-registration-controller", clusterManagerName) + var hubRegistrationWebhookDeployment = fmt.Sprintf("%s-registration-webhook", clusterManagerName) + var hubWorkWebhookDeployment = fmt.Sprintf("%s-work-webhook", clusterManagerName) ginkgo.BeforeEach(func() { var ctx context.Context @@ -69,7 +71,6 @@ var _ = ginkgo.Describe("ClusterManager Default Mode", func() { } return nil }, eventuallyTimeout, eventuallyInterval).Should(gomega.BeNil()) - // Check clusterrole/clusterrolebinding hubRegistrationClusterRole := fmt.Sprintf("open-cluster-management:%s-registration:controller", clusterManagerName) hubRegistrationWebhookClusterRole := fmt.Sprintf("open-cluster-management:%s-registration:webhook", clusterManagerName) @@ -80,6 +81,7 @@ var _ = ginkgo.Describe("ClusterManager Default Mode", func() { } return nil }, eventuallyTimeout, eventuallyInterval).Should(gomega.BeNil()) + gomega.Eventually(func() error { if _, err := kubeClient.RbacV1().ClusterRoles().Get(context.Background(), hubRegistrationWebhookClusterRole, metav1.GetOptions{}); err != nil { return err @@ -104,6 +106,7 @@ var _ = ginkgo.Describe("ClusterManager Default Mode", func() { } return nil }, eventuallyTimeout, eventuallyInterval).Should(gomega.BeNil()) + gomega.Eventually(func() error { if _, err := kubeClient.RbacV1().ClusterRoleBindings().Get(context.Background(), hubWorkWebhookClusterRole, metav1.GetOptions{}); err != nil { return err @@ -142,7 +145,6 @@ var _ = ginkgo.Describe("ClusterManager Default Mode", func() { return nil }, eventuallyTimeout, eventuallyInterval).Should(gomega.BeNil()) - hubRegistrationWebhookDeployment := fmt.Sprintf("%s-registration-webhook", clusterManagerName) gomega.Eventually(func() error { if _, err := kubeClient.AppsV1().Deployments(hubNamespace).Get(context.Background(), hubRegistrationWebhookDeployment, metav1.GetOptions{}); err != nil { return err @@ -150,7 +152,6 @@ var _ = ginkgo.Describe("ClusterManager Default Mode", func() { return nil }, eventuallyTimeout, eventuallyInterval).Should(gomega.BeNil()) - hubWorkWebhookDeployment := fmt.Sprintf("%s-work-webhook", clusterManagerName) gomega.Eventually(func() error { if _, err := kubeClient.AppsV1().Deployments(hubNamespace).Get(context.Background(), hubWorkWebhookDeployment, metav1.GetOptions{}); err != nil { return err @@ -208,21 +209,34 @@ var _ = ginkgo.Describe("ClusterManager Default Mode", func() { // Check validating webhook registrationValidtingWebhook := "managedclustervalidators.admission.cluster.open-cluster-management.io" + + //Should not apply the webhook config if the replica and observed is not set + _, err := kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Get(context.Background(), registrationValidtingWebhook, metav1.GetOptions{}) + gomega.Expect(err).To(gomega.HaveOccurred()) + // Update readyreplica of deployment + + updateDeploymentStatus(kubeClient, hubNamespace, hubRegistrationWebhookDeployment) + gomega.Eventually(func() error { if _, err := kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Get(context.Background(), registrationValidtingWebhook, metav1.GetOptions{}); err != nil { return err } return nil - }, eventuallyTimeout, eventuallyInterval).Should(gomega.BeNil()) + }, eventuallyTimeout*10, eventuallyInterval).Should(gomega.BeNil()) workValidtingWebhook := "manifestworkvalidators.admission.work.open-cluster-management.io" + //Should not apply the webhook config if the replica and observed is not set + _, err = kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Get(context.Background(), workValidtingWebhook, metav1.GetOptions{}) + gomega.Expect(err).To(gomega.HaveOccurred()) + + updateDeploymentStatus(kubeClient, hubNamespace, hubWorkWebhookDeployment) + gomega.Eventually(func() error { if _, err := kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Get(context.Background(), workValidtingWebhook, metav1.GetOptions{}); err != nil { return err } return nil }, eventuallyTimeout, eventuallyInterval).Should(gomega.BeNil()) - util.AssertClusterManagerCondition(clusterManagerName, operatorClient, "Applied", "ClusterManagerApplied", metav1.ConditionTrue) }) @@ -266,6 +280,8 @@ var _ = ginkgo.Describe("ClusterManager Default Mode", func() { return nil }, eventuallyTimeout, eventuallyInterval).Should(gomega.BeNil()) + updateDeploymentStatus(kubeClient, hubNamespace, hubRegistrationWebhookDeployment) + // Check if generations are correct gomega.Eventually(func() error { actual, err := operatorClient.OperatorV1().ClusterManagers().Get(context.Background(), clusterManagerName, metav1.GetOptions{}) @@ -286,8 +302,8 @@ var _ = ginkgo.Describe("ClusterManager Default Mode", func() { if err != nil { return err } - if len(actual.Status.RelatedResources) != 35 { - return fmt.Errorf("should get 35 relatedResources, actual got %v", len(actual.Status.RelatedResources)) + if len(actual.Status.RelatedResources) != 34 { + return fmt.Errorf("should get 34 relatedResources, actual got %v", len(actual.Status.RelatedResources)) } return nil }, eventuallyTimeout, eventuallyInterval).ShouldNot(gomega.HaveOccurred()) @@ -333,6 +349,8 @@ var _ = ginkgo.Describe("ClusterManager Default Mode", func() { return fmt.Errorf("no key equals to node-role.kubernetes.io/infra") }, eventuallyTimeout, eventuallyInterval).Should(gomega.BeNil()) + updateDeploymentStatus(kubeClient, hubNamespace, hubRegistrationWebhookDeployment) + updateDeploymentStatus(kubeClient, hubNamespace, hubWorkWebhookDeployment) }) ginkgo.It("Deployment should be reconciled when manually updated", func() { gomega.Eventually(func() error {