mirror of
https://github.com/open-cluster-management-io/ocm.git
synced 2026-05-20 16:14:23 +00:00
use controller-runtime webhook (#293)
Signed-off-by: ldpliu <daliu@redhat.com> Signed-off-by: ldpliu <daliu@redhat.com>
This commit is contained in:
DangPeng Liu
GitHub
@@ -160,8 +160,7 @@ We mainly provide deployment in two scenarios:
|
||||
selector:
|
||||
app: cluster-manager-registration-webhook
|
||||
ports:
|
||||
- port: 443
|
||||
targetPort: 6443
|
||||
- port: 9443
|
||||
nodePort: 30443
|
||||
---
|
||||
apiVersion: v1
|
||||
@@ -174,8 +173,7 @@ We mainly provide deployment in two scenarios:
|
||||
selector:
|
||||
app: cluster-manager-work-webhook
|
||||
ports:
|
||||
- port: 443
|
||||
targetPort: 6443
|
||||
- port: 9443
|
||||
nodePort: 31443
|
||||
EOF
|
||||
```
|
||||
|
||||
@@ -9,7 +9,7 @@
|
||||
namespace: {{ .ClusterManagerNamespace }}
|
||||
name: cluster-manager-registration-webhook
|
||||
path: /convert
|
||||
port: 9443
|
||||
port: {{.RegistrationWebhook.Port}}
|
||||
caBundle: {{ .RegistrationAPIServiceCABundle }}
|
||||
conversionReviewVersions:
|
||||
- v1beta1
|
||||
|
||||
@@ -12,7 +12,7 @@ spec:
|
||||
name: cluster-manager-registration-webhook
|
||||
namespace: {{ .ClusterManagerNamespace }}
|
||||
path: /convert
|
||||
port: 9443
|
||||
port: {{.RegistrationWebhook.Port}}
|
||||
conversionReviewVersions:
|
||||
- v1beta1
|
||||
- v1beta2
|
||||
|
||||
@@ -8,11 +8,6 @@ spec:
|
||||
service:
|
||||
name: cluster-manager-registration-webhook
|
||||
namespace: {{ .ClusterManagerNamespace }}
|
||||
{{if eq .RegistrationWebhook.Port 0}}
|
||||
port: 443
|
||||
{{else}}
|
||||
port: {{.RegistrationWebhook.Port}}
|
||||
{{end}}
|
||||
caBundle: {{ .RegistrationAPIServiceCABundle }}
|
||||
groupPriorityMinimum: 10000
|
||||
versionPriority: 20
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
apiVersion: admissionregistration.k8s.io/v1
|
||||
kind: ValidatingWebhookConfiguration
|
||||
metadata:
|
||||
name: managedclustersetbindingv1beta1validators.admission.cluster.open-cluster-management.io
|
||||
webhooks:
|
||||
- name: managedclustersetbindingv1beta1validators.admission.cluster.open-cluster-management.io
|
||||
failurePolicy: Fail
|
||||
clientConfig:
|
||||
service:
|
||||
namespace: {{ .ClusterManagerNamespace }}
|
||||
name: cluster-manager-registration-webhook
|
||||
path: /validate-cluster-open-cluster-management-io-v1beta1-managedclustersetbinding
|
||||
port: {{.RegistrationWebhook.Port}}
|
||||
caBundle: {{ .RegistrationAPIServiceCABundle }}
|
||||
rules:
|
||||
- operations:
|
||||
- CREATE
|
||||
- UPDATE
|
||||
apiGroups:
|
||||
- cluster.open-cluster-management.io
|
||||
apiVersions:
|
||||
- v1beta1
|
||||
resources:
|
||||
- managedclustersetbindings
|
||||
admissionReviewVersions: ["v1beta1","v1"]
|
||||
sideEffects: None
|
||||
timeoutSeconds: 10
|
||||
@@ -7,10 +7,11 @@ webhooks:
|
||||
failurePolicy: Fail
|
||||
clientConfig:
|
||||
service:
|
||||
# reach the webhook via the registered aggregated API
|
||||
namespace: default
|
||||
name: kubernetes
|
||||
path: /apis/admission.cluster.open-cluster-management.io/v1/managedclustersetbindingvalidators
|
||||
namespace: {{ .ClusterManagerNamespace }}
|
||||
name: cluster-manager-registration-webhook
|
||||
path: /validate-cluster-open-cluster-management-io-v1beta2-managedclustersetbinding
|
||||
port: {{.RegistrationWebhook.Port}}
|
||||
caBundle: {{ .RegistrationAPIServiceCABundle }}
|
||||
rules:
|
||||
- operations:
|
||||
- CREATE
|
||||
@@ -18,9 +19,9 @@ webhooks:
|
||||
apiGroups:
|
||||
- cluster.open-cluster-management.io
|
||||
apiVersions:
|
||||
- "*"
|
||||
- v1beta2
|
||||
resources:
|
||||
- managedclustersetbindings
|
||||
admissionReviewVersions: ["v1beta1"]
|
||||
admissionReviewVersions: ["v1beta1","v1"]
|
||||
sideEffects: None
|
||||
timeoutSeconds: 10
|
||||
|
||||
@@ -7,10 +7,11 @@ webhooks:
|
||||
failurePolicy: Fail
|
||||
clientConfig:
|
||||
service:
|
||||
# reach the webhook via the registered aggregated API
|
||||
namespace: default
|
||||
name: kubernetes
|
||||
path: /apis/admission.cluster.open-cluster-management.io/v1/managedclustermutators
|
||||
namespace: {{ .ClusterManagerNamespace }}
|
||||
name: cluster-manager-registration-webhook
|
||||
path: /mutate-cluster-open-cluster-management-io-v1-managedcluster
|
||||
port: {{.RegistrationWebhook.Port}}
|
||||
caBundle: {{ .RegistrationAPIServiceCABundle }}
|
||||
rules:
|
||||
- operations:
|
||||
- CREATE
|
||||
@@ -21,6 +22,6 @@ webhooks:
|
||||
- "*"
|
||||
resources:
|
||||
- managedclusters
|
||||
admissionReviewVersions: ["v1beta1"]
|
||||
admissionReviewVersions: ["v1beta1","v1"]
|
||||
sideEffects: None
|
||||
timeoutSeconds: 10
|
||||
|
||||
@@ -10,6 +10,6 @@ spec:
|
||||
- name: webhook-server
|
||||
port: 443
|
||||
targetPort: 6443
|
||||
- name: conversion-webhook
|
||||
- name: webhook
|
||||
port: 9443
|
||||
targetPort: 9443
|
||||
|
||||
@@ -7,10 +7,11 @@ webhooks:
|
||||
failurePolicy: Fail
|
||||
clientConfig:
|
||||
service:
|
||||
# reach the webhook via the registered aggregated API
|
||||
namespace: default
|
||||
name: kubernetes
|
||||
path: /apis/admission.cluster.open-cluster-management.io/v1/managedclustervalidators
|
||||
namespace: {{ .ClusterManagerNamespace }}
|
||||
name: cluster-manager-registration-webhook
|
||||
path: /validate-cluster-open-cluster-management-io-v1-managedcluster
|
||||
port: {{.RegistrationWebhook.Port}}
|
||||
caBundle: {{ .RegistrationAPIServiceCABundle }}
|
||||
rules:
|
||||
- operations:
|
||||
- CREATE
|
||||
@@ -21,6 +22,6 @@ webhooks:
|
||||
- "*"
|
||||
resources:
|
||||
- managedclusters
|
||||
admissionReviewVersions: ["v1beta1"]
|
||||
admissionReviewVersions: ["v1beta1","v1"]
|
||||
sideEffects: None
|
||||
timeoutSeconds: 10
|
||||
|
||||
@@ -8,11 +8,6 @@ spec:
|
||||
service:
|
||||
name: cluster-manager-work-webhook
|
||||
namespace: {{ .ClusterManagerNamespace }}
|
||||
{{if eq .WorkWebhook.Port 0}}
|
||||
port: 443
|
||||
{{else}}
|
||||
port: {{.WorkWebhook.Port}}
|
||||
{{end}}
|
||||
caBundle: {{ .WorkAPIServiceCABundle }}
|
||||
groupPriorityMinimum: 10000
|
||||
versionPriority: 20
|
||||
|
||||
@@ -7,5 +7,9 @@ spec:
|
||||
selector:
|
||||
app: {{ .ClusterManagerName }}-work-webhook
|
||||
ports:
|
||||
- port: 443
|
||||
- name: webhook-server
|
||||
port: 443
|
||||
targetPort: 6443
|
||||
- name: webhook
|
||||
port: 9443
|
||||
targetPort: 9443
|
||||
|
||||
@@ -7,10 +7,11 @@ webhooks:
|
||||
failurePolicy: Fail
|
||||
clientConfig:
|
||||
service:
|
||||
# reach the webhook via the registered aggregated API
|
||||
namespace: default
|
||||
name: kubernetes
|
||||
path: /apis/admission.work.open-cluster-management.io/v1/manifestworkvalidators
|
||||
namespace: {{ .ClusterManagerNamespace }}
|
||||
name: cluster-manager-work-webhook
|
||||
path: /validate-work-open-cluster-management-io-v1-manifestwork
|
||||
port: {{.RegistrationWebhook.Port}}
|
||||
caBundle: {{ .RegistrationAPIServiceCABundle }}
|
||||
rules:
|
||||
- operations:
|
||||
- CREATE
|
||||
@@ -21,6 +22,6 @@ webhooks:
|
||||
- "*"
|
||||
resources:
|
||||
- manifestworks
|
||||
admissionReviewVersions: ["v1beta1"]
|
||||
admissionReviewVersions: ["v1beta1","v1"]
|
||||
sideEffects: None
|
||||
timeoutSeconds: 10
|
||||
|
||||
@@ -94,13 +94,22 @@ spec:
|
||||
name: kubeconfig
|
||||
readOnly: true
|
||||
{{ end }}
|
||||
{{ if not .HostedMode }}
|
||||
- name: {{ .ClusterManagerName }}-registration-conversion-webhook
|
||||
- name: {{ .ClusterManagerName }}-webhook
|
||||
image: {{ .RegistrationImage }}
|
||||
args:
|
||||
- /registration
|
||||
- "webhook-server"
|
||||
- "port=9443"
|
||||
{{ if gt (len .RegistrationFeatureGates) 0 }}
|
||||
{{range .RegistrationFeatureGates}}
|
||||
- {{ . }}
|
||||
{{ end }}
|
||||
{{ else }}
|
||||
- "--feature-gates=DefaultClusterSet=true"
|
||||
{{ end }}
|
||||
{{ if .HostedMode }}
|
||||
- "--kubeconfig=/var/run/secrets/hub/kubeconfig"
|
||||
{{ end }}
|
||||
imagePullPolicy: Always
|
||||
resources:
|
||||
requests:
|
||||
@@ -133,7 +142,11 @@ spec:
|
||||
- mountPath: /tmp/k8s-webhook-server/serving-certs
|
||||
name: webhook-secret
|
||||
readOnly: true
|
||||
{{ end }}
|
||||
{{ if .HostedMode }}
|
||||
- mountPath: /var/run/secrets/hub
|
||||
name: kubeconfig
|
||||
readOnly: true
|
||||
{{ end }}
|
||||
volumes:
|
||||
- name: webhook-secret
|
||||
secret:
|
||||
|
||||
@@ -40,10 +40,60 @@ spec:
|
||||
serviceAccountName: {{ .ClusterManagerName }}-work-webhook-sa
|
||||
{{ end }}
|
||||
containers:
|
||||
- name: {{ .ClusterManagerName }}-webhook
|
||||
image: {{ .WorkImage }}
|
||||
args:
|
||||
- /work
|
||||
- "webhook-server"
|
||||
- "port=9443"
|
||||
{{ if gt (len .WorkFeatureGates) 0 }}
|
||||
{{range .WorkFeatureGates}}
|
||||
- {{ . }}
|
||||
{{ end }}
|
||||
{{ end }}
|
||||
{{ if .HostedMode }}
|
||||
- "--kubeconfig=/var/run/secrets/hub/kubeconfig"
|
||||
{{ end }}
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop:
|
||||
- ALL
|
||||
privileged: false
|
||||
runAsNonRoot: true
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /healthz
|
||||
scheme: HTTP
|
||||
port: 8000
|
||||
initialDelaySeconds: 2
|
||||
periodSeconds: 10
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /healthz
|
||||
scheme: HTTP
|
||||
port: 8000
|
||||
initialDelaySeconds: 2
|
||||
resources:
|
||||
requests:
|
||||
cpu: 2m
|
||||
memory: 16Mi
|
||||
ports:
|
||||
- containerPort: 9443
|
||||
protocol: TCP
|
||||
volumeMounts:
|
||||
- mountPath: /tmp/k8s-webhook-server/serving-certs
|
||||
name: webhook-secret
|
||||
readOnly: true
|
||||
{{ if .HostedMode }}
|
||||
- mountPath: /var/run/secrets/hub
|
||||
name: kubeconfig
|
||||
readOnly: true
|
||||
{{ end }}
|
||||
- name: {{ .ClusterManagerName }}-work-webhook-sa
|
||||
image: {{ .WorkImage }}
|
||||
args:
|
||||
- "/work"
|
||||
- /work
|
||||
- "webhook"
|
||||
{{ if gt (len .WorkFeatureGates) 0 }}
|
||||
{{range .WorkFeatureGates}}
|
||||
@@ -83,6 +133,9 @@ spec:
|
||||
requests:
|
||||
cpu: 2m
|
||||
memory: 16Mi
|
||||
ports:
|
||||
- containerPort: 9443
|
||||
protocol: TCP
|
||||
volumeMounts:
|
||||
- name: webhook-secret
|
||||
mountPath: "/serving-cert"
|
||||
@@ -101,4 +154,3 @@ spec:
|
||||
secret:
|
||||
secretName: {{ .ClusterManagerName }}-work-webhook-sa-kubeconfig
|
||||
{{ end }}
|
||||
|
||||
|
||||
22
pkg/helpers/error.go
Normal file
22
pkg/helpers/error.go
Normal file
@@ -0,0 +1,22 @@
|
||||
package helpers
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
type RequeueError struct {
|
||||
RequeueTime time.Duration
|
||||
Message string
|
||||
}
|
||||
|
||||
func (r RequeueError) Error() string {
|
||||
return fmt.Sprintf(r.Message+", Requeue time: %v", r.RequeueTime)
|
||||
}
|
||||
|
||||
func NewRequeueError(msg string, requeueTime time.Duration) RequeueError {
|
||||
return RequeueError{
|
||||
RequeueTime: requeueTime,
|
||||
Message: msg,
|
||||
}
|
||||
}
|
||||
@@ -273,8 +273,7 @@ func ApplyValidatingWebhookConfiguration(
|
||||
if !*modified {
|
||||
return existing, false, nil
|
||||
}
|
||||
|
||||
actual, err := client.ValidatingWebhookConfigurations().Update(context.TODO(), existingCopy, metav1.UpdateOptions{})
|
||||
actual, err := client.ValidatingWebhookConfigurations().Update(context.TODO(), existing, metav1.UpdateOptions{})
|
||||
return actual, true, err
|
||||
}
|
||||
|
||||
@@ -301,7 +300,7 @@ func ApplyMutatingWebhookConfiguration(
|
||||
return existing, false, nil
|
||||
}
|
||||
|
||||
actual, err := client.MutatingWebhookConfigurations().Update(context.TODO(), existingCopy, metav1.UpdateOptions{})
|
||||
actual, err := client.MutatingWebhookConfigurations().Update(context.TODO(), existing, metav1.UpdateOptions{})
|
||||
return actual, true, err
|
||||
}
|
||||
|
||||
|
||||
@@ -3,6 +3,8 @@ package clustermanagercontroller
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
|
||||
errorhelpers "errors"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strings"
|
||||
@@ -48,14 +50,13 @@ var (
|
||||
"managedclusters.cluster.open-cluster-management.io",
|
||||
}
|
||||
|
||||
namespaceResource = "cluster-manager/cluster-manager-namespace.yaml"
|
||||
hostedClusterSetCrdFile = "cluster-manager/hub/0000_00_clusters.open-cluster-management.io_managedclustersets.crd-hosted.yaml"
|
||||
clusterSetCrdFile = "cluster-manager/hub/0000_00_clusters.open-cluster-management.io_managedclustersets.crd.yaml"
|
||||
namespaceResource = "cluster-manager/cluster-manager-namespace.yaml"
|
||||
|
||||
// crdResourceFiles should be deployed in the hub cluster
|
||||
hubCRDResourceFiles = []string{
|
||||
"cluster-manager/hub/0000_00_addon.open-cluster-management.io_clustermanagementaddons.crd.yaml",
|
||||
"cluster-manager/hub/0000_00_clusters.open-cluster-management.io_managedclusters.crd.yaml",
|
||||
"cluster-manager/hub/0000_00_clusters.open-cluster-management.io_managedclustersets.crd.yaml",
|
||||
"cluster-manager/hub/0000_00_work.open-cluster-management.io_manifestworks.crd.yaml",
|
||||
"cluster-manager/hub/0000_01_addon.open-cluster-management.io_managedclusteraddons.crd.yaml",
|
||||
"cluster-manager/hub/0000_01_clusters.open-cluster-management.io_managedclustersetbindings.crd.yaml",
|
||||
@@ -75,17 +76,17 @@ var (
|
||||
|
||||
// The hubWebhookResourceFiles should be deployed in the hub cluster
|
||||
// The service should may point to a external url which represent the webhook-server's address.
|
||||
hubWebhookResourceFiles = []string{
|
||||
// registration-webhook
|
||||
hubRegistrationWebhookResourceFiles = []string{
|
||||
"cluster-manager/hub/cluster-manager-registration-webhook-validatingconfiguration.yaml",
|
||||
"cluster-manager/hub/cluster-manager-registration-webhook-mutatingconfiguration.yaml",
|
||||
"cluster-manager/hub/cluster-manager-registration-webhook-clustersetbinding-validatingconfiguration.yaml",
|
||||
// work-webhook
|
||||
"cluster-manager/hub/cluster-manager-registration-webhook-clustersetbinding-validatingconfiguration-v1beta1.yaml",
|
||||
}
|
||||
hubWorkWebhookResourceFiles = []string{
|
||||
"cluster-manager/hub/cluster-manager-work-webhook-validatingconfiguration.yaml",
|
||||
}
|
||||
|
||||
// The apiservice resources should be applied after CABundle created.
|
||||
// And also should be deployed in the hub cluster.
|
||||
// The apiservice resources should be deleted
|
||||
hubApiserviceFiles = []string{
|
||||
"cluster-manager/hub/cluster-manager-work-webhook-apiservice.yaml",
|
||||
"cluster-manager/hub/cluster-manager-registration-webhook-apiservice.yaml",
|
||||
@@ -136,10 +137,13 @@ var (
|
||||
const (
|
||||
clusterManagerFinalizer = "operator.open-cluster-management.io/cluster-manager-cleanup"
|
||||
clusterManagerApplied = "Applied"
|
||||
caBundleConfigmap = "ca-bundle-configmap"
|
||||
|
||||
caBundleConfigmap = "ca-bundle-configmap"
|
||||
|
||||
hubRegistrationFeatureGatesValid = "ValidRegistrationFeatureGates"
|
||||
hubWorkFeatureGatesValid = "ValidWorkFeatureGates"
|
||||
defaultWebhookPort = int32(9443)
|
||||
clusterManagerReSyncTime = 5 * time.Second
|
||||
)
|
||||
|
||||
type clusterManagerController struct {
|
||||
@@ -292,6 +296,10 @@ func (n *clusterManagerController) sync(ctx context.Context, controllerContext f
|
||||
meta.SetStatusCondition(conditions, condition)
|
||||
}
|
||||
|
||||
//Set default port
|
||||
config.RegistrationWebhook.Port = defaultWebhookPort
|
||||
config.WorkWebhook.Port = defaultWebhookPort
|
||||
|
||||
// If we are deploying in the hosted mode, it requires us to create webhook in a different way with the default mode.
|
||||
// In the hosted mode, the webhook servers is running in the management cluster but the users are accessing the hub cluster.
|
||||
// So we need to add configuration to make the apiserver of the hub cluster could access the webhook servers on the management cluster.
|
||||
@@ -345,6 +353,23 @@ func (n *clusterManagerController) sync(ctx context.Context, controllerContext f
|
||||
return err
|
||||
}
|
||||
|
||||
//get caBundle
|
||||
caBundle := "placeholder"
|
||||
configmap, err := n.configMapLister.ConfigMaps(clusterManagerNamespace).Get(caBundleConfigmap)
|
||||
switch {
|
||||
case errors.IsNotFound(err):
|
||||
// do nothing
|
||||
case err != nil:
|
||||
return err
|
||||
default:
|
||||
if cb := configmap.Data["ca-bundle.crt"]; len(cb) > 0 {
|
||||
caBundle = cb
|
||||
}
|
||||
}
|
||||
encodedCaBundle := base64.StdEncoding.EncodeToString([]byte(caBundle))
|
||||
config.RegistrationAPIServiceCABundle = encodedCaBundle
|
||||
config.WorkAPIServiceCABundle = encodedCaBundle
|
||||
|
||||
// Apply resources on the hub cluster
|
||||
hubAppliedErrs, err := applyHubResources(
|
||||
ctx,
|
||||
@@ -353,7 +378,7 @@ func (n *clusterManagerController) sync(ctx context.Context, controllerContext f
|
||||
config,
|
||||
&relatedResources,
|
||||
hubClient, hubApiExtensionClient, hubApiRegistrationClient,
|
||||
n.configMapLister, n.recorder, n.cache)
|
||||
n.recorder, n.cache)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -370,9 +395,34 @@ func (n *clusterManagerController) sync(ctx context.Context, controllerContext f
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
errs := append(hubAppliedErrs, managementAppliedErrs...)
|
||||
|
||||
//Check registration and work webhook pod running, then apply webhook config files
|
||||
registrationWebhookName := clusterManagerName + "-registration-webhook"
|
||||
workWebhookName := clusterManagerName + "-work-webhook"
|
||||
rqe := helpers.RequeueError{}
|
||||
|
||||
err = applyWebhookConfig(ctx, registrationWebhookName, hubRegistrationWebhookResourceFiles, &relatedResources, config, hubClient, managementClient, n.recorder, n.cache)
|
||||
if err != nil {
|
||||
if errorhelpers.As(err, &rqe) {
|
||||
klog.Warning("Apply webhook config %v fail. Error: %v", registrationWebhookName, err)
|
||||
controllerContext.Queue().AddAfter(clusterManagerName, err.(helpers.RequeueError).RequeueTime)
|
||||
} else {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
|
||||
err = applyWebhookConfig(ctx, workWebhookName, hubWorkWebhookResourceFiles, &relatedResources, config, hubClient, managementClient, n.recorder, n.cache)
|
||||
if err != nil {
|
||||
if errorhelpers.As(err, &rqe) {
|
||||
klog.Warning("Apply webhook config %v fail. Error: %v", registrationWebhookName, err)
|
||||
controllerContext.Queue().AddAfter(clusterManagerName, err.(helpers.RequeueError).RequeueTime)
|
||||
} else {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Update status
|
||||
errs := append(hubAppliedErrs, managementAppliedErrs...)
|
||||
observedGeneration := clusterManager.Status.ObservedGeneration
|
||||
if len(errs) == 0 {
|
||||
meta.SetStatusCondition(conditions, metav1.Condition{
|
||||
@@ -416,31 +466,14 @@ func applyHubResources(
|
||||
relatedResources *[]operatorapiv1.RelatedResourceMeta,
|
||||
// hub clients
|
||||
hubClient kubernetes.Interface, hubApiExtensionClient apiextensionsclient.Interface, hubApiRegistrationClient apiregistrationclient.APIServicesGetter,
|
||||
configMapLister corev1listers.ConfigMapLister,
|
||||
recorder events.Recorder,
|
||||
cache resourceapply.ResourceCache,
|
||||
) (appliedErrs []error, err error) {
|
||||
// Try to load ca bundle from configmap
|
||||
// If the configmap is found, populate it into configmap.
|
||||
// If the configmap not found yet, skip this and apply other resources first.
|
||||
caBundle := "placeholder"
|
||||
configmap, err := configMapLister.ConfigMaps(clusterManagerNamespace).Get(caBundleConfigmap)
|
||||
switch {
|
||||
case errors.IsNotFound(err):
|
||||
// do nothing
|
||||
case err != nil:
|
||||
return appliedErrs, err
|
||||
default:
|
||||
if cb := configmap.Data["ca-bundle.crt"]; len(cb) > 0 {
|
||||
caBundle = cb
|
||||
}
|
||||
}
|
||||
encodedCaBundle := base64.StdEncoding.EncodeToString([]byte(caBundle))
|
||||
manifestsConfig.RegistrationAPIServiceCABundle = encodedCaBundle
|
||||
manifestsConfig.WorkAPIServiceCABundle = encodedCaBundle
|
||||
|
||||
// Apply hub cluster resources
|
||||
// ClusterSet crd need caBundle, so we need to apply the hubresources after get caBundle
|
||||
hubResources := getHubResources(clusterManagerMode, manifestsConfig.RegistrationWebhook.IsIPFormat, manifestsConfig.WorkWebhook.IsIPFormat, false)
|
||||
resourceResults := helpers.ApplyDirectly(
|
||||
ctx,
|
||||
@@ -467,14 +500,34 @@ func applyHubResources(
|
||||
}
|
||||
}
|
||||
|
||||
// Apply Apiservice files to hub cluster.
|
||||
// The reason why apply Apiservice after apply other staticfiles(including namespace) is because Apiservices requires the CABundleConfigmap.
|
||||
// And it will return an error(uncatchable with NotFound type) if the namespace is not created.
|
||||
apiserviceResults := helpers.ApplyDirectly(
|
||||
return appliedErrs, nil
|
||||
}
|
||||
|
||||
func applyWebhookConfig(
|
||||
ctx context.Context,
|
||||
webhookDeploymentName string,
|
||||
webhookResourceFiles []string,
|
||||
relatedResources *[]operatorapiv1.RelatedResourceMeta,
|
||||
manifestsConfig manifests.HubConfig,
|
||||
// hub clients
|
||||
hubClient kubernetes.Interface,
|
||||
managementKubeClient kubernetes.Interface,
|
||||
recorder events.Recorder,
|
||||
cache resourceapply.ResourceCache,
|
||||
) error {
|
||||
var appliedErrs []error
|
||||
//Check registration webhook running
|
||||
err := checkWebhookPodRunning(manifestsConfig.ClusterManagerNamespace, managementKubeClient, webhookDeploymentName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// If all webhook pod running , then apply webhook config files
|
||||
resourceResults := helpers.ApplyDirectly(
|
||||
ctx,
|
||||
hubClient,
|
||||
hubApiExtensionClient,
|
||||
hubApiRegistrationClient,
|
||||
nil,
|
||||
nil,
|
||||
nil,
|
||||
recorder,
|
||||
cache,
|
||||
@@ -487,15 +540,32 @@ func applyHubResources(
|
||||
helpers.SetRelatedResourcesStatusesWithObj(relatedResources, objData)
|
||||
return objData, nil
|
||||
},
|
||||
hubApiserviceFiles...,
|
||||
webhookResourceFiles...,
|
||||
)
|
||||
for _, result := range apiserviceResults {
|
||||
for _, result := range resourceResults {
|
||||
if result.Error != nil {
|
||||
appliedErrs = append(appliedErrs, fmt.Errorf("%q (%T): %v", result.File, result.Type, result.Error))
|
||||
}
|
||||
}
|
||||
return operatorhelpers.NewMultiLineAggregate(appliedErrs)
|
||||
}
|
||||
|
||||
return appliedErrs, nil
|
||||
func checkWebhookPodRunning(clusterManagerNamespace string, managementKubeClient kubernetes.Interface, webhookName string) error {
|
||||
webhookDeployment, err := managementKubeClient.AppsV1().Deployments(clusterManagerNamespace).Get(context.Background(),
|
||||
webhookName, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if webhookDeployment.Generation != webhookDeployment.Status.ObservedGeneration {
|
||||
return helpers.NewRequeueError(fmt.Sprintf("New %v webhook deployment not observed. Expect generation:%v, Observed Generation:%v", webhookName, webhookDeployment.Generation, webhookDeployment.Status.ObservedGeneration), clusterManagerReSyncTime)
|
||||
}
|
||||
replicas := *webhookDeployment.Spec.Replicas
|
||||
readyReplicas := webhookDeployment.Status.ReadyReplicas
|
||||
|
||||
if readyReplicas != replicas {
|
||||
return helpers.NewRequeueError(fmt.Sprintf("Deployment %s not ready. Replica:%v, ReadyReplicas:%v", webhookName, replicas, readyReplicas), clusterManagerReSyncTime)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func applyManagementResources(
|
||||
@@ -679,7 +749,12 @@ func cleanUpHub(ctx context.Context, controllerContext factory.SyncContext,
|
||||
}
|
||||
|
||||
// Remove All Static files
|
||||
// API service should be deleted in 0.10.0
|
||||
// TODO: should remove apiservice files future
|
||||
hubResources := append(getHubResources(mode, config.RegistrationWebhook.IsIPFormat, config.WorkWebhook.IsIPFormat, skipRemoveCRDs), hubApiserviceFiles...)
|
||||
hubResources = append(hubResources, hubRegistrationWebhookResourceFiles...)
|
||||
hubResources = append(hubResources, hubWorkWebhookResourceFiles...)
|
||||
|
||||
for _, file := range hubResources {
|
||||
err := helpers.CleanUpStaticObject(
|
||||
ctx,
|
||||
@@ -781,16 +856,9 @@ func getSAs(clusterManagerName string) []string {
|
||||
func getHubResources(mode operatorapiv1.InstallMode, isRegistrationIPFormat, isWorkIPFormat, skipAddCRDs bool) []string {
|
||||
hubResources := []string{namespaceResource}
|
||||
if !skipAddCRDs {
|
||||
//TODO: Currently, in hosted mode, clusterset only support v1beta1 version, will fix it in future releases
|
||||
if mode == operatorapiv1.InstallModeHosted {
|
||||
hubResources = append(hubResources, hostedClusterSetCrdFile)
|
||||
} else {
|
||||
hubResources = append(hubResources, clusterSetCrdFile)
|
||||
}
|
||||
hubResources = append(hubResources, hubCRDResourceFiles...)
|
||||
}
|
||||
|
||||
hubResources = append(hubResources, hubWebhookResourceFiles...)
|
||||
hubResources = append(hubResources, hubRbacResourceFiles...)
|
||||
|
||||
// the hubHostedWebhookServiceFiles are only used in hosted mode
|
||||
|
||||
@@ -87,9 +87,62 @@ func newTestController(t *testing.T, clustermanager *operatorapiv1.ClusterManage
|
||||
}
|
||||
}
|
||||
|
||||
func setup(t *testing.T, tc *testController, crds ...runtime.Object) {
|
||||
func setWebhookDeployment(clusterManagerName, clusterManagerNamespace string) []runtime.Object {
|
||||
var replicas = int32(1)
|
||||
|
||||
return []runtime.Object{
|
||||
&appsv1.Deployment{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: clusterManagerName + "-registration-webhook",
|
||||
Namespace: clusterManagerNamespace,
|
||||
Generation: 1,
|
||||
},
|
||||
Spec: appsv1.DeploymentSpec{
|
||||
Template: corev1.PodTemplateSpec{
|
||||
Spec: corev1.PodSpec{
|
||||
Containers: []corev1.Container{
|
||||
{
|
||||
Name: clusterManagerName + "-webhook",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
Replicas: &replicas,
|
||||
},
|
||||
Status: appsv1.DeploymentStatus{
|
||||
ReadyReplicas: replicas,
|
||||
ObservedGeneration: 1,
|
||||
},
|
||||
},
|
||||
&appsv1.Deployment{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: clusterManagerName + "-work-webhook",
|
||||
Namespace: clusterManagerNamespace,
|
||||
Generation: 1,
|
||||
},
|
||||
Spec: appsv1.DeploymentSpec{
|
||||
Template: corev1.PodTemplateSpec{
|
||||
Spec: corev1.PodSpec{
|
||||
Containers: []corev1.Container{
|
||||
{
|
||||
Name: clusterManagerName + "-webhook",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
Replicas: &replicas,
|
||||
},
|
||||
Status: appsv1.DeploymentStatus{
|
||||
ReadyReplicas: replicas,
|
||||
ObservedGeneration: 1,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func setup(t *testing.T, tc *testController, cd []runtime.Object, crds ...runtime.Object) {
|
||||
fakeHubKubeClient := fakekube.NewSimpleClientset()
|
||||
fakeManagementKubeClient := fakekube.NewSimpleClientset()
|
||||
fakeManagementKubeClient := fakekube.NewSimpleClientset(cd...)
|
||||
fakeAPIExtensionClient := fakeapiextensions.NewSimpleClientset(crds...)
|
||||
fakeAPIRegistrationClient := fakeapiregistration.NewSimpleClientset()
|
||||
fakeMigrationClient := fakemigrationclient.NewSimpleClientset()
|
||||
@@ -134,7 +187,9 @@ func ensureObject(t *testing.T, object runtime.Object, hubCore *operatorapiv1.Cl
|
||||
func TestSyncDeploy(t *testing.T) {
|
||||
clusterManager := newClusterManager("testhub")
|
||||
tc := newTestController(t, clusterManager)
|
||||
setup(t, tc)
|
||||
clusterManagerNamespace := helpers.ClusterManagerNamespace(clusterManager.Name, clusterManager.Spec.DeployOption.Mode)
|
||||
cd := setWebhookDeployment(clusterManager.Name, clusterManagerNamespace)
|
||||
setup(t, tc, cd)
|
||||
|
||||
syncContext := testinghelper.NewFakeSyncContext(t, "testhub")
|
||||
|
||||
@@ -154,7 +209,7 @@ func TestSyncDeploy(t *testing.T) {
|
||||
|
||||
// Check if resources are created as expected
|
||||
// We expect creat the namespace twice respectively in the management cluster and the hub cluster.
|
||||
testinghelper.AssertEqualNumber(t, len(createKubeObjects), 24)
|
||||
testinghelper.AssertEqualNumber(t, len(createKubeObjects), 23)
|
||||
for _, object := range createKubeObjects {
|
||||
ensureObject(t, object, clusterManager)
|
||||
}
|
||||
@@ -169,24 +224,46 @@ func TestSyncDeploy(t *testing.T) {
|
||||
}
|
||||
// Check if resources are created as expected
|
||||
testinghelper.AssertEqualNumber(t, len(createCRDObjects), 10)
|
||||
}
|
||||
|
||||
createAPIServiceObjects := []runtime.Object{}
|
||||
apiServiceActions := tc.apiRegistrationClient.Actions()
|
||||
for _, action := range apiServiceActions {
|
||||
func TestSyncDeployNoWebhook(t *testing.T) {
|
||||
clusterManager := newClusterManager("testhub")
|
||||
tc := newTestController(t, clusterManager)
|
||||
setup(t, tc, nil)
|
||||
|
||||
syncContext := testinghelper.NewFakeSyncContext(t, "testhub")
|
||||
|
||||
err := tc.clusterManagerController.sync(ctx, syncContext)
|
||||
if err != nil {
|
||||
t.Fatalf("Expected no error when sync, %v", err)
|
||||
}
|
||||
|
||||
createKubeObjects := []runtime.Object{}
|
||||
kubeActions := append(tc.hubKubeClient.Actions(), tc.managementKubeClient.Actions()...) // record objects from both hub and management cluster
|
||||
for _, action := range kubeActions {
|
||||
if action.GetVerb() == "create" {
|
||||
object := action.(clienttesting.CreateActionImpl).Object
|
||||
createAPIServiceObjects = append(createAPIServiceObjects, object)
|
||||
createKubeObjects = append(createKubeObjects, object)
|
||||
}
|
||||
}
|
||||
|
||||
// Check if resources are created as expected
|
||||
// We expect creat the namespace twice respectively in the management cluster and the hub cluster.
|
||||
testinghelper.AssertEqualNumber(t, len(createKubeObjects), 20)
|
||||
for _, object := range createKubeObjects {
|
||||
ensureObject(t, object, clusterManager)
|
||||
}
|
||||
|
||||
createCRDObjects := []runtime.Object{}
|
||||
crdActions := tc.apiExtensionClient.Actions()
|
||||
for _, action := range crdActions {
|
||||
if action.GetVerb() == "create" {
|
||||
object := action.(clienttesting.CreateActionImpl).Object
|
||||
createCRDObjects = append(createCRDObjects, object)
|
||||
}
|
||||
}
|
||||
// Check if resources are created as expected
|
||||
testinghelper.AssertEqualNumber(t, len(createAPIServiceObjects), 2)
|
||||
|
||||
clusterManagerAction := tc.operatorClient.Actions()
|
||||
testinghelper.AssertEqualNumber(t, len(clusterManagerAction), 2)
|
||||
testinghelper.AssertAction(t, clusterManagerAction[1], "update")
|
||||
testinghelper.AssertOnlyConditions(
|
||||
t, clusterManagerAction[1].(clienttesting.UpdateActionImpl).Object,
|
||||
testinghelper.NamedCondition(clusterManagerApplied, "ClusterManagerApplied", metav1.ConditionTrue))
|
||||
testinghelper.AssertEqualNumber(t, len(createCRDObjects), 10)
|
||||
}
|
||||
|
||||
// TestSyncDelete test cleanup hub deploy
|
||||
@@ -196,7 +273,7 @@ func TestSyncDelete(t *testing.T) {
|
||||
clusterManager.ObjectMeta.SetDeletionTimestamp(&now)
|
||||
|
||||
tc := newTestController(t, clusterManager)
|
||||
setup(t, tc)
|
||||
setup(t, tc, nil)
|
||||
|
||||
syncContext := testinghelper.NewFakeSyncContext(t, "testhub")
|
||||
clusterManagerNamespace := helpers.ClusterManagerNamespace(clusterManager.Name, clusterManager.Spec.DeployOption.Mode)
|
||||
@@ -214,7 +291,7 @@ func TestSyncDelete(t *testing.T) {
|
||||
deleteKubeActions = append(deleteKubeActions, deleteKubeAction)
|
||||
}
|
||||
}
|
||||
testinghelper.AssertEqualNumber(t, len(deleteKubeActions), 20) // delete namespace both from the hub cluster and the mangement cluster
|
||||
testinghelper.AssertEqualNumber(t, len(deleteKubeActions), 21) // delete namespace both from the hub cluster and the mangement cluster
|
||||
|
||||
deleteCRDActions := []clienttesting.DeleteActionImpl{}
|
||||
crdActions := tc.apiExtensionClient.Actions()
|
||||
@@ -258,7 +335,7 @@ func TestDeleteCRD(t *testing.T) {
|
||||
}
|
||||
|
||||
tc := newTestController(t, clusterManager)
|
||||
setup(t, tc, crd)
|
||||
setup(t, tc, nil, crd)
|
||||
|
||||
// Return crd with the first get, and return not found with the 2nd get
|
||||
getCount := 0
|
||||
|
||||
@@ -492,16 +492,33 @@ func (t *Tester) CheckHubReady() error {
|
||||
Get(context.TODO(), t.hubRegistrationDeployment, metav1.GetOptions{}); err != nil {
|
||||
return err
|
||||
}
|
||||
gomega.Eventually(func() error {
|
||||
registrationWebhookDeployment, err := t.KubeClient.AppsV1().Deployments(t.clusterManagerNamespace).
|
||||
Get(context.TODO(), t.hubRegistrationWebhookDeployment, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
replicas := *registrationWebhookDeployment.Spec.Replicas
|
||||
readyReplicas := registrationWebhookDeployment.Status.ReadyReplicas
|
||||
if readyReplicas != replicas {
|
||||
return fmt.Errorf("deployment %s should have %d but got %d ready replicas", t.hubRegistrationWebhookDeployment, replicas, readyReplicas)
|
||||
}
|
||||
return nil
|
||||
}, t.EventuallyTimeout*5, t.EventuallyInterval*5).Should(gomega.BeNil())
|
||||
|
||||
if _, err := t.KubeClient.AppsV1().Deployments(t.clusterManagerNamespace).
|
||||
Get(context.TODO(), t.hubRegistrationWebhookDeployment, metav1.GetOptions{}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if _, err := t.KubeClient.AppsV1().Deployments(t.clusterManagerNamespace).
|
||||
Get(context.TODO(), t.hubWorkWebhookDeployment, metav1.GetOptions{}); err != nil {
|
||||
return err
|
||||
}
|
||||
gomega.Eventually(func() error {
|
||||
workWebhookDeployment, err := t.KubeClient.AppsV1().Deployments(t.clusterManagerNamespace).
|
||||
Get(context.TODO(), t.hubWorkWebhookDeployment, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
replicas := *workWebhookDeployment.Spec.Replicas
|
||||
readyReplicas := workWebhookDeployment.Status.ReadyReplicas
|
||||
if readyReplicas != replicas {
|
||||
return fmt.Errorf("deployment %s should have %d but got %d ready replicas", t.hubWorkWebhookDeployment, replicas, readyReplicas)
|
||||
}
|
||||
return nil
|
||||
}, t.EventuallyTimeout*5, t.EventuallyInterval*5).Should(gomega.BeNil())
|
||||
|
||||
if _, err := t.KubeClient.AppsV1().Deployments(t.clusterManagerNamespace).
|
||||
Get(context.TODO(), t.hubPlacementDeployment, metav1.GetOptions{}); err != nil {
|
||||
|
||||
@@ -7,23 +7,34 @@ import (
|
||||
|
||||
"github.com/onsi/ginkgo/v2"
|
||||
"github.com/onsi/gomega"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
v1 "open-cluster-management.io/api/operator/v1"
|
||||
|
||||
appsv1 "k8s.io/api/apps/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/util/cert"
|
||||
|
||||
"github.com/openshift/library-go/pkg/operator/resource/resourceapply"
|
||||
|
||||
v1 "open-cluster-management.io/api/operator/v1"
|
||||
"open-cluster-management.io/registration-operator/pkg/helpers"
|
||||
"open-cluster-management.io/registration-operator/test/integration/util"
|
||||
|
||||
appsv1 "k8s.io/api/apps/v1"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
"k8s.io/client-go/util/cert"
|
||||
)
|
||||
|
||||
func updateDeploymentStatus(kubeClient kubernetes.Interface, namespace, deploymentName string) {
|
||||
deployment, err := kubeClient.AppsV1().Deployments(namespace).Get(context.Background(), deploymentName, metav1.GetOptions{})
|
||||
gomega.Expect(err).ToNot(gomega.HaveOccurred())
|
||||
deployment.Status.Replicas = *deployment.Spec.Replicas
|
||||
deployment.Status.ReadyReplicas = *deployment.Spec.Replicas
|
||||
deployment.Status.ObservedGeneration = *&deployment.Generation
|
||||
_, err = kubeClient.AppsV1().Deployments(namespace).UpdateStatus(context.Background(), deployment, metav1.UpdateOptions{})
|
||||
gomega.Expect(err).ToNot(gomega.HaveOccurred())
|
||||
}
|
||||
|
||||
var _ = ginkgo.Describe("ClusterManager Hosted Mode", func() {
|
||||
var hostedCtx context.Context
|
||||
var hostedCancel context.CancelFunc
|
||||
var hubRegistrationDeployment = fmt.Sprintf("%s-registration-controller", clusterManagerName)
|
||||
var hubRegistrationWebhookDeployment = fmt.Sprintf("%s-registration-webhook", clusterManagerName)
|
||||
var hubWorkWebhookDeployment = fmt.Sprintf("%s-work-webhook", clusterManagerName)
|
||||
|
||||
ginkgo.BeforeEach(func() {
|
||||
hostedCtx, hostedCancel = context.WithCancel(context.Background())
|
||||
@@ -143,7 +154,6 @@ var _ = ginkgo.Describe("ClusterManager Hosted Mode", func() {
|
||||
return nil
|
||||
}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeNil())
|
||||
|
||||
hubRegistrationWebhookDeployment := fmt.Sprintf("%s-registration-webhook", clusterManagerName)
|
||||
gomega.Eventually(func() error {
|
||||
if _, err := hostedKubeClient.AppsV1().Deployments(hubNamespaceHosted).Get(hostedCtx, hubRegistrationWebhookDeployment, metav1.GetOptions{}); err != nil {
|
||||
return err
|
||||
@@ -151,7 +161,6 @@ var _ = ginkgo.Describe("ClusterManager Hosted Mode", func() {
|
||||
return nil
|
||||
}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeNil())
|
||||
|
||||
hubWorkWebhookDeployment := fmt.Sprintf("%s-work-webhook", clusterManagerName)
|
||||
gomega.Eventually(func() error {
|
||||
if _, err := hostedKubeClient.AppsV1().Deployments(hubNamespaceHosted).Get(hostedCtx, hubWorkWebhookDeployment, metav1.GetOptions{}); err != nil {
|
||||
return err
|
||||
@@ -209,6 +218,13 @@ var _ = ginkgo.Describe("ClusterManager Hosted Mode", func() {
|
||||
|
||||
// Check validating webhook
|
||||
registrationValidtingWebhook := "managedclustervalidators.admission.cluster.open-cluster-management.io"
|
||||
|
||||
//Should not apply the webhook config if the replica and observed is not set
|
||||
_, err := hostedKubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Get(hostedCtx, registrationValidtingWebhook, metav1.GetOptions{})
|
||||
gomega.Expect(err).To(gomega.HaveOccurred())
|
||||
|
||||
updateDeploymentStatus(hostedKubeClient, hubNamespaceHosted, hubRegistrationWebhookDeployment)
|
||||
|
||||
gomega.Eventually(func() error {
|
||||
if _, err := hostedKubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Get(hostedCtx, registrationValidtingWebhook, metav1.GetOptions{}); err != nil {
|
||||
return err
|
||||
@@ -217,6 +233,13 @@ var _ = ginkgo.Describe("ClusterManager Hosted Mode", func() {
|
||||
}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeNil())
|
||||
|
||||
workValidtingWebhook := "manifestworkvalidators.admission.work.open-cluster-management.io"
|
||||
//Should not apply the webhook config if the replica and observed is not set
|
||||
_, err = hostedKubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Get(hostedCtx, workValidtingWebhook, metav1.GetOptions{})
|
||||
|
||||
gomega.Expect(err).To(gomega.HaveOccurred())
|
||||
|
||||
updateDeploymentStatus(hostedKubeClient, hubNamespaceHosted, hubWorkWebhookDeployment)
|
||||
|
||||
gomega.Eventually(func() error {
|
||||
if _, err := hostedKubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Get(hostedCtx, workValidtingWebhook, metav1.GetOptions{}); err != nil {
|
||||
return err
|
||||
@@ -267,6 +290,8 @@ var _ = ginkgo.Describe("ClusterManager Hosted Mode", func() {
|
||||
return nil
|
||||
}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeNil())
|
||||
|
||||
updateDeploymentStatus(hostedKubeClient, hubNamespaceHosted, hubRegistrationWebhookDeployment)
|
||||
|
||||
// Check if generations are correct
|
||||
gomega.Eventually(func() error {
|
||||
actual, err := hostedOperatorClient.OperatorV1().ClusterManagers().Get(hostedCtx, clusterManagerName, metav1.GetOptions{})
|
||||
@@ -287,12 +312,11 @@ var _ = ginkgo.Describe("ClusterManager Hosted Mode", func() {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(actual.Status.RelatedResources) != 35 {
|
||||
return fmt.Errorf("should get 35 relatedResources, actual got %v", len(actual.Status.RelatedResources))
|
||||
if len(actual.Status.RelatedResources) != 34 {
|
||||
return fmt.Errorf("should get 34 relatedResources, actual got %v", len(actual.Status.RelatedResources))
|
||||
}
|
||||
return nil
|
||||
}, eventuallyTimeout, eventuallyInterval).ShouldNot(gomega.HaveOccurred())
|
||||
|
||||
})
|
||||
|
||||
ginkgo.It("Deployment should be added nodeSelector and toleration when add nodePlacement into clustermanager", func() {
|
||||
@@ -334,6 +358,10 @@ var _ = ginkgo.Describe("ClusterManager Hosted Mode", func() {
|
||||
|
||||
return fmt.Errorf("no key equals to node-role.kubernetes.io/infra")
|
||||
}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeNil())
|
||||
|
||||
updateDeploymentStatus(hostedKubeClient, hubNamespaceHosted, hubRegistrationWebhookDeployment)
|
||||
updateDeploymentStatus(hostedKubeClient, hubNamespaceHosted, hubWorkWebhookDeployment)
|
||||
|
||||
})
|
||||
ginkgo.It("Deployment should be reconciled when manually updated", func() {
|
||||
gomega.Eventually(func() error {
|
||||
|
||||
@@ -47,6 +47,8 @@ func startHubOperator(ctx context.Context, mode v1.InstallMode) {
|
||||
var _ = ginkgo.Describe("ClusterManager Default Mode", func() {
|
||||
var cancel context.CancelFunc
|
||||
var hubRegistrationDeployment = fmt.Sprintf("%s-registration-controller", clusterManagerName)
|
||||
var hubRegistrationWebhookDeployment = fmt.Sprintf("%s-registration-webhook", clusterManagerName)
|
||||
var hubWorkWebhookDeployment = fmt.Sprintf("%s-work-webhook", clusterManagerName)
|
||||
|
||||
ginkgo.BeforeEach(func() {
|
||||
var ctx context.Context
|
||||
@@ -69,7 +71,6 @@ var _ = ginkgo.Describe("ClusterManager Default Mode", func() {
|
||||
}
|
||||
return nil
|
||||
}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeNil())
|
||||
|
||||
// Check clusterrole/clusterrolebinding
|
||||
hubRegistrationClusterRole := fmt.Sprintf("open-cluster-management:%s-registration:controller", clusterManagerName)
|
||||
hubRegistrationWebhookClusterRole := fmt.Sprintf("open-cluster-management:%s-registration:webhook", clusterManagerName)
|
||||
@@ -80,6 +81,7 @@ var _ = ginkgo.Describe("ClusterManager Default Mode", func() {
|
||||
}
|
||||
return nil
|
||||
}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeNil())
|
||||
|
||||
gomega.Eventually(func() error {
|
||||
if _, err := kubeClient.RbacV1().ClusterRoles().Get(context.Background(), hubRegistrationWebhookClusterRole, metav1.GetOptions{}); err != nil {
|
||||
return err
|
||||
@@ -104,6 +106,7 @@ var _ = ginkgo.Describe("ClusterManager Default Mode", func() {
|
||||
}
|
||||
return nil
|
||||
}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeNil())
|
||||
|
||||
gomega.Eventually(func() error {
|
||||
if _, err := kubeClient.RbacV1().ClusterRoleBindings().Get(context.Background(), hubWorkWebhookClusterRole, metav1.GetOptions{}); err != nil {
|
||||
return err
|
||||
@@ -142,7 +145,6 @@ var _ = ginkgo.Describe("ClusterManager Default Mode", func() {
|
||||
return nil
|
||||
}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeNil())
|
||||
|
||||
hubRegistrationWebhookDeployment := fmt.Sprintf("%s-registration-webhook", clusterManagerName)
|
||||
gomega.Eventually(func() error {
|
||||
if _, err := kubeClient.AppsV1().Deployments(hubNamespace).Get(context.Background(), hubRegistrationWebhookDeployment, metav1.GetOptions{}); err != nil {
|
||||
return err
|
||||
@@ -150,7 +152,6 @@ var _ = ginkgo.Describe("ClusterManager Default Mode", func() {
|
||||
return nil
|
||||
}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeNil())
|
||||
|
||||
hubWorkWebhookDeployment := fmt.Sprintf("%s-work-webhook", clusterManagerName)
|
||||
gomega.Eventually(func() error {
|
||||
if _, err := kubeClient.AppsV1().Deployments(hubNamespace).Get(context.Background(), hubWorkWebhookDeployment, metav1.GetOptions{}); err != nil {
|
||||
return err
|
||||
@@ -208,21 +209,34 @@ var _ = ginkgo.Describe("ClusterManager Default Mode", func() {
|
||||
|
||||
// Check validating webhook
|
||||
registrationValidtingWebhook := "managedclustervalidators.admission.cluster.open-cluster-management.io"
|
||||
|
||||
//Should not apply the webhook config if the replica and observed is not set
|
||||
_, err := kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Get(context.Background(), registrationValidtingWebhook, metav1.GetOptions{})
|
||||
gomega.Expect(err).To(gomega.HaveOccurred())
|
||||
// Update readyreplica of deployment
|
||||
|
||||
updateDeploymentStatus(kubeClient, hubNamespace, hubRegistrationWebhookDeployment)
|
||||
|
||||
gomega.Eventually(func() error {
|
||||
if _, err := kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Get(context.Background(), registrationValidtingWebhook, metav1.GetOptions{}); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeNil())
|
||||
}, eventuallyTimeout*10, eventuallyInterval).Should(gomega.BeNil())
|
||||
|
||||
workValidtingWebhook := "manifestworkvalidators.admission.work.open-cluster-management.io"
|
||||
//Should not apply the webhook config if the replica and observed is not set
|
||||
_, err = kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Get(context.Background(), workValidtingWebhook, metav1.GetOptions{})
|
||||
gomega.Expect(err).To(gomega.HaveOccurred())
|
||||
|
||||
updateDeploymentStatus(kubeClient, hubNamespace, hubWorkWebhookDeployment)
|
||||
|
||||
gomega.Eventually(func() error {
|
||||
if _, err := kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Get(context.Background(), workValidtingWebhook, metav1.GetOptions{}); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeNil())
|
||||
|
||||
util.AssertClusterManagerCondition(clusterManagerName, operatorClient, "Applied", "ClusterManagerApplied", metav1.ConditionTrue)
|
||||
})
|
||||
|
||||
@@ -266,6 +280,8 @@ var _ = ginkgo.Describe("ClusterManager Default Mode", func() {
|
||||
return nil
|
||||
}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeNil())
|
||||
|
||||
updateDeploymentStatus(kubeClient, hubNamespace, hubRegistrationWebhookDeployment)
|
||||
|
||||
// Check if generations are correct
|
||||
gomega.Eventually(func() error {
|
||||
actual, err := operatorClient.OperatorV1().ClusterManagers().Get(context.Background(), clusterManagerName, metav1.GetOptions{})
|
||||
@@ -286,8 +302,8 @@ var _ = ginkgo.Describe("ClusterManager Default Mode", func() {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(actual.Status.RelatedResources) != 35 {
|
||||
return fmt.Errorf("should get 35 relatedResources, actual got %v", len(actual.Status.RelatedResources))
|
||||
if len(actual.Status.RelatedResources) != 34 {
|
||||
return fmt.Errorf("should get 34 relatedResources, actual got %v", len(actual.Status.RelatedResources))
|
||||
}
|
||||
return nil
|
||||
}, eventuallyTimeout, eventuallyInterval).ShouldNot(gomega.HaveOccurred())
|
||||
@@ -333,6 +349,8 @@ var _ = ginkgo.Describe("ClusterManager Default Mode", func() {
|
||||
|
||||
return fmt.Errorf("no key equals to node-role.kubernetes.io/infra")
|
||||
}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeNil())
|
||||
updateDeploymentStatus(kubeClient, hubNamespace, hubRegistrationWebhookDeployment)
|
||||
updateDeploymentStatus(kubeClient, hubNamespace, hubWorkWebhookDeployment)
|
||||
})
|
||||
ginkgo.It("Deployment should be reconciled when manually updated", func() {
|
||||
gomega.Eventually(func() error {
|
||||
|
||||
Reference in New Issue
Block a user