From 1b804b848823e98c2faf67c08e67ef5324eaf685 Mon Sep 17 00:00:00 2001 From: Jian Qiu Date: Fri, 17 Feb 2023 21:50:11 +0800 Subject: [PATCH] Add initial e2e/integration for placeManifestWork (#180) Signed-off-by: Jian Qiu --- .github/workflows/go-presubmit.yml | 3 +- Makefile | 28 +++++----- .../cluster_namespace.yaml | 0 deploy/cluster_namespace/kustomization.yaml | 10 ++++ deploy/hub/clusterrole.yaml | 22 ++++++++ .../{webhook => hub}/clusterrole_binding.yaml | 4 +- .../{webhook => hub}/component_namespace.yaml | 0 deploy/hub/kustomization.yaml | 27 ++++++++- deploy/hub/manager_deployment.yaml | 31 ++++++++++ .../{webhook => hub}/manifestworks.crd.yaml | 0 .../placemanifestworks.crd.yaml | 0 deploy/{webhook => hub}/service.yaml | 0 deploy/{webhook => hub}/service_account.yaml | 2 +- deploy/{webhook => hub}/webhook.yaml | 0 .../webhook_deployment.yaml} | 2 +- deploy/webhook/clusterrole.yaml | 13 ----- deploy/webhook/kustomization.yaml | 52 ----------------- hack/inject-ca.sh | 6 +- test/e2e/placemanifestwork_test.go | 33 +++++++++++ test/integration/placemanifestwork_test.go | 56 +++++++++++++++++++ test/integration/suite_test.go | 2 +- .../integration/unmanaged_appliedwork_test.go | 2 +- 22 files changed, 202 insertions(+), 91 deletions(-) rename deploy/{hub => cluster_namespace}/cluster_namespace.yaml (100%) create mode 100644 deploy/cluster_namespace/kustomization.yaml create mode 100644 deploy/hub/clusterrole.yaml rename deploy/{webhook => hub}/clusterrole_binding.yaml (78%) rename deploy/{webhook => hub}/component_namespace.yaml (100%) create mode 100644 deploy/hub/manager_deployment.yaml rename deploy/{webhook => hub}/manifestworks.crd.yaml (100%) rename deploy/{webhook => hub}/placemanifestworks.crd.yaml (100%) rename deploy/{webhook => hub}/service.yaml (100%) rename deploy/{webhook => hub}/service_account.yaml (65%) rename deploy/{webhook => hub}/webhook.yaml (100%) rename deploy/{webhook/deployment.yaml => hub/webhook_deployment.yaml} (97%) delete mode 100644 deploy/webhook/clusterrole.yaml delete mode 100644 deploy/webhook/kustomization.yaml create mode 100644 test/e2e/placemanifestwork_test.go create mode 100644 test/integration/placemanifestwork_test.go diff --git a/.github/workflows/go-presubmit.yml b/.github/workflows/go-presubmit.yml index 82330d7c1..ee045a828 100644 --- a/.github/workflows/go-presubmit.yml +++ b/.github/workflows/go-presubmit.yml @@ -12,6 +12,7 @@ env: GO_VERSION: '1.19' GO_REQUIRED_MIN_VERSION: '' GOPATH: '/home/runner/work/work/work/go' + IMAGEBUILDER_VERSION: '1.2.3' defaults: run: working-directory: go/src/open-cluster-management.io/work @@ -111,7 +112,7 @@ jobs: with: go-version: ${{ env.GO_VERSION }} - name: install imagebuilder - run: go install github.com/openshift/imagebuilder/cmd/imagebuilder@v1.2.3 + run: go install github.com/openshift/imagebuilder/cmd/imagebuilder@v${{ env.IMAGEBUILDER_VERSION }} - name: images run: make images - name: setup kind diff --git a/Makefile b/Makefile index 977cc5f07..1e59e2c6c 100644 --- a/Makefile +++ b/Makefile @@ -67,7 +67,7 @@ e2e-hub-kubeconfig-secret: cluster-ip $(RM) ./e2e-hub-kubeconfig create-cluster-ns: - $(KUSTOMIZE) build deploy/hub | $(KUBECTL) --kubeconfig $(HUB_KUBECONFIG) apply -f - + $(KUSTOMIZE) build deploy/cluster_namespae | $(KUBECTL) --kubeconfig $(HUB_KUBECONFIG) apply -f - deploy-work-agent: ensure-kustomize create-cluster-ns hub-kubeconfig-secret cp deploy/spoke/kustomization.yaml deploy/spoke/kustomization.yaml.tmp @@ -78,36 +78,36 @@ deploy-work-agent: ensure-kustomize create-cluster-ns hub-kubeconfig-secret $(KUBECTL) --kubeconfig $(SPOKE_KUBECONFIG) apply -f deploy/spoke/role_extension-apiserver.yaml $(KUBECTL) --kubeconfig $(SPOKE_KUBECONFIG) apply -f deploy/spoke/role_binding_extension-apiserver.yaml -deploy-webhook: ensure-kustomize - cp deploy/webhook/kustomization.yaml deploy/webhook/kustomization.yaml.tmp - cp deploy/webhook/webhook.yaml deploy/webhook/webhook.yaml.tmp +deploy-hub: ensure-kustomize + cp deploy/hub/kustomization.yaml deploy/hub/kustomization.yaml.tmp + cp deploy/hub/webhook.yaml deploy/hub/webhook.yaml.tmp bash -x hack/inject-ca.sh - cd deploy/webhook && ../../$(KUSTOMIZE) edit set image quay.io/open-cluster-management/work:latest=$(IMAGE_NAME) + cd deploy/hub && ../../$(KUSTOMIZE) edit set image quay.io/open-cluster-management/work:latest=$(IMAGE_NAME) $(KUBECTL) config use-context $(HUB_KUBECONFIG_CONTEXT) --kubeconfig $(HUB_KUBECONFIG) - $(KUSTOMIZE) build deploy/webhook | $(KUBECTL) --kubeconfig $(HUB_KUBECONFIG) apply -f - - mv deploy/webhook/kustomization.yaml.tmp deploy/webhook/kustomization.yaml - mv deploy/webhook/webhook.yaml.tmp deploy/webhook/webhook.yaml + $(KUSTOMIZE) build deploy/hub | $(KUBECTL) --kubeconfig $(HUB_KUBECONFIG) apply -f - + mv deploy/hub/kustomization.yaml.tmp deploy/hub/kustomization.yaml + mv deploy/hub/webhook.yaml.tmp deploy/hub/webhook.yaml clean-work-agent: $(KUBECTL) config use-context $(SPOKE_KUBECONFIG_CONTEXT) --kubeconfig $(SPOKE_KUBECONFIG) $(KUSTOMIZE) build deploy/spoke | $(KUBECTL) --kubeconfig $(SPOKE_KUBECONFIG) delete --ignore-not-found -f - -clean-webhook: +clean-hub: $(KUBECTL) config use-context $(HUB_KUBECONFIG_CONTEXT) --kubeconfig $(HUB_KUBECONFIG) - $(KUSTOMIZE) build deploy/webhook | $(KUBECTL) --kubeconfig $(SPOKE_KUBECONFIG) delete --ignore-not-found -f - + $(KUSTOMIZE) build deploy/hub | $(KUBECTL) --kubeconfig $(SPOKE_KUBECONFIG) delete --ignore-not-found -f - remove-cluster-ns: $(KUBECTL) config use-context $(HUB_KUBECONFIG_CONTEXT) --kubeconfig $(HUB_KUBECONFIG) - $(KUSTOMIZE) build deploy/hub | $(KUBECTL) --kubeconfig $(HUB_KUBECONFIG) delete --ignore-not-found -f - + $(KUSTOMIZE) build deploy/cluster_namespae | $(KUBECTL) --kubeconfig $(HUB_KUBECONFIG) delete --ignore-not-found -f - -deploy: deploy-webhook deploy-work-agent +deploy: deploy-hub deploy-work-agent -undeploy: remove-cluster-ns clean-work-agent clean-webhook +undeploy: remove-cluster-ns clean-work-agent clean-hub build-e2e: go test -c ./test/e2e -mod=vendor -test-e2e: build-e2e deploy-webhook e2e-hub-kubeconfig-secret +test-e2e: build-e2e deploy-hub e2e-hub-kubeconfig-secret ./e2e.test -test.v -ginkgo.v clean-e2e: diff --git a/deploy/hub/cluster_namespace.yaml b/deploy/cluster_namespace/cluster_namespace.yaml similarity index 100% rename from deploy/hub/cluster_namespace.yaml rename to deploy/cluster_namespace/cluster_namespace.yaml diff --git a/deploy/cluster_namespace/kustomization.yaml b/deploy/cluster_namespace/kustomization.yaml new file mode 100644 index 000000000..7c819a444 --- /dev/null +++ b/deploy/cluster_namespace/kustomization.yaml @@ -0,0 +1,10 @@ + +resources: +- ./cluster_namespace.yaml + +images: + - name: quay.io/open-cluster-management/work:latest + newName: quay.io/open-cluster-management/work + newTag: latest +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization \ No newline at end of file diff --git a/deploy/hub/clusterrole.yaml b/deploy/hub/clusterrole.yaml new file mode 100644 index 000000000..74b9fd2f5 --- /dev/null +++ b/deploy/hub/clusterrole.yaml @@ -0,0 +1,22 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: open-cluster-management:work-hub-controller +rules: +# Allow get/list/watch configmaps +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch"] +# Allow create subjectaccessreviews +- apiGroups: ["authorization.k8s.io"] + resources: ["subjectaccessreviews"] + verbs: ["create"] +- apiGroups: ["work.open-cluster-management.io"] + resources: ["manifestworks"] + verbs: ["get", "list", "watch","create", "update", "delete", "deletecollection", "patch"] +- apiGroups: ["work.open-cluster-management.io"] + resources: ["placemanifestworks"] + verbs: ["get", "list", "watch", "update"] +- apiGroups: ["work.open-cluster-management.io"] + resources: ["placemanifestworks/status"] + verbs: ["patch", "update"] diff --git a/deploy/webhook/clusterrole_binding.yaml b/deploy/hub/clusterrole_binding.yaml similarity index 78% rename from deploy/webhook/clusterrole_binding.yaml rename to deploy/hub/clusterrole_binding.yaml index d1b6b0061..175cb9a81 100644 --- a/deploy/webhook/clusterrole_binding.yaml +++ b/deploy/hub/clusterrole_binding.yaml @@ -5,8 +5,8 @@ metadata: roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: open-cluster-management:work-webhook + name: open-cluster-management:work-hub-controller subjects: - kind: ServiceAccount - name: work-webhook-sa + name: work-hub-sa namespace: open-cluster-management-hub diff --git a/deploy/webhook/component_namespace.yaml b/deploy/hub/component_namespace.yaml similarity index 100% rename from deploy/webhook/component_namespace.yaml rename to deploy/hub/component_namespace.yaml diff --git a/deploy/hub/kustomization.yaml b/deploy/hub/kustomization.yaml index f990a094b..60ac7b5ae 100644 --- a/deploy/hub/kustomization.yaml +++ b/deploy/hub/kustomization.yaml @@ -1,5 +1,28 @@ -resources: -- ./cluster_namespace.yaml +namespace: open-cluster-management-hub + +resources: +- ./placemanifestworks.crd.yaml +- ./manifestworks.crd.yaml +- ./component_namespace.yaml +- ./clusterrole_binding.yaml +- ./clusterrole.yaml +- ./manager_deployment.yaml +- ./webhook_deployment.yaml +- ./service_account.yaml +- ./service.yaml +- ./webhook.yaml + +images: + - name: quay.io/open-cluster-management/work:latest + newName: quay.io/open-cluster-management/work + newTag: latest apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization + +# generate servering cert for webhook +secretGenerator: + - files: + - cert/tls.crt + - cert/tls.key + name: work-webhook-serving-cert \ No newline at end of file diff --git a/deploy/hub/manager_deployment.yaml b/deploy/hub/manager_deployment.yaml new file mode 100644 index 000000000..ef791a13b --- /dev/null +++ b/deploy/hub/manager_deployment.yaml @@ -0,0 +1,31 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: work-hub-controller + labels: + app: work-hub-controller +spec: + replicas: 1 + selector: + matchLabels: + app: work-hub-controller + template: + metadata: + labels: + app: work-hub-controller + spec: + serviceAccountName: work-hub-sa + containers: + - name: work-hub-controller + image: quay.io/open-cluster-management/work:latest + imagePullPolicy: IfNotPresent + args: + - "/work" + - "manager" + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + privileged: false + runAsNonRoot: true diff --git a/deploy/webhook/manifestworks.crd.yaml b/deploy/hub/manifestworks.crd.yaml similarity index 100% rename from deploy/webhook/manifestworks.crd.yaml rename to deploy/hub/manifestworks.crd.yaml diff --git a/deploy/webhook/placemanifestworks.crd.yaml b/deploy/hub/placemanifestworks.crd.yaml similarity index 100% rename from deploy/webhook/placemanifestworks.crd.yaml rename to deploy/hub/placemanifestworks.crd.yaml diff --git a/deploy/webhook/service.yaml b/deploy/hub/service.yaml similarity index 100% rename from deploy/webhook/service.yaml rename to deploy/hub/service.yaml diff --git a/deploy/webhook/service_account.yaml b/deploy/hub/service_account.yaml similarity index 65% rename from deploy/webhook/service_account.yaml rename to deploy/hub/service_account.yaml index 68dd2b12a..1e1656768 100644 --- a/deploy/webhook/service_account.yaml +++ b/deploy/hub/service_account.yaml @@ -1,4 +1,4 @@ apiVersion: v1 kind: ServiceAccount metadata: - name: work-webhook-sa + name: work-hub-sa diff --git a/deploy/webhook/webhook.yaml b/deploy/hub/webhook.yaml similarity index 100% rename from deploy/webhook/webhook.yaml rename to deploy/hub/webhook.yaml diff --git a/deploy/webhook/deployment.yaml b/deploy/hub/webhook_deployment.yaml similarity index 97% rename from deploy/webhook/deployment.yaml rename to deploy/hub/webhook_deployment.yaml index 57cb2e8e1..af52451c6 100644 --- a/deploy/webhook/deployment.yaml +++ b/deploy/hub/webhook_deployment.yaml @@ -14,7 +14,7 @@ spec: labels: app: work-webhook spec: - serviceAccountName: work-webhook-sa + serviceAccountName: work-hub-sa containers: - name: work-webhook image: quay.io/open-cluster-management/work:latest diff --git a/deploy/webhook/clusterrole.yaml b/deploy/webhook/clusterrole.yaml deleted file mode 100644 index c8cbb1566..000000000 --- a/deploy/webhook/clusterrole.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: open-cluster-management:work-webhook -rules: -# Allow manifestwork admission to get/list/watch configmaps -- apiGroups: [""] - resources: ["configmaps"] - verbs: ["get", "list", "watch"] -# Allow manifestwork admission to create subjectaccessreviews -- apiGroups: ["authorization.k8s.io"] - resources: ["subjectaccessreviews"] - verbs: ["create"] diff --git a/deploy/webhook/kustomization.yaml b/deploy/webhook/kustomization.yaml deleted file mode 100644 index 2915a36c3..000000000 --- a/deploy/webhook/kustomization.yaml +++ /dev/null @@ -1,52 +0,0 @@ - -# Adds namespace to all resources. -namespace: open-cluster-management-hub - -# Value of this field is prepended to the -# names of all resources, e.g. a deployment named -# "wordpress" becomes "alices-wordpress". -# Note that it should also match with the prefix (text before '-') of the namespace -# field above. -#namePrefix: multicloud- - -# Labels to add to all resources and selectors. -#commonLabels: -# someName: someValue - -# Each entry in this list must resolve to an existing -# resource definition in YAML. These are the resource -# files that kustomize reads, modifies and emits as a -# YAML string, with resources separated by document -# markers ("---"). -# -# General rule here is anything deployed by OLM bundles should go here as well, -# this is used in "make deploy" for developers and should mimic what OLM deploys -# for you. CRDs are an exception to this as we don't want to have to list them all -# here. These are deployed via a "make install" dependency. - -namespace: open-cluster-management-hub - -resources: -- ./placemanifestworks.crd.yaml -- ./manifestworks.crd.yaml -- ./component_namespace.yaml -- ./clusterrole_binding.yaml -- ./clusterrole.yaml -- ./deployment.yaml -- ./service_account.yaml -- ./service.yaml -- ./webhook.yaml - -images: -- name: quay.io/open-cluster-management/work:latest - newName: quay.io/open-cluster-management/work - newTag: latest -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -# generate servering cert for webhook -secretGenerator: -- files: - - cert/tls.crt - - cert/tls.key - name: work-webhook-serving-cert diff --git a/hack/inject-ca.sh b/hack/inject-ca.sh index 0f5338273..45d9a04d2 100644 --- a/hack/inject-ca.sh +++ b/hack/inject-ca.sh @@ -6,8 +6,8 @@ sh -x $BASE_DIR/generate-cert.sh CA=`cat $BASE_DIR/cert/tls.crt |base64 -w 0` -sed -i "s/CA_PLACE_HOLDER/${CA}/g" $BASE_DIR/../deploy/webhook/webhook.yaml +sed -i "s/CA_PLACE_HOLDER/${CA}/g" $BASE_DIR/../deploy/hub/webhook.yaml -rm -rf $BASE_DIR/../deploy/webhook/cert +rm -rf $BASE_DIR/../deploy/hub/cert -mv -f $BASE_DIR/cert $BASE_DIR/../deploy/webhook/cert +mv -f $BASE_DIR/cert $BASE_DIR/../deploy/hub/cert diff --git a/test/e2e/placemanifestwork_test.go b/test/e2e/placemanifestwork_test.go new file mode 100644 index 000000000..27f220d7b --- /dev/null +++ b/test/e2e/placemanifestwork_test.go @@ -0,0 +1,33 @@ +package e2e + +import ( + "context" + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + workapiv1alpha1 "open-cluster-management.io/api/work/v1alpha1" +) + +var _ = ginkgo.Describe("PlaceManifestWork", func() { + var err error + ginkgo.Context("Creating a PlaceManifestWork", func() { + ginkgo.It("Should create PlaceManifestWork successfullt", func() { + work := newManifestWork("", "", []runtime.Object{newConfigmap("default", "cm1", nil, nil)}...) + placeManifestWork := &workapiv1alpha1.PlaceManifestWork{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "placework-", + Namespace: "default", + }, + Spec: workapiv1alpha1.PlaceManifestWorkSpec{ + ManifestWorkTemplate: work.Spec, + }, + } + placeManifestWork, err = hubWorkClient.WorkV1alpha1().PlaceManifestWorks("default").Create(context.TODO(), placeManifestWork, metav1.CreateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + err = hubWorkClient.WorkV1alpha1().PlaceManifestWorks("default").Delete(context.TODO(), placeManifestWork.Name, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + }) +}) diff --git a/test/integration/placemanifestwork_test.go b/test/integration/placemanifestwork_test.go new file mode 100644 index 000000000..cf061f1fa --- /dev/null +++ b/test/integration/placemanifestwork_test.go @@ -0,0 +1,56 @@ +package integration + +import ( + "context" + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + utilrand "k8s.io/apimachinery/pkg/util/rand" + workapiv1 "open-cluster-management.io/api/work/v1" + workapiv1alpha1 "open-cluster-management.io/api/work/v1alpha1" + "open-cluster-management.io/work/test/integration/util" +) + +var _ = ginkgo.Describe("PlaceManifestWork", func() { + var namespaceName string + + ginkgo.BeforeEach(func() { + namespaceName = utilrand.String(5) + ns := &corev1.Namespace{} + ns.Name = namespaceName + _, err := spokeKubeClient.CoreV1().Namespaces().Create(context.Background(), ns, metav1.CreateOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + }) + + ginkgo.AfterEach(func() { + err := spokeKubeClient.CoreV1().Namespaces().Delete(context.Background(), namespaceName, metav1.DeleteOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + }) + + // A sanity check ensuring crd is created correctly which should be refactored later + ginkgo.Context("Create a placeManifestWork", func() { + ginkgo.It("should create successfully", func() { + manifests := []workapiv1.Manifest{ + util.ToManifest(util.NewConfigmap("defaut", "cm1", map[string]string{"a": "b"}, nil)), + } + + placeManifestWork := &workapiv1alpha1.PlaceManifestWork{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-work", + Namespace: namespaceName, + }, + Spec: workapiv1alpha1.PlaceManifestWorkSpec{ + ManifestWorkTemplate: workapiv1.ManifestWorkSpec{ + Workload: workapiv1.ManifestsTemplate{ + Manifests: manifests, + }, + }, + }, + } + + _, err := hubWorkClient.WorkV1alpha1().PlaceManifestWorks(namespaceName).Create(context.TODO(), placeManifestWork, metav1.CreateOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + }) + }) +}) diff --git a/test/integration/suite_test.go b/test/integration/suite_test.go index 35d21c9de..81d04cfb1 100644 --- a/test/integration/suite_test.go +++ b/test/integration/suite_test.go @@ -49,8 +49,8 @@ var _ = ginkgo.BeforeSuite(func() { testEnv = &envtest.Environment{ ErrorIfCRDPathMissing: true, CRDDirectoryPaths: []string{ - filepath.Join(".", "deploy", "webhook"), filepath.Join(".", "deploy", "spoke"), + filepath.Join(".", "deploy", "hub"), }, } diff --git a/test/integration/unmanaged_appliedwork_test.go b/test/integration/unmanaged_appliedwork_test.go index b720bd5a6..48786fbae 100644 --- a/test/integration/unmanaged_appliedwork_test.go +++ b/test/integration/unmanaged_appliedwork_test.go @@ -56,7 +56,7 @@ var _ = ginkgo.Describe("Unmanaged ApplieManifestWork", func() { newHub = &envtest.Environment{ ErrorIfCRDPathMissing: true, CRDDirectoryPaths: []string{ - filepath.Join(".", "deploy", "webhook"), + filepath.Join(".", "deploy", "hub"), filepath.Join(".", "deploy", "spoke"), }, }