Add initial e2e/integration for placeManifestWork (#180)

Signed-off-by: Jian Qiu <jqiu@redhat.com>

.github/workflows/go-presubmit.yml

@@ -12,6 +12,7 @@ env:
   GO_VERSION: '1.19'
   GO_REQUIRED_MIN_VERSION: ''
   GOPATH: '/home/runner/work/work/work/go'
+  IMAGEBUILDER_VERSION: '1.2.3'
 defaults:
   run:
     working-directory: go/src/open-cluster-management.io/work
@@ -111,7 +112,7 @@ jobs:
         with:
           go-version: ${{ env.GO_VERSION }}
       - name: install imagebuilder
-        run: go install github.com/openshift/imagebuilder/cmd/imagebuilder@v1.2.3
+        run: go install github.com/openshift/imagebuilder/cmd/imagebuilder@v${{ env.IMAGEBUILDER_VERSION }}
       - name: images
         run: make images
       - name: setup kind

Makefile

@@ -67,7 +67,7 @@ e2e-hub-kubeconfig-secret: cluster-ip
 	$(RM) ./e2e-hub-kubeconfig
 
 create-cluster-ns:
-	$(KUSTOMIZE) build deploy/hub | $(KUBECTL) --kubeconfig $(HUB_KUBECONFIG) apply -f -
+	$(KUSTOMIZE) build deploy/cluster_namespae | $(KUBECTL) --kubeconfig $(HUB_KUBECONFIG) apply -f -
 
 deploy-work-agent: ensure-kustomize create-cluster-ns hub-kubeconfig-secret
 	cp deploy/spoke/kustomization.yaml deploy/spoke/kustomization.yaml.tmp
@@ -78,36 +78,36 @@ deploy-work-agent: ensure-kustomize create-cluster-ns hub-kubeconfig-secret
 	$(KUBECTL) --kubeconfig $(SPOKE_KUBECONFIG) apply -f deploy/spoke/role_extension-apiserver.yaml 
 	$(KUBECTL) --kubeconfig $(SPOKE_KUBECONFIG) apply -f deploy/spoke/role_binding_extension-apiserver.yaml
 
-deploy-webhook: ensure-kustomize
-	cp deploy/webhook/kustomization.yaml deploy/webhook/kustomization.yaml.tmp
-	cp deploy/webhook/webhook.yaml deploy/webhook/webhook.yaml.tmp
+deploy-hub: ensure-kustomize
+	cp deploy/hub/kustomization.yaml deploy/hub/kustomization.yaml.tmp
+	cp deploy/hub/webhook.yaml deploy/hub/webhook.yaml.tmp
 	bash -x hack/inject-ca.sh
-	cd deploy/webhook && ../../$(KUSTOMIZE) edit set image quay.io/open-cluster-management/work:latest=$(IMAGE_NAME)
+	cd deploy/hub && ../../$(KUSTOMIZE) edit set image quay.io/open-cluster-management/work:latest=$(IMAGE_NAME)
 	$(KUBECTL) config use-context $(HUB_KUBECONFIG_CONTEXT) --kubeconfig $(HUB_KUBECONFIG)
-	$(KUSTOMIZE) build deploy/webhook | $(KUBECTL) --kubeconfig $(HUB_KUBECONFIG) apply -f -
-	mv deploy/webhook/kustomization.yaml.tmp deploy/webhook/kustomization.yaml
-	mv deploy/webhook/webhook.yaml.tmp deploy/webhook/webhook.yaml
+	$(KUSTOMIZE) build deploy/hub | $(KUBECTL) --kubeconfig $(HUB_KUBECONFIG) apply -f -
+	mv deploy/hub/kustomization.yaml.tmp deploy/hub/kustomization.yaml
+	mv deploy/hub/webhook.yaml.tmp deploy/hub/webhook.yaml
 
 clean-work-agent:
 	$(KUBECTL) config use-context $(SPOKE_KUBECONFIG_CONTEXT) --kubeconfig $(SPOKE_KUBECONFIG)
 	$(KUSTOMIZE) build deploy/spoke | $(KUBECTL) --kubeconfig $(SPOKE_KUBECONFIG) delete --ignore-not-found -f -
 
-clean-webhook:
+clean-hub:
 	$(KUBECTL) config use-context $(HUB_KUBECONFIG_CONTEXT) --kubeconfig $(HUB_KUBECONFIG)
-	$(KUSTOMIZE) build deploy/webhook | $(KUBECTL) --kubeconfig $(SPOKE_KUBECONFIG) delete --ignore-not-found -f -
+	$(KUSTOMIZE) build deploy/hub | $(KUBECTL) --kubeconfig $(SPOKE_KUBECONFIG) delete --ignore-not-found -f -
 
 remove-cluster-ns:
 	$(KUBECTL) config use-context $(HUB_KUBECONFIG_CONTEXT) --kubeconfig $(HUB_KUBECONFIG)
-	$(KUSTOMIZE) build deploy/hub | $(KUBECTL) --kubeconfig $(HUB_KUBECONFIG) delete --ignore-not-found -f -
+	$(KUSTOMIZE) build deploy/cluster_namespae | $(KUBECTL) --kubeconfig $(HUB_KUBECONFIG) delete --ignore-not-found -f -
 
-deploy: deploy-webhook deploy-work-agent
+deploy: deploy-hub deploy-work-agent
 
-undeploy: remove-cluster-ns clean-work-agent clean-webhook
+undeploy: remove-cluster-ns clean-work-agent clean-hub
 
 build-e2e:
 	go test -c ./test/e2e -mod=vendor
 
-test-e2e: build-e2e deploy-webhook e2e-hub-kubeconfig-secret
+test-e2e: build-e2e deploy-hub e2e-hub-kubeconfig-secret
 	./e2e.test -test.v -ginkgo.v
 
 clean-e2e:

deploy/cluster_namespace/kustomization.yaml

@@ -0,0 +1,10 @@
+
+resources:
+- ./cluster_namespace.yaml
+
+images:
+  - name: quay.io/open-cluster-management/work:latest
+    newName: quay.io/open-cluster-management/work
+    newTag: latest
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization

deploy/hub/clusterrole.yaml

@@ -0,0 +1,22 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: open-cluster-management:work-hub-controller
+rules:
+# Allow get/list/watch configmaps
+- apiGroups: [""]
+  resources: ["configmaps"]
+  verbs: ["get", "list", "watch"]
+# Allow create subjectaccessreviews
+- apiGroups: ["authorization.k8s.io"]
+  resources: ["subjectaccessreviews"]
+  verbs: ["create"]
+- apiGroups: ["work.open-cluster-management.io"]
+  resources: ["manifestworks"]
+  verbs: ["get", "list", "watch","create", "update", "delete", "deletecollection", "patch"]
+- apiGroups: ["work.open-cluster-management.io"]
+  resources: ["placemanifestworks"]
+  verbs: ["get", "list", "watch", "update"]
+- apiGroups: ["work.open-cluster-management.io"]
+  resources: ["placemanifestworks/status"]
+  verbs: ["patch", "update"]

deploy/hub/clusterrole_binding.yaml

@@ -5,8 +5,8 @@ metadata:
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
-  name: open-cluster-management:work-webhook
+  name: open-cluster-management:work-hub-controller
 subjects:
   - kind: ServiceAccount
-    name: work-webhook-sa
+    name: work-hub-sa
     namespace: open-cluster-management-hub

deploy/hub/kustomization.yaml

@@ -1,5 +1,28 @@
-resources:
-- ./cluster_namespace.yaml
 
+namespace: open-cluster-management-hub
+
+resources:
+- ./placemanifestworks.crd.yaml
+- ./manifestworks.crd.yaml
+- ./component_namespace.yaml
+- ./clusterrole_binding.yaml
+- ./clusterrole.yaml
+- ./manager_deployment.yaml
+- ./webhook_deployment.yaml
+- ./service_account.yaml
+- ./service.yaml
+- ./webhook.yaml
+
+images:
+  - name: quay.io/open-cluster-management/work:latest
+    newName: quay.io/open-cluster-management/work
+    newTag: latest
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
+
+# generate servering cert for webhook
+secretGenerator:
+  - files:
+      - cert/tls.crt
+      - cert/tls.key
+    name: work-webhook-serving-cert

deploy/hub/manager_deployment.yaml

@@ -0,0 +1,31 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: work-hub-controller
+  labels:
+    app: work-hub-controller
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: work-hub-controller
+  template:
+    metadata:
+      labels:
+        app: work-hub-controller
+    spec:
+      serviceAccountName: work-hub-sa
+      containers:
+      - name: work-hub-controller
+        image: quay.io/open-cluster-management/work:latest
+        imagePullPolicy: IfNotPresent
+        args:
+          - "/work"
+          - "manager"
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop:
+              - ALL
+          privileged: false
+          runAsNonRoot: true

deploy/hub/service_account.yaml

@@ -1,4 +1,4 @@
 apiVersion: v1
 kind: ServiceAccount
 metadata:
-  name: work-webhook-sa
+  name: work-hub-sa

deploy/hub/webhook_deployment.yaml

@@ -14,7 +14,7 @@ spec:
       labels:
         app: work-webhook
     spec:
-      serviceAccountName: work-webhook-sa
+      serviceAccountName: work-hub-sa
       containers:
       - name: work-webhook
         image: quay.io/open-cluster-management/work:latest

deploy/webhook/clusterrole.yaml

@@ -1,13 +0,0 @@
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: open-cluster-management:work-webhook
-rules:
-# Allow manifestwork admission to get/list/watch configmaps
-- apiGroups: [""]
-  resources: ["configmaps"]
-  verbs: ["get", "list", "watch"]
-# Allow manifestwork admission to create subjectaccessreviews
-- apiGroups: ["authorization.k8s.io"]
-  resources: ["subjectaccessreviews"]
-  verbs: ["create"]

deploy/webhook/kustomization.yaml

@@ -1,52 +0,0 @@
-
-# Adds namespace to all resources.
-namespace: open-cluster-management-hub
-
-# Value of this field is prepended to the
-# names of all resources, e.g. a deployment named
-# "wordpress" becomes "alices-wordpress".
-# Note that it should also match with the prefix (text before '-') of the namespace
-# field above.
-#namePrefix: multicloud-
-
-# Labels to add to all resources and selectors.
-#commonLabels:
-#  someName: someValue
-
-# Each entry in this list must resolve to an existing
-# resource definition in YAML.  These are the resource
-# files that kustomize reads, modifies and emits as a
-# YAML string, with resources separated by document
-# markers ("---").
-#
-# General rule here is anything deployed by OLM bundles should go here as well,
-# this is used in "make deploy" for developers and should mimic what OLM deploys
-# for you. CRDs are an exception to this as we don't want to have to list them all
-# here. These are deployed via a "make install" dependency.
-
-namespace: open-cluster-management-hub
-
-resources:
-- ./placemanifestworks.crd.yaml
-- ./manifestworks.crd.yaml
-- ./component_namespace.yaml
-- ./clusterrole_binding.yaml
-- ./clusterrole.yaml
-- ./deployment.yaml
-- ./service_account.yaml
-- ./service.yaml
-- ./webhook.yaml
-
-images:
-- name: quay.io/open-cluster-management/work:latest
-  newName: quay.io/open-cluster-management/work
-  newTag: latest
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-
-# generate servering cert for webhook
-secretGenerator:
-- files:
-  - cert/tls.crt
-  - cert/tls.key
-  name: work-webhook-serving-cert

hack/inject-ca.sh

@@ -6,8 +6,8 @@ sh -x $BASE_DIR/generate-cert.sh
 
 CA=`cat $BASE_DIR/cert/tls.crt |base64 -w 0`
 
-sed -i "s/CA_PLACE_HOLDER/${CA}/g" $BASE_DIR/../deploy/webhook/webhook.yaml
+sed -i "s/CA_PLACE_HOLDER/${CA}/g" $BASE_DIR/../deploy/hub/webhook.yaml
 
-rm -rf $BASE_DIR/../deploy/webhook/cert
+rm -rf $BASE_DIR/../deploy/hub/cert
 
-mv -f $BASE_DIR/cert $BASE_DIR/../deploy/webhook/cert
+mv -f $BASE_DIR/cert $BASE_DIR/../deploy/hub/cert

test/e2e/placemanifestwork_test.go

@@ -0,0 +1,33 @@
+package e2e
+
+import (
+	"context"
+	"github.com/onsi/ginkgo/v2"
+	"github.com/onsi/gomega"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	workapiv1alpha1 "open-cluster-management.io/api/work/v1alpha1"
+)
+
+var _ = ginkgo.Describe("PlaceManifestWork", func() {
+	var err error
+	ginkgo.Context("Creating a PlaceManifestWork", func() {
+		ginkgo.It("Should create PlaceManifestWork successfullt", func() {
+			work := newManifestWork("", "", []runtime.Object{newConfigmap("default", "cm1", nil, nil)}...)
+			placeManifestWork := &workapiv1alpha1.PlaceManifestWork{
+				ObjectMeta: metav1.ObjectMeta{
+					GenerateName: "placework-",
+					Namespace:    "default",
+				},
+				Spec: workapiv1alpha1.PlaceManifestWorkSpec{
+					ManifestWorkTemplate: work.Spec,
+				},
+			}
+			placeManifestWork, err = hubWorkClient.WorkV1alpha1().PlaceManifestWorks("default").Create(context.TODO(), placeManifestWork, metav1.CreateOptions{})
+			gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+			err = hubWorkClient.WorkV1alpha1().PlaceManifestWorks("default").Delete(context.TODO(), placeManifestWork.Name, metav1.DeleteOptions{})
+			gomega.Expect(err).NotTo(gomega.HaveOccurred())
+		})
+	})
+})

test/integration/placemanifestwork_test.go

@@ -0,0 +1,56 @@
+package integration
+
+import (
+	"context"
+	"github.com/onsi/ginkgo/v2"
+	"github.com/onsi/gomega"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	utilrand "k8s.io/apimachinery/pkg/util/rand"
+	workapiv1 "open-cluster-management.io/api/work/v1"
+	workapiv1alpha1 "open-cluster-management.io/api/work/v1alpha1"
+	"open-cluster-management.io/work/test/integration/util"
+)
+
+var _ = ginkgo.Describe("PlaceManifestWork", func() {
+	var namespaceName string
+
+	ginkgo.BeforeEach(func() {
+		namespaceName = utilrand.String(5)
+		ns := &corev1.Namespace{}
+		ns.Name = namespaceName
+		_, err := spokeKubeClient.CoreV1().Namespaces().Create(context.Background(), ns, metav1.CreateOptions{})
+		gomega.Expect(err).ToNot(gomega.HaveOccurred())
+	})
+
+	ginkgo.AfterEach(func() {
+		err := spokeKubeClient.CoreV1().Namespaces().Delete(context.Background(), namespaceName, metav1.DeleteOptions{})
+		gomega.Expect(err).ToNot(gomega.HaveOccurred())
+	})
+
+	// A sanity check ensuring crd is created correctly which should be refactored later
+	ginkgo.Context("Create a placeManifestWork", func() {
+		ginkgo.It("should create successfully", func() {
+			manifests := []workapiv1.Manifest{
+				util.ToManifest(util.NewConfigmap("defaut", "cm1", map[string]string{"a": "b"}, nil)),
+			}
+
+			placeManifestWork := &workapiv1alpha1.PlaceManifestWork{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "test-work",
+					Namespace: namespaceName,
+				},
+				Spec: workapiv1alpha1.PlaceManifestWorkSpec{
+					ManifestWorkTemplate: workapiv1.ManifestWorkSpec{
+						Workload: workapiv1.ManifestsTemplate{
+							Manifests: manifests,
+						},
+					},
+				},
+			}
+
+			_, err := hubWorkClient.WorkV1alpha1().PlaceManifestWorks(namespaceName).Create(context.TODO(), placeManifestWork, metav1.CreateOptions{})
+			gomega.Expect(err).ToNot(gomega.HaveOccurred())
+		})
+	})
+})

test/integration/suite_test.go

@@ -49,8 +49,8 @@ var _ = ginkgo.BeforeSuite(func() {
 	testEnv = &envtest.Environment{
 		ErrorIfCRDPathMissing: true,
 		CRDDirectoryPaths: []string{
-			filepath.Join(".", "deploy", "webhook"),
 			filepath.Join(".", "deploy", "spoke"),
+			filepath.Join(".", "deploy", "hub"),
 		},
 	}
 

test/integration/unmanaged_appliedwork_test.go

@@ -56,7 +56,7 @@ var _ = ginkgo.Describe("Unmanaged ApplieManifestWork", func() {
 		newHub = &envtest.Environment{
 			ErrorIfCRDPathMissing: true,
 			CRDDirectoryPaths: []string{
-				filepath.Join(".", "deploy", "webhook"),
+				filepath.Join(".", "deploy", "hub"),
 				filepath.Join(".", "deploy", "spoke"),
 			},
 		}