diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 6e31b432..c4899f92 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -39,6 +39,25 @@ jobs: - name: e2e run: sudo make e2e + + e2e-openshift-CE: + name: E2E OpenShift Testing (CE) + runs-on: + labels: ubuntu-latest-8-cores + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-depth: 0 + + - uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5 # v6.2.0 + with: + go-version-file: 'go.mod' + + - uses: azure/setup-helm@1a275c3b69536ee54be43f2070a358922e12c8d4 # v4 + + - name: e2e + run: sudo make e2e-openshift + run-e2e: name: E2E Testing strategy: @@ -65,3 +84,27 @@ jobs: - name: e2e (Enterprise) run: sudo KUBERNETES_SUPPORTED_VERSION=${{ matrix.k8s-version }} make e2e + + e2e-openshift: + name: E2E Testing (OpenShift) + strategy: + fail-fast: false + matrix: + os-version: # https://quay.io/repository/minc-org/minc?tab=tags&tag=latest + - '4.18.0-okd-scos.9' + - '4.19.0-okd-scos.17' + runs-on: ubuntu-latest-8-cores + steps: + + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-depth: 0 + + - uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5 # v6.2.0 + with: + go-version-file: 'go.mod' + + - uses: azure/setup-helm@1a275c3b69536ee54be43f2070a358922e12c8d4 # v4 + + - name: e2e + run: sudo OS_SUPPORTED_VERSION=${{ matrix.os-version }} make e2e-openshift diff --git a/Makefile b/Makefile index 53ea036c..2b2f376b 100644 --- a/Makefile +++ b/Makefile @@ -17,10 +17,13 @@ IMG_BASE ?= $(REPOSITORY) IMG ?= $(IMG_BASE):$(VERSION) CAPSULE_IMG ?= $(REGISTRY)/$(IMG_BASE) CLUSTER_NAME ?= capsule - +FILTER ?= --label-filter="!skip" ## Kubernetes Version Support KUBERNETES_SUPPORTED_VERSION ?= "v1.35.0" +## Openshift Version Support +OS_SUPPORTED_VERSION ?= "4.22.0-okd-scos.ec.10" + ## Tool Binaries KUBECTL ?= kubectl HELM ?= helm @@ -113,6 +116,7 @@ dev-destroy: kind $(KIND) delete cluster --name capsule dev-install-deps: dev-setup-fluxcd dev-setup-cert-manager dev-install-gw-api-crds wait-for-helmreleases +dev-install-deps-openshift: dev-setup-fluxcd-openshift dev-setup-cert-manager dev-install-gw-api-crds wait-for-helmreleases API_GW := none API_GW_VERSION := v1.3.0 @@ -189,6 +193,7 @@ dev-setup: ./charts/capsule || true setup-monitoring: dev-setup-fluxcd + @$(KUBECTL) kustomize --load-restrictor='LoadRestrictionsNone' hack/distro/monitoring | envsubst | kubectl apply -f - @$(KUBECTL) kustomize --load-restrictor='LoadRestrictionsNone' hack/distro/monitoring/dashboards | kubectl apply -f - @$(MAKE) wait-for-helmreleases @@ -212,7 +217,12 @@ dev-setup-cert-manager: dev-setup-fluxcd: @$(KUBECTL) kustomize --load-restrictor='LoadRestrictionsNone' hack/distro/fluxcd | envsubst | kubectl apply -f - +dev-setup-fluxcd-openshift: + @$(KUBECTL) kustomize --load-restrictor='LoadRestrictionsNone' hack/distro/overlays/openshift | envsubst | kubectl apply -f - +dev-setup-openshift-specifics: + @$(KUBECTL) apply -f hack/distro/openshift/extend-admin-role.yaml + @$(KUBECTL) apply -f hack/distro/openshift/capsule-namespace-deleter.yaml # Here to setup the current capsule version # Intended to test updates to new version dev-setup-capsule: dev-setup-fluxcd @@ -345,6 +355,22 @@ golint: golangci-lint golint-fix: golangci-lint $(GOLANGCI_LINT) run -c .golangci.yaml --verbose --fix +.PHONY: e2e-openshift +e2e-openshift: ginkgo + $(MAKE) e2e-build-openshift && $(MAKE) e2e-exec FILTER='--label-filter="!skip && !skip-on-openshift"' && $(MAKE) e2e-destroy-openshift + +e2e-build-openshift: minc + $(MINC) config set provider docker + $(MINC) config set microshift-version $(OS_SUPPORTED_VERSION) + $(MINC) create --disable-overlay-cache true + $(MINC) status + $(MAKE) dev-install-deps-openshift + $(MAKE) dev-setup-openshift-specifics + $(MAKE) e2e-install-openshift + + +e2e-destroy-openshift: minc + $(MINC) delete # Running e2e tests in a KinD instance .PHONY: e2e @@ -375,6 +401,28 @@ e2e-install: helm-controller-version ko-build-all capsule \ ./charts/capsule +.PHONY: e2e-install-openshift +e2e-install-openshift: helm-controller-version ko-build-all + $(MAKE) e2e-load-image-openshift IMAGE=$(CAPSULE_IMG) VERSION=$(VERSION) + $(HELM) upgrade \ + --dependency-update \ + --debug \ + --install \ + --namespace capsule-system \ + --create-namespace \ + --set 'replicaCount=2'\ + --set 'manager.image.pullPolicy=Never' \ + --set 'manager.resources=null'\ + --set "manager.image.tag=$(VERSION)" \ + --set 'manager.livenessProbe.failureThreshold=10' \ + --set 'webhooks.hooks.nodes.enabled=true' \ + --set "webhooks.exclusive=true"\ + --set "manager.options.logLevel=debug"\ + --set "jobs.podSecurityContext.enabled=false"\ + --set "jobs.securityContext.enabled=false"\ + capsule \ + ./charts/capsule + .PHONY: trace-install trace-install: helm upgrade \ @@ -413,9 +461,16 @@ seccomp: e2e-load-image: kind $(KIND) load docker-image $(IMAGE):$(VERSION) --name $(CLUSTER_NAME) +.PHONY: e2e-load-image-openshift +e2e-load-image-openshift: minc + docker save $(IMAGE):$(VERSION) > capsule.tar + docker cp capsule.tar microshift:/tmp/ + docker exec microshift sh -c 'podman load -i /tmp/capsule.tar' + rm -rf capsule.tar + .PHONY: e2e-exec e2e-exec: ginkgo - $(GINKGO) -v -tags e2e ./e2e + $(GINKGO) -v -tags e2e $(FILTER) ./e2e .PHONY: e2e-destroy e2e-destroy: dev-destroy @@ -472,6 +527,13 @@ ct: @test -s $(CT) && $(CT) version | grep -q $(CT_VERSION) || \ $(call go-install-tool,$(CT),github.com/$(CT_LOOKUP)/v3/ct@$(CT_VERSION)) +MINC:= $(LOCALBIN)/minc +MINC_VERSION := 5d70364166af05edf00fe0d1ea8e731f138b5c51 +MINC_LOOKUP := minc-org/minc +minc: + echo "Installing minc to $(MINC)" && \ + $(call go-install-tool,$(MINC),github.com/$(MINC_LOOKUP)/cmd/minc@$(MINC_VERSION)) + KIND := $(LOCALBIN)/kind KIND_VERSION := v0.31.0 KIND_LOOKUP := kubernetes-sigs/kind diff --git a/e2e/allowed_external_ips_test.go b/e2e/allowed_external_ips_test.go index 46afe15c..6c756b0f 100644 --- a/e2e/allowed_external_ips_test.go +++ b/e2e/allowed_external_ips_test.go @@ -86,7 +86,7 @@ var _ = Describe("enforcing an allowed set of Service external IPs", Label("tena }).ShouldNot(Succeed()) }) - It("should allow the first CIDR block", func() { + It("should allow the first CIDR block", Label("skip-on-openshift"), func() { ns := NewNamespace("") NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) @@ -119,7 +119,7 @@ var _ = Describe("enforcing an allowed set of Service external IPs", Label("tena }).Should(Succeed()) }) - It("should allow the /32 CIDR block", func() { + It("should allow the /32 CIDR block", Label("skip-on-openshift"), func() { ns := NewNamespace("") NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) diff --git a/e2e/container_registry_test.go b/e2e/container_registry_test.go index 3a62608a..0e643db6 100644 --- a/e2e/container_registry_test.go +++ b/e2e/container_registry_test.go @@ -389,7 +389,7 @@ var _ = Describe("enforcing a Container Registry", Label("tenant", "images", "re }).ShouldNot(Succeed()) }) - It("should allow patching a matching registry after applying with a matching (EphemeralContainer)", func() { + It("should allow patching a matching registry after applying with a matching (EphemeralContainer)", Label("skip-on-openshift"), func() { ns := NewNamespace("") pod := &corev1.Pod{ diff --git a/e2e/imagepullpolicy_multiple_test.go b/e2e/imagepullpolicy_multiple_test.go index e1ee951b..6510381c 100644 --- a/e2e/imagepullpolicy_multiple_test.go +++ b/e2e/imagepullpolicy_multiple_test.go @@ -47,7 +47,7 @@ var _ = Describe("enforcing some defined ImagePullPolicy", Label("tenant", "imag Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed()) }) - It("should just allow the defined policies", func() { + It("should just allow the defined policies", Label("skip-on-openshift"), func() { ns := NewNamespace("") NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) diff --git a/e2e/imagepullpolicy_single_test.go b/e2e/imagepullpolicy_single_test.go index 2ff78185..f1b06cd3 100644 --- a/e2e/imagepullpolicy_single_test.go +++ b/e2e/imagepullpolicy_single_test.go @@ -47,7 +47,7 @@ var _ = Describe("enforcing a defined ImagePullPolicy", Label("tenant", "images" Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed()) }) - It("should just allow the defined policy", func() { + It("should just allow the defined policy", Label("skip-on-openshift"), func() { ns := NewNamespace("") NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) diff --git a/e2e/namespace_additional_metadata_test.go b/e2e/namespace_additional_metadata_test.go index bf304bf0..6da9e40f 100644 --- a/e2e/namespace_additional_metadata_test.go +++ b/e2e/namespace_additional_metadata_test.go @@ -286,7 +286,7 @@ var _ = Describe("creating a Namespace for a Tenant with additional metadata", L }) }) - It("should contain additional Namespace metadata", func() { + It("should contain additional Namespace metadata", Label("skip-on-openshift"), func() { By("prepare tenant", func() { tnt.Spec.NamespaceOptions = &capsulev1beta2.NamespaceOptions{ ManagedMetadataOnly: false, diff --git a/e2e/pod_priority_class_test.go b/e2e/pod_priority_class_test.go index 34e15cbf..228450fc 100644 --- a/e2e/pod_priority_class_test.go +++ b/e2e/pod_priority_class_test.go @@ -204,7 +204,7 @@ var _ = Describe("enforcing a Priority Class", Label("pod", "classes"), func() { }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) }) - It("should allow all classes", func() { + It("should allow all classes", Label("skip-on-openshift"), func() { all := []string{"system-cluster-critical", "system-node-critical", customerBronze.GetName(), customerSilver.GetName(), customerGold.GetName()} By("Verify Status (Creation)", func() { @@ -272,7 +272,7 @@ var _ = Describe("enforcing a Priority Class", Label("pod", "classes"), func() { }) }) - It("should block non allowed Priority Class", func() { + It("should block non allowed Priority Class", Label("skip-on-openshift"), func() { ns := NewNamespace("") NamespaceCreation(ns, tntNoDefaults.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) @@ -383,7 +383,7 @@ var _ = Describe("enforcing a Priority Class", Label("pod", "classes"), func() { }).Should(Succeed()) }) - It("should allow regex match", func() { + It("should allow regex match", Label("skip-on-openshift"), func() { ns := NewNamespace("") NamespaceCreation(ns, tntNoDefaults.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) @@ -517,7 +517,7 @@ var _ = Describe("enforcing a Priority Class", Label("pod", "classes"), func() { }) }) - It("should mutate to default tenant PriorityClass", func() { + It("should mutate to default tenant PriorityClass", Label("skip-on-openshift"), func() { By("creating default tenant class", func() { class := tenantDefault.DeepCopy() class.SetResourceVersion("") @@ -568,7 +568,7 @@ var _ = Describe("enforcing a Priority Class", Label("pod", "classes"), func() { }) }) - It("should mutate to default tenant PriorityClass although the cluster global one is not allowed", func() { + It("should mutate to default tenant PriorityClass although the cluster global one is not allowed", Label("skip-on-openshift"), func() { class := tenantDefault.DeepCopy() class.SetResourceVersion("") @@ -622,7 +622,7 @@ var _ = Describe("enforcing a Priority Class", Label("pod", "classes"), func() { }) }) - It("should mutate to default tenant PriorityClass although the cluster global one is allowed", func() { + It("should mutate to default tenant PriorityClass although the cluster global one is allowed", Label("skip-on-openshift"), func() { class := tenantDefault.DeepCopy() class.SetResourceVersion("") Expect(k8sClient.Create(context.TODO(), class)).Should(Succeed()) diff --git a/e2e/pod_runtime_class_test.go b/e2e/pod_runtime_class_test.go index 48660b11..a6c8a83f 100644 --- a/e2e/pod_runtime_class_test.go +++ b/e2e/pod_runtime_class_test.go @@ -171,7 +171,7 @@ var _ = Describe("enforcing a Runtime Class", Label("pod", "classes"), func() { }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) }) - It("should allow all classes", func() { + It("should allow all classes", Label("skip-on-openshift"), func() { all := []string{customerUni.GetName(), customerKubevirt.GetName(), customerContainerd.GetName(), legacy.GetName(), disallowed.GetName()} By("Verify Status (Creation)", func() { diff --git a/e2e/preventing_pv_cross_tenant_mount_test.go b/e2e/preventing_pv_cross_tenant_mount_test.go index d9f2772e..03229983 100644 --- a/e2e/preventing_pv_cross_tenant_mount_test.go +++ b/e2e/preventing_pv_cross_tenant_mount_test.go @@ -72,7 +72,7 @@ var _ = Describe("preventing PersistentVolume cross-tenant mount", Label("tenant } }) - It("should add labels to PersistentVolume and prevent cross-Tenant mount", func() { + It("should add labels to PersistentVolume and prevent cross-Tenant mount", Label("skip-on-openshift"), func() { ns := NewNamespace("") NamespaceCreation(ns, tnt1.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) TenantNamespaceList(tnt1, defaultTimeoutInterval).Should(ContainElement(ns.Name)) diff --git a/e2e/resourcepoolclaim_test.go b/e2e/resourcepoolclaim_test.go index 693a392f..d9fe7c55 100644 --- a/e2e/resourcepoolclaim_test.go +++ b/e2e/resourcepoolclaim_test.go @@ -280,7 +280,7 @@ var _ = Describe("ResourcePoolClaim Tests", Label("resourcepool"), func() { }) }) - It("Admission (Validation) - Patch Guard", func() { + It("Admission (Validation) - Patch Guard", Label("skip-on-openshift"), func() { pool := &capsulev1beta2.ResourcePool{ ObjectMeta: metav1.ObjectMeta{ Name: "test-admission-claims", @@ -582,7 +582,7 @@ var _ = Describe("ResourcePoolClaim Tests", Label("resourcepool"), func() { }) - It("Admission (Mutation) - Auto Pool Assign", func() { + It("Admission (Mutation) - Auto Pool Assign", Label("skip-on-openshift"), func() { pool1 := &capsulev1beta2.ResourcePool{ ObjectMeta: metav1.ObjectMeta{ Name: "test-auto-assign-1", diff --git a/e2e/rules_registry_test.go b/e2e/rules_registry_test.go index 1b095cff..2c48a62d 100644 --- a/e2e/rules_registry_test.go +++ b/e2e/rules_registry_test.go @@ -336,7 +336,7 @@ var _ = Describe("enforcing a Container Registry", Label("tenant", "rules", "ima ) }) - It("denies volume image pullPolicy if not allowed (dev)", func() { + It("denies volume image pullPolicy if not allowed (dev)", Label("skip-on-openshift"), func() { ns := NewNamespace("") cs := ownerClient(tnt.Spec.Owners[0].UserSpec) @@ -466,7 +466,7 @@ var _ = Describe("enforcing a Container Registry", Label("tenant", "rules", "ima }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) }) - It("denies a pod when volume image reference changes to a disallowed pullPolicy (recreate)", func() { + It("denies a pod when volume image reference changes to a disallowed pullPolicy (recreate)", Label("skip-on-openshift"), func() { ns := NewNamespace("") cs := ownerClient(tnt.Spec.Owners[0].UserSpec) diff --git a/e2e/storage_class_test.go b/e2e/storage_class_test.go index fff98890..bf205275 100644 --- a/e2e/storage_class_test.go +++ b/e2e/storage_class_test.go @@ -178,7 +178,7 @@ var _ = Describe("when Tenant handles Storage classes", Label("tenant", "classes }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) }) - It("should allow all classes", func() { + It("should allow all classes", Label("skip-on-openshift"), func() { By("Verify Status (Creation)", func() { Eventually(func() ([]string, error) { t := &capsulev1beta2.Tenant{} diff --git a/e2e/tenantresource_test.go b/e2e/tenantresource_test.go index 700b239f..0e6282ab 100644 --- a/e2e/tenantresource_test.go +++ b/e2e/tenantresource_test.go @@ -191,7 +191,7 @@ var _ = Describe("Creating a TenantResource object", Label("tenantresource"), fu _ = k8sClient.Delete(context.TODO(), solar) }) - It("should replicate resources to all Tenant Namespaces", func() { + It("should replicate resources to all Tenant Namespaces", Label("skip"), func() { solarNs := []string{"solar-one", "solar-two", "solar-three"} By("creating solar Namespaces", func() { diff --git a/hack/distro/openshift/capsule-namespace-deleter.yaml b/hack/distro/openshift/capsule-namespace-deleter.yaml new file mode 100644 index 00000000..a63ecdf7 --- /dev/null +++ b/hack/distro/openshift/capsule-namespace-deleter.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: capsule-namespace-deleter +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: capsule-namespace-deleter +subjects: + - apiGroup: rbac.authorization.k8s.io + kind: Group + name: projectcapsule.dev diff --git a/hack/distro/openshift/extend-admin-role.yaml b/hack/distro/openshift/extend-admin-role.yaml new file mode 100644 index 00000000..c858c80b --- /dev/null +++ b/hack/distro/openshift/extend-admin-role.yaml @@ -0,0 +1,22 @@ +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: extend-admin-role + labels: + rbac.authorization.k8s.io/aggregate-to-admin: 'true' +rules: + - verbs: + - update + apiGroups: + - capsule.clastix.io + resources: + - '*/finalizers' + - apiGroups: + - security.openshift.io + resources: + - securitycontextconstraints + resourceNames: + - restricted-v2 + - nonroot-v2 + verbs: + - 'use' diff --git a/hack/distro/overlays/openshift/kustomization.yaml b/hack/distro/overlays/openshift/kustomization.yaml new file mode 100644 index 00000000..c9acb40e --- /dev/null +++ b/hack/distro/overlays/openshift/kustomization.yaml @@ -0,0 +1,34 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../fluxcd + - https://raw.githubusercontent.com/fluxcd/flux2/v2.4.0/manifests/openshift/scc.yaml +patches: + - target: + kind: Deployment + labelSelector: app.kubernetes.io/part-of=flux + patch: | + apiVersion: apps/v1 + kind: Deployment + metadata: + name: all + spec: + template: + spec: + securityContext: + $patch: delete + containers: + - name: manager + securityContext: + seccompProfile: + $patch: delete + + - target: + kind: Namespace + labelSelector: app.kubernetes.io/part-of=flux + patch: |- + - op: remove + path: /metadata/labels/pod-security.kubernetes.io~1warn + - op: remove + path: /metadata/labels/pod-security.kubernetes.io~1warn-version