feat: add e2e openshift support (#1894)

* feat: add e2e openshift support

Signed-off-by: Hristo Hristov <me@hhristov.info>

* feat: add e2e openshift support

Signed-off-by: Hristo Hristov <me@hhristov.info>

* feat: add e2e openshift support

Signed-off-by: Hristo Hristov <me@hhristov.info>

* feat: add e2e openshift support

Signed-off-by: Hristo Hristov <me@hhristov.info>

* feat: add e2e openshift support

Signed-off-by: Hristo Hristov <me@hhristov.info>

* feat: add e2e openshift support

Signed-off-by: Hristo Hristov <me@hhristov.info>

* feat: add e2e openshift support

Signed-off-by: Hristo Hristov <me@hhristov.info>

* feat: add e2e openshift support

Signed-off-by: Hristo Hristov <me@hhristov.info>

* feat: add e2e openshift support

Signed-off-by: Hristo Hristov <me@hhristov.info>

* feat: add e2e openshift support

Signed-off-by: Hristo Hristov <me@hhristov.info>

* feat: add e2e openshift support

Signed-off-by: Hristo Hristov <me@hhristov.info>

* feat: add e2e openshift support

Signed-off-by: Hristo Hristov <me@hhristov.info>

* feat: add e2e openshift support

Signed-off-by: Hristo Hristov <me@hhristov.info>

* feat: add e2e openshift support

Signed-off-by: Hristo Hristov <me@hhristov.info>

* feat: add e2e openshift support

Signed-off-by: Hristo Hristov <me@hhristov.info>

---------

Signed-off-by: Hristo Hristov <me@hhristov.info>
This commit is contained in:
Hristo Hristov
2026-04-15 11:27:39 +03:00
committed by GitHub
parent 58b25e32e8
commit 07449240e5
17 changed files with 195 additions and 22 deletions
+43
View File
@@ -39,6 +39,25 @@ jobs:
- name: e2e
run: sudo make e2e
e2e-openshift-CE:
name: E2E OpenShift Testing (CE)
runs-on:
labels: ubuntu-latest-8-cores
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
fetch-depth: 0
- uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5 # v6.2.0
with:
go-version-file: 'go.mod'
- uses: azure/setup-helm@1a275c3b69536ee54be43f2070a358922e12c8d4 # v4
- name: e2e
run: sudo make e2e-openshift
run-e2e:
name: E2E Testing
strategy:
@@ -65,3 +84,27 @@ jobs:
- name: e2e (Enterprise)
run: sudo KUBERNETES_SUPPORTED_VERSION=${{ matrix.k8s-version }} make e2e
e2e-openshift:
name: E2E Testing (OpenShift)
strategy:
fail-fast: false
matrix:
os-version: # https://quay.io/repository/minc-org/minc?tab=tags&tag=latest
- '4.18.0-okd-scos.9'
- '4.19.0-okd-scos.17'
runs-on: ubuntu-latest-8-cores
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
fetch-depth: 0
- uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5 # v6.2.0
with:
go-version-file: 'go.mod'
- uses: azure/setup-helm@1a275c3b69536ee54be43f2070a358922e12c8d4 # v4
- name: e2e
run: sudo OS_SUPPORTED_VERSION=${{ matrix.os-version }} make e2e-openshift
+64 -2
View File
@@ -17,10 +17,13 @@ IMG_BASE ?= $(REPOSITORY)
IMG ?= $(IMG_BASE):$(VERSION)
CAPSULE_IMG ?= $(REGISTRY)/$(IMG_BASE)
CLUSTER_NAME ?= capsule
FILTER ?= --label-filter="!skip"
## Kubernetes Version Support
KUBERNETES_SUPPORTED_VERSION ?= "v1.35.0"
## Openshift Version Support
OS_SUPPORTED_VERSION ?= "4.22.0-okd-scos.ec.10"
## Tool Binaries
KUBECTL ?= kubectl
HELM ?= helm
@@ -113,6 +116,7 @@ dev-destroy: kind
$(KIND) delete cluster --name capsule
dev-install-deps: dev-setup-fluxcd dev-setup-cert-manager dev-install-gw-api-crds wait-for-helmreleases
dev-install-deps-openshift: dev-setup-fluxcd-openshift dev-setup-cert-manager dev-install-gw-api-crds wait-for-helmreleases
API_GW := none
API_GW_VERSION := v1.3.0
@@ -189,6 +193,7 @@ dev-setup:
./charts/capsule || true
setup-monitoring: dev-setup-fluxcd
@$(KUBECTL) kustomize --load-restrictor='LoadRestrictionsNone' hack/distro/monitoring | envsubst | kubectl apply -f -
@$(KUBECTL) kustomize --load-restrictor='LoadRestrictionsNone' hack/distro/monitoring/dashboards | kubectl apply -f -
@$(MAKE) wait-for-helmreleases
@@ -212,7 +217,12 @@ dev-setup-cert-manager:
dev-setup-fluxcd:
@$(KUBECTL) kustomize --load-restrictor='LoadRestrictionsNone' hack/distro/fluxcd | envsubst | kubectl apply -f -
dev-setup-fluxcd-openshift:
@$(KUBECTL) kustomize --load-restrictor='LoadRestrictionsNone' hack/distro/overlays/openshift | envsubst | kubectl apply -f -
dev-setup-openshift-specifics:
@$(KUBECTL) apply -f hack/distro/openshift/extend-admin-role.yaml
@$(KUBECTL) apply -f hack/distro/openshift/capsule-namespace-deleter.yaml
# Here to setup the current capsule version
# Intended to test updates to new version
dev-setup-capsule: dev-setup-fluxcd
@@ -345,6 +355,22 @@ golint: golangci-lint
golint-fix: golangci-lint
$(GOLANGCI_LINT) run -c .golangci.yaml --verbose --fix
.PHONY: e2e-openshift
e2e-openshift: ginkgo
$(MAKE) e2e-build-openshift && $(MAKE) e2e-exec FILTER='--label-filter="!skip && !skip-on-openshift"' && $(MAKE) e2e-destroy-openshift
e2e-build-openshift: minc
$(MINC) config set provider docker
$(MINC) config set microshift-version $(OS_SUPPORTED_VERSION)
$(MINC) create --disable-overlay-cache true
$(MINC) status
$(MAKE) dev-install-deps-openshift
$(MAKE) dev-setup-openshift-specifics
$(MAKE) e2e-install-openshift
e2e-destroy-openshift: minc
$(MINC) delete
# Running e2e tests in a KinD instance
.PHONY: e2e
@@ -375,6 +401,28 @@ e2e-install: helm-controller-version ko-build-all
capsule \
./charts/capsule
.PHONY: e2e-install-openshift
e2e-install-openshift: helm-controller-version ko-build-all
$(MAKE) e2e-load-image-openshift IMAGE=$(CAPSULE_IMG) VERSION=$(VERSION)
$(HELM) upgrade \
--dependency-update \
--debug \
--install \
--namespace capsule-system \
--create-namespace \
--set 'replicaCount=2'\
--set 'manager.image.pullPolicy=Never' \
--set 'manager.resources=null'\
--set "manager.image.tag=$(VERSION)" \
--set 'manager.livenessProbe.failureThreshold=10' \
--set 'webhooks.hooks.nodes.enabled=true' \
--set "webhooks.exclusive=true"\
--set "manager.options.logLevel=debug"\
--set "jobs.podSecurityContext.enabled=false"\
--set "jobs.securityContext.enabled=false"\
capsule \
./charts/capsule
.PHONY: trace-install
trace-install:
helm upgrade \
@@ -413,9 +461,16 @@ seccomp:
e2e-load-image: kind
$(KIND) load docker-image $(IMAGE):$(VERSION) --name $(CLUSTER_NAME)
.PHONY: e2e-load-image-openshift
e2e-load-image-openshift: minc
docker save $(IMAGE):$(VERSION) > capsule.tar
docker cp capsule.tar microshift:/tmp/
docker exec microshift sh -c 'podman load -i /tmp/capsule.tar'
rm -rf capsule.tar
.PHONY: e2e-exec
e2e-exec: ginkgo
$(GINKGO) -v -tags e2e ./e2e
$(GINKGO) -v -tags e2e $(FILTER) ./e2e
.PHONY: e2e-destroy
e2e-destroy: dev-destroy
@@ -472,6 +527,13 @@ ct:
@test -s $(CT) && $(CT) version | grep -q $(CT_VERSION) || \
$(call go-install-tool,$(CT),github.com/$(CT_LOOKUP)/v3/ct@$(CT_VERSION))
MINC:= $(LOCALBIN)/minc
MINC_VERSION := 5d70364166af05edf00fe0d1ea8e731f138b5c51
MINC_LOOKUP := minc-org/minc
minc:
echo "Installing minc to $(MINC)" && \
$(call go-install-tool,$(MINC),github.com/$(MINC_LOOKUP)/cmd/minc@$(MINC_VERSION))
KIND := $(LOCALBIN)/kind
KIND_VERSION := v0.31.0
KIND_LOOKUP := kubernetes-sigs/kind
+2 -2
View File
@@ -86,7 +86,7 @@ var _ = Describe("enforcing an allowed set of Service external IPs", Label("tena
}).ShouldNot(Succeed())
})
It("should allow the first CIDR block", func() {
It("should allow the first CIDR block", Label("skip-on-openshift"), func() {
ns := NewNamespace("")
NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed())
@@ -119,7 +119,7 @@ var _ = Describe("enforcing an allowed set of Service external IPs", Label("tena
}).Should(Succeed())
})
It("should allow the /32 CIDR block", func() {
It("should allow the /32 CIDR block", Label("skip-on-openshift"), func() {
ns := NewNamespace("")
NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed())
+1 -1
View File
@@ -389,7 +389,7 @@ var _ = Describe("enforcing a Container Registry", Label("tenant", "images", "re
}).ShouldNot(Succeed())
})
It("should allow patching a matching registry after applying with a matching (EphemeralContainer)", func() {
It("should allow patching a matching registry after applying with a matching (EphemeralContainer)", Label("skip-on-openshift"), func() {
ns := NewNamespace("")
pod := &corev1.Pod{
+1 -1
View File
@@ -47,7 +47,7 @@ var _ = Describe("enforcing some defined ImagePullPolicy", Label("tenant", "imag
Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed())
})
It("should just allow the defined policies", func() {
It("should just allow the defined policies", Label("skip-on-openshift"), func() {
ns := NewNamespace("")
NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed())
+1 -1
View File
@@ -47,7 +47,7 @@ var _ = Describe("enforcing a defined ImagePullPolicy", Label("tenant", "images"
Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed())
})
It("should just allow the defined policy", func() {
It("should just allow the defined policy", Label("skip-on-openshift"), func() {
ns := NewNamespace("")
NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed())
+1 -1
View File
@@ -286,7 +286,7 @@ var _ = Describe("creating a Namespace for a Tenant with additional metadata", L
})
})
It("should contain additional Namespace metadata", func() {
It("should contain additional Namespace metadata", Label("skip-on-openshift"), func() {
By("prepare tenant", func() {
tnt.Spec.NamespaceOptions = &capsulev1beta2.NamespaceOptions{
ManagedMetadataOnly: false,
+6 -6
View File
@@ -204,7 +204,7 @@ var _ = Describe("enforcing a Priority Class", Label("pod", "classes"), func() {
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
})
It("should allow all classes", func() {
It("should allow all classes", Label("skip-on-openshift"), func() {
all := []string{"system-cluster-critical", "system-node-critical", customerBronze.GetName(), customerSilver.GetName(), customerGold.GetName()}
By("Verify Status (Creation)", func() {
@@ -272,7 +272,7 @@ var _ = Describe("enforcing a Priority Class", Label("pod", "classes"), func() {
})
})
It("should block non allowed Priority Class", func() {
It("should block non allowed Priority Class", Label("skip-on-openshift"), func() {
ns := NewNamespace("")
NamespaceCreation(ns, tntNoDefaults.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed())
@@ -383,7 +383,7 @@ var _ = Describe("enforcing a Priority Class", Label("pod", "classes"), func() {
}).Should(Succeed())
})
It("should allow regex match", func() {
It("should allow regex match", Label("skip-on-openshift"), func() {
ns := NewNamespace("")
NamespaceCreation(ns, tntNoDefaults.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed())
@@ -517,7 +517,7 @@ var _ = Describe("enforcing a Priority Class", Label("pod", "classes"), func() {
})
})
It("should mutate to default tenant PriorityClass", func() {
It("should mutate to default tenant PriorityClass", Label("skip-on-openshift"), func() {
By("creating default tenant class", func() {
class := tenantDefault.DeepCopy()
class.SetResourceVersion("")
@@ -568,7 +568,7 @@ var _ = Describe("enforcing a Priority Class", Label("pod", "classes"), func() {
})
})
It("should mutate to default tenant PriorityClass although the cluster global one is not allowed", func() {
It("should mutate to default tenant PriorityClass although the cluster global one is not allowed", Label("skip-on-openshift"), func() {
class := tenantDefault.DeepCopy()
class.SetResourceVersion("")
@@ -622,7 +622,7 @@ var _ = Describe("enforcing a Priority Class", Label("pod", "classes"), func() {
})
})
It("should mutate to default tenant PriorityClass although the cluster global one is allowed", func() {
It("should mutate to default tenant PriorityClass although the cluster global one is allowed", Label("skip-on-openshift"), func() {
class := tenantDefault.DeepCopy()
class.SetResourceVersion("")
Expect(k8sClient.Create(context.TODO(), class)).Should(Succeed())
+1 -1
View File
@@ -171,7 +171,7 @@ var _ = Describe("enforcing a Runtime Class", Label("pod", "classes"), func() {
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
})
It("should allow all classes", func() {
It("should allow all classes", Label("skip-on-openshift"), func() {
all := []string{customerUni.GetName(), customerKubevirt.GetName(), customerContainerd.GetName(), legacy.GetName(), disallowed.GetName()}
By("Verify Status (Creation)", func() {
+1 -1
View File
@@ -72,7 +72,7 @@ var _ = Describe("preventing PersistentVolume cross-tenant mount", Label("tenant
}
})
It("should add labels to PersistentVolume and prevent cross-Tenant mount", func() {
It("should add labels to PersistentVolume and prevent cross-Tenant mount", Label("skip-on-openshift"), func() {
ns := NewNamespace("")
NamespaceCreation(ns, tnt1.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed())
TenantNamespaceList(tnt1, defaultTimeoutInterval).Should(ContainElement(ns.Name))
+2 -2
View File
@@ -280,7 +280,7 @@ var _ = Describe("ResourcePoolClaim Tests", Label("resourcepool"), func() {
})
})
It("Admission (Validation) - Patch Guard", func() {
It("Admission (Validation) - Patch Guard", Label("skip-on-openshift"), func() {
pool := &capsulev1beta2.ResourcePool{
ObjectMeta: metav1.ObjectMeta{
Name: "test-admission-claims",
@@ -582,7 +582,7 @@ var _ = Describe("ResourcePoolClaim Tests", Label("resourcepool"), func() {
})
It("Admission (Mutation) - Auto Pool Assign", func() {
It("Admission (Mutation) - Auto Pool Assign", Label("skip-on-openshift"), func() {
pool1 := &capsulev1beta2.ResourcePool{
ObjectMeta: metav1.ObjectMeta{
Name: "test-auto-assign-1",
+2 -2
View File
@@ -336,7 +336,7 @@ var _ = Describe("enforcing a Container Registry", Label("tenant", "rules", "ima
)
})
It("denies volume image pullPolicy if not allowed (dev)", func() {
It("denies volume image pullPolicy if not allowed (dev)", Label("skip-on-openshift"), func() {
ns := NewNamespace("")
cs := ownerClient(tnt.Spec.Owners[0].UserSpec)
@@ -466,7 +466,7 @@ var _ = Describe("enforcing a Container Registry", Label("tenant", "rules", "ima
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
})
It("denies a pod when volume image reference changes to a disallowed pullPolicy (recreate)", func() {
It("denies a pod when volume image reference changes to a disallowed pullPolicy (recreate)", Label("skip-on-openshift"), func() {
ns := NewNamespace("")
cs := ownerClient(tnt.Spec.Owners[0].UserSpec)
+1 -1
View File
@@ -178,7 +178,7 @@ var _ = Describe("when Tenant handles Storage classes", Label("tenant", "classes
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
})
It("should allow all classes", func() {
It("should allow all classes", Label("skip-on-openshift"), func() {
By("Verify Status (Creation)", func() {
Eventually(func() ([]string, error) {
t := &capsulev1beta2.Tenant{}
+1 -1
View File
@@ -191,7 +191,7 @@ var _ = Describe("Creating a TenantResource object", Label("tenantresource"), fu
_ = k8sClient.Delete(context.TODO(), solar)
})
It("should replicate resources to all Tenant Namespaces", func() {
It("should replicate resources to all Tenant Namespaces", Label("skip"), func() {
solarNs := []string{"solar-one", "solar-two", "solar-three"}
By("creating solar Namespaces", func() {
@@ -0,0 +1,12 @@
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: capsule-namespace-deleter
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: capsule-namespace-deleter
subjects:
- apiGroup: rbac.authorization.k8s.io
kind: Group
name: projectcapsule.dev
@@ -0,0 +1,22 @@
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: extend-admin-role
labels:
rbac.authorization.k8s.io/aggregate-to-admin: 'true'
rules:
- verbs:
- update
apiGroups:
- capsule.clastix.io
resources:
- '*/finalizers'
- apiGroups:
- security.openshift.io
resources:
- securitycontextconstraints
resourceNames:
- restricted-v2
- nonroot-v2
verbs:
- 'use'
@@ -0,0 +1,34 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- ../../fluxcd
- https://raw.githubusercontent.com/fluxcd/flux2/v2.4.0/manifests/openshift/scc.yaml
patches:
- target:
kind: Deployment
labelSelector: app.kubernetes.io/part-of=flux
patch: |
apiVersion: apps/v1
kind: Deployment
metadata:
name: all
spec:
template:
spec:
securityContext:
$patch: delete
containers:
- name: manager
securityContext:
seccompProfile:
$patch: delete
- target:
kind: Namespace
labelSelector: app.kubernetes.io/part-of=flux
patch: |-
- op: remove
path: /metadata/labels/pod-security.kubernetes.io~1warn
- op: remove
path: /metadata/labels/pod-security.kubernetes.io~1warn-version