Refactor E2E tests to use a matrix strategy (#1061)

* Refactor E2E tests to use a matrix strategy and update labels for better organization

* Enhance E2E test labels for better categorization and clarity

* Refactor E2E test labels for improved clarity and organization

.github/workflows/test-e2e.yaml

@@ -17,137 +17,26 @@ jobs:
   tests-e2e:
     runs-on: ubuntu-latest
 
-    steps:
-    - name: Checkout code
-      uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
-      with:
-        fetch-depth: 0
-        fetch-tags: true
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+        - name: update-config
+          label-filter: "update && config"
+        - name: update-version
+          label-filter: "update && upgrade"
+        - name: update-scaling
+          label-filter: "update && scaling"
+        - name: lifecycle
+          label-filter: "lifecycle || kubelet"
+        - name: persistence
+          label-filter: "persistence"
+        - name: sync
+          label-filter: "pod || app || sync || policy || status"
+        - name: features
+          label-filter: "networking || certificates || registry || addons"
 
-    - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6
-      with:
-        go-version-file: go.mod
-
-    - name: Install Ginkgo
-      run: go install github.com/onsi/ginkgo/v2/ginkgo@9ff1646a26f77a4c0d33ddba3e6368c42c0e8842 # v2.32.0
-
-    - name: Setup environment
-      run: |
-        mkdir ${{ github.workspace }}/covdata
-
-        echo "COVERAGE=true" >> $GITHUB_ENV
-        echo "GOCOVERDIR=${{ github.workspace }}/covdata" >> $GITHUB_ENV
-        echo "REPO=ttl.sh/$(uuidgen)" >> $GITHUB_ENV
-        echo "VERSION=1h" >> $GITHUB_ENV
-        echo "K3S_HOST_VERSION=${{ env.KUBERNETES_VERSION }}+k3s1" >> $GITHUB_ENV
-
-    - name: Install k3s
-      run: |
-        curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=${K3S_HOST_VERSION} INSTALL_K3S_EXEC="--write-kubeconfig-mode=777" sh -s -
-
-    - name: Read host k3s node token
-      id: k3s-token
-      run: |
-        TOKEN=$(sudo cat /var/lib/rancher/k3s/server/node-token)
-        echo "::add-mask::${TOKEN}"
-        echo "token=${TOKEN}" >> "$GITHUB_OUTPUT"
-
-    - name: Provision worker VMs and join them to the host cluster
-      uses: ./.github/actions/provision-k3s-vm-workers
-      env:
-        K3S_HOST_VERSION: ${{ env.K3S_HOST_VERSION }}
-      with:
-        worker-count: 2
-        k3s-url: https://192.168.100.1:6443
-        k3s-token: ${{ steps.k3s-token.outputs.token }}
-        k3s-version: ${{ env.K3S_HOST_VERSION }}
-
-    - name: Wait for all host nodes to be Ready
-      env:
-        KUBECONFIG: /etc/rancher/k3s/k3s.yaml
-      run: |
-        kubectl get nodes
-
-        timeout 180s bash -c '
-          until [ $(kubectl get nodes --no-headers 2>/dev/null | grep -c "Ready") -eq 3 ]; do
-            echo "Waiting for all 3 nodes to show Ready..."
-            kubectl get nodes || true
-            sleep 5
-          done
-        '
-
-    - name: Build and package and push dev images
-      env:
-        KUBECONFIG: /etc/rancher/k3s/k3s.yaml
-        REPO: ${{ env.REPO }}
-        VERSION: ${{ env.VERSION }}
-      run: |
-        make build
-        make package
-        make push
-        make install
-
-    - name: Run e2e tests
-      env:
-        KUBECONFIG: /etc/rancher/k3s/k3s.yaml
-        REPO: ${{ env.REPO }}
-        VERSION: ${{ env.VERSION }}
-      run: make E2E_LABEL_FILTER="e2e && !slow" test-e2e
-
-    - name: Convert coverage data
-      run: go tool covdata textfmt -i=${GOCOVERDIR} -o ${GOCOVERDIR}/cover.out
-
-    - name: Upload coverage reports to Codecov (controller)
-      uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7
-      with:
-        token: ${{ secrets.CODECOV_TOKEN }}
-        files: ${GOCOVERDIR}/cover.out
-        flags: controller
-
-    - name: Upload coverage reports to Codecov (e2e)
-      uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7
-      with:
-        token: ${{ secrets.CODECOV_TOKEN }}
-        files: ./cover.out
-        flags: e2e
-
-    - name: Export logs
-      if: always()
-      env:
-        KUBECONFIG: /etc/rancher/k3s/k3s.yaml
-      run: |
-        journalctl -u k3s -o cat --no-pager > /tmp/k3s.log
-        kubectl logs -n k3k-system -l "app.kubernetes.io/name=k3k" --tail=-1 > /tmp/k3k.log
-
-        for i in 1 2; do
-          IP="192.168.100.1${i}"
-          ssh -i ./id_rsa -o StrictHostKeyChecking=no ubuntu@${IP} \
-            "sudo journalctl -u k3s-agent -o cat --no-pager" > /tmp/worker-${i}.log || true
-        done
-
-    - name: Archive k3s logs
-      uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
-      if: always()
-      with:
-        name: e2e-k3s-logs
-        path: /tmp/k3s.log
-
-    - name: Archive k3k logs
-      uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
-      if: always()
-      with:
-        name: e2e-k3k-logs
-        path: /tmp/k3k.log
-
-    - name: Archive worker VM logs
-      uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
-      if: always()
-      with:
-        name: e2e-worker-logs
-        path: /tmp/worker-*.log
-
-  tests-e2e-slow:
-    runs-on: ubuntu-latest
+    name: tests-e2e (${{ matrix.name }})
 
     steps:
     - name: Checkout code
@@ -224,7 +113,7 @@ jobs:
         KUBECONFIG: /etc/rancher/k3s/k3s.yaml
         REPO: ${{ env.REPO }}
         VERSION: ${{ env.VERSION }}
-      run: make E2E_LABEL_FILTER="e2e && slow" test-e2e
+      run: make E2E_LABEL_FILTER="${{ matrix.label-filter }}" test-e2e
 
     - name: Convert coverage data
       run: go tool covdata textfmt -i=${GOCOVERDIR} -o ${GOCOVERDIR}/cover.out
@@ -261,19 +150,19 @@ jobs:
       uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
       if: always()
       with:
-        name: e2e-slow-k3s-logs
+        name: e2e-${{ matrix.name }}-k3s-logs
         path: /tmp/k3s.log
 
     - name: Archive k3k logs
       uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
       if: always()
       with:
-        name: e2e-slow-k3k-logs
+        name: e2e-${{ matrix.name }}-k3k-logs
         path: /tmp/k3k.log
 
     - name: Archive worker VM logs
       uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
       if: always()
       with:
-        name: e2e-slow-worker-logs
+        name: e2e-${{ matrix.name }}-worker-logs
         path: /tmp/worker-*.log

Makefile

@@ -24,7 +24,7 @@ PANDOC := $(shell which pandoc 2> /dev/null)
 ENVTEST ?= go run sigs.k8s.io/controller-runtime/tools/setup-envtest@$(ENVTEST_VERSION)
 ENVTEST_DIR ?= $(shell pwd)/.envtest
 
-E2E_LABEL_FILTER ?= e2e
+E2E_LABEL_FILTER ?=
 
 export KUBEBUILDER_ASSETS ?= $(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(ENVTEST_DIR) -p path)
 

tests/e2e/cluster_addons_test.go

@@ -16,13 +16,12 @@ import (
 )
 
 const (
-	addonsTestsLabel             = "addons"
 	addonsSecretName             = "k3s-addons"
 	secretMountManifestMountPath = "/var/lib/rancher/k3s/server/manifests/nginx.yaml"
 	addonManifestMountPath       = "/var/lib/rancher/k3s/server/manifests/k3s-addons/nginx.yaml"
 )
 
-var _ = When("a cluster with secretMounts configuration is used to load addons", Label("e2e"), Label(addonsTestsLabel), func() {
+var _ = When("a cluster with secretMounts configuration is used to load addons", Label(addonsTestsLabel), func() {
 	var virtualCluster *VirtualCluster
 
 	BeforeEach(func() {
@@ -91,7 +90,7 @@ var _ = When("a cluster with secretMounts configuration is used to load addons",
 	})
 })
 
-var _ = When("a cluster with addon configuration is used with addons secret in the same namespace", Label("e2e"), Label(addonsTestsLabel), func() {
+var _ = When("a cluster with addon configuration is used with addons secret in the same namespace", Label(addonsTestsLabel), func() {
 	var virtualCluster *VirtualCluster
 
 	BeforeEach(func() {
@@ -149,7 +148,7 @@ var _ = When("a cluster with addon configuration is used with addons secret in t
 	})
 })
 
-var _ = When("a cluster with addon configuration is used with addons secret in the different namespace", Label("e2e"), Label(addonsTestsLabel), func() {
+var _ = When("a cluster with addon configuration is used with addons secret in the different namespace", Label(addonsTestsLabel), func() {
 	var virtualCluster *VirtualCluster
 
 	BeforeEach(func() {

tests/e2e/cluster_app_test.go

@@ -19,7 +19,7 @@ import (
 	. "github.com/onsi/gomega"
 )
 
-var _ = Context("In a shared cluster", Label(e2eTestLabel), Ordered, func() {
+var _ = Context("In a shared cluster", Label(appTestsLabel), Ordered, func() {
 	var (
 		virtualCluster *VirtualCluster
 		translator     *translate.ToHostTranslator

tests/e2e/cluster_certs_test.go

@@ -12,7 +12,7 @@ import (
 	. "github.com/onsi/gomega"
 )
 
-var _ = When("a cluster with custom certificates is installed with individual cert secrets", Label(e2eTestLabel), Label(certificatesTestsLabel), func() {
+var _ = When("a cluster with custom certificates is installed with individual cert secrets", Label(certificatesTestsLabel), func() {
 	var virtualCluster *VirtualCluster
 
 	BeforeEach(func() {

tests/e2e/cluster_create_test.go

@@ -17,7 +17,7 @@ import (
 	. "github.com/onsi/gomega/gstruct"
 )
 
-var _ = When("creating a shared mode cluster", Label(e2eTestLabel), Label(slowTestsLabel), func() {
+var _ = When("creating a shared mode cluster", Label(lifecycleTestsLabel), Label(slowTestsLabel), func() {
 	var virtualCluster *VirtualCluster
 
 	BeforeEach(func() {
@@ -139,7 +139,7 @@ var _ = When("creating a shared mode cluster", Label(e2eTestLabel), Label(slowTe
 	})
 })
 
-var _ = When("creating an HCP mode cluster", Label(e2eTestLabel), Label(slowTestsLabel), func() {
+var _ = When("creating an HCP mode cluster", Label(lifecycleTestsLabel), Label(slowTestsLabel), func() {
 	var virtualCluster *VirtualCluster
 
 	BeforeEach(func() {

tests/e2e/cluster_kubelet_restart_test.go

@@ -18,7 +18,7 @@ import (
 	. "github.com/onsi/gomega"
 )
 
-var _ = Context("In a shared cluster", Label(e2eTestLabel), Label(slowTestsLabel), Ordered, func() {
+var _ = Context("In a shared cluster", Label(kubeletTestsLabel), Label(slowTestsLabel), Ordered, func() {
 	var (
 		virtualCluster *VirtualCluster
 		translator     *translate.ToHostTranslator

tests/e2e/cluster_network_test.go

@@ -15,7 +15,7 @@ import (
 	. "github.com/onsi/gomega"
 )
 
-var _ = When("two virtual clusters are installed", Label(e2eTestLabel), Label(networkingTestsLabel), func() {
+var _ = When("two virtual clusters are installed", Label(networkingTestsLabel), Label(slowTestsLabel), func() {
 	var (
 		cluster1 *VirtualCluster
 		cluster2 *VirtualCluster

tests/e2e/cluster_persistence_test.go

@@ -20,7 +20,7 @@ import (
 	. "github.com/onsi/gomega"
 )
 
-var _ = When("an ephemeral cluster is installed", Label(e2eTestLabel), Label(persistenceTestsLabel), func() {
+var _ = When("an ephemeral cluster is installed", Label(persistenceTestsLabel), Label(slowTestsLabel), func() {
 	var virtualCluster *VirtualCluster
 
 	BeforeEach(func() {
@@ -112,7 +112,7 @@ var _ = When("an ephemeral cluster is installed", Label(e2eTestLabel), Label(per
 	})
 })
 
-var _ = When("a dynamic cluster is installed", Label(e2eTestLabel), Label(persistenceTestsLabel), func() {
+var _ = When("a dynamic cluster is installed", Label(persistenceTestsLabel), Label(slowTestsLabel), func() {
 	var virtualCluster *VirtualCluster
 
 	BeforeEach(func() {

tests/e2e/cluster_pod_test.go

@@ -20,7 +20,7 @@ import (
 	. "github.com/onsi/gomega"
 )
 
-var _ = Context("In a shared cluster", Label(e2eTestLabel), Ordered, func() {
+var _ = Context("In a shared cluster", Label(podTestsLabel), Ordered, func() {
 	var (
 		virtualCluster *VirtualCluster
 		translator     *translate.ToHostTranslator

tests/e2e/cluster_policy_sync_storageclass_test.go

@@ -21,7 +21,7 @@ import (
 	. "github.com/onsi/gomega"
 )
 
-var _ = When("a shared mode cluster is created in a namespace with a policy", Ordered, Label(e2eTestLabel), func() {
+var _ = When("a shared mode cluster is created in a namespace with a policy", Ordered, Label(policyTestsLabel), func() {
 	var (
 		ctx            context.Context
 		virtualCluster *VirtualCluster

tests/e2e/cluster_registry_test.go

@@ -19,7 +19,7 @@ import (
 	. "github.com/onsi/gomega"
 )
 
-var _ = When("a cluster with private registry configuration is used", Label("e2e"), Label(registryTestsLabel), func() {
+var _ = When("a cluster with private registry configuration is used", Label(registryTestsLabel), Label(slowTestsLabel), func() {
 	var virtualCluster *VirtualCluster
 
 	BeforeEach(func() {

tests/e2e/cluster_status_test.go

@@ -20,7 +20,7 @@ import (
 	. "github.com/onsi/gomega"
 )
 
-var _ = When("a cluster's status is tracked", Label(e2eTestLabel), Label(statusTestsLabel), func() {
+var _ = When("a cluster's status is tracked", Label(statusTestsLabel), func() {
 	var (
 		namespace *corev1.Namespace
 		vcp       *v1beta1.VirtualClusterPolicy

tests/e2e/cluster_sync_storageclass_test.go

@@ -17,7 +17,7 @@ import (
 	. "github.com/onsi/gomega"
 )
 
-var _ = When("a shared mode cluster is created", Ordered, Label(e2eTestLabel), func() {
+var _ = When("a shared mode cluster is created", Ordered, Label(syncTestsLabel), func() {
 	var (
 		ctx            context.Context
 		virtualCluster *VirtualCluster

tests/e2e/cluster_sync_test.go

@@ -15,7 +15,7 @@ import (
 	. "github.com/onsi/gomega"
 )
 
-var _ = When("a shared mode cluster is created", Ordered, Label(e2eTestLabel), func() {
+var _ = When("a shared mode cluster is created", Ordered, Label(syncTestsLabel), func() {
 	var (
 		virtualCluster   *VirtualCluster
 		virtualConfigMap *corev1.ConfigMap

tests/e2e/cluster_update_test.go

@@ -19,7 +19,7 @@ import (
 	. "github.com/onsi/gomega"
 )
 
-var _ = When("a shared mode cluster update its envs", Label(e2eTestLabel), Label(updateTestsLabel), Label(slowTestsLabel), func() {
+var _ = When("a shared mode cluster update its envs", Label(updateTestsLabel), Label(configTestsLabel), Label(slowTestsLabel), func() {
 	var virtualCluster *VirtualCluster
 
 	ctx := context.Background()
@@ -175,7 +175,7 @@ var _ = When("a shared mode cluster update its envs", Label(e2eTestLabel), Label
 	})
 })
 
-var _ = When("a shared mode cluster update its server args", Label(e2eTestLabel), Label(updateTestsLabel), Label(slowTestsLabel), func() {
+var _ = When("a shared mode cluster update its server args", Label(updateTestsLabel), Label(configTestsLabel), Label(slowTestsLabel), func() {
 	var virtualCluster *VirtualCluster
 
 	ctx := context.Background()
@@ -241,7 +241,7 @@ var _ = When("a shared mode cluster update its server args", Label(e2eTestLabel)
 	})
 })
 
-var _ = When("a virtual mode cluster update its envs", Label(e2eTestLabel), Label(updateTestsLabel), Label(slowTestsLabel), func() {
+var _ = When("a virtual mode cluster update its envs", Label(updateTestsLabel), Label(configTestsLabel), Label(slowTestsLabel), func() {
 	var virtualCluster *VirtualCluster
 
 	ctx := context.Background()
@@ -394,7 +394,7 @@ var _ = When("a virtual mode cluster update its envs", Label(e2eTestLabel), Labe
 	})
 })
 
-var _ = When("a virtual mode cluster update its server args", Label(e2eTestLabel), Label(updateTestsLabel), Label(slowTestsLabel), func() {
+var _ = When("a virtual mode cluster update its server args", Label(updateTestsLabel), Label(configTestsLabel), Label(slowTestsLabel), func() {
 	var virtualCluster *VirtualCluster
 
 	ctx := context.Background()
@@ -463,7 +463,7 @@ var _ = When("a virtual mode cluster update its server args", Label(e2eTestLabel
 	})
 })
 
-var _ = When("a shared mode cluster update its version", Label(e2eTestLabel), Label(updateTestsLabel), Label(slowTestsLabel), func() {
+var _ = When("a shared mode cluster update its version", Label(updateTestsLabel), Label(upgradeTestsLabel), Label(slowTestsLabel), func() {
 	var (
 		virtualCluster *VirtualCluster
 		nginxPod       *corev1.Pod
@@ -552,7 +552,7 @@ var _ = When("a shared mode cluster update its version", Label(e2eTestLabel), La
 	})
 })
 
-var _ = When("a virtual mode cluster update its version", Label(e2eTestLabel), Label(updateTestsLabel), Label(slowTestsLabel), func() {
+var _ = When("a virtual mode cluster update its version", Label(updateTestsLabel), Label(upgradeTestsLabel), Label(slowTestsLabel), func() {
 	var (
 		virtualCluster *VirtualCluster
 		nginxPod       *corev1.Pod
@@ -656,7 +656,7 @@ var _ = When("a virtual mode cluster update its version", Label(e2eTestLabel), L
 	})
 })
 
-var _ = When("a shared mode cluster scales up servers", Label(e2eTestLabel), Label(updateTestsLabel), Label(slowTestsLabel), func() {
+var _ = When("a shared mode cluster scales up servers", Label(updateTestsLabel), Label(scalingTestsLabel), Label(slowTestsLabel), func() {
 	var (
 		virtualCluster *VirtualCluster
 		nginxPod       *corev1.Pod
@@ -744,7 +744,7 @@ var _ = When("a shared mode cluster scales up servers", Label(e2eTestLabel), Lab
 	})
 })
 
-var _ = When("a shared mode cluster scales down servers", Label(e2eTestLabel), Label(updateTestsLabel), Label(slowTestsLabel), func() {
+var _ = When("a shared mode cluster scales down servers", Label(updateTestsLabel), Label(scalingTestsLabel), Label(slowTestsLabel), func() {
 	var (
 		virtualCluster *VirtualCluster
 		nginxPod       *corev1.Pod
@@ -834,7 +834,7 @@ var _ = When("a shared mode cluster scales down servers", Label(e2eTestLabel), L
 	})
 })
 
-var _ = When("a virtual mode cluster scales up servers", Label(e2eTestLabel), Label(updateTestsLabel), Label(slowTestsLabel), func() {
+var _ = When("a virtual mode cluster scales up servers", Label(updateTestsLabel), Label(scalingTestsLabel), Label(slowTestsLabel), func() {
 	var (
 		virtualCluster *VirtualCluster
 		nginxPod       *corev1.Pod
@@ -922,7 +922,7 @@ var _ = When("a virtual mode cluster scales up servers", Label(e2eTestLabel), La
 	})
 })
 
-var _ = When("a virtual mode cluster scales down servers", Label(e2eTestLabel), Label(updateTestsLabel), Label(slowTestsLabel), func() {
+var _ = When("a virtual mode cluster scales down servers", Label(updateTestsLabel), Label(scalingTestsLabel), Label(slowTestsLabel), func() {
 	var (
 		virtualCluster *VirtualCluster
 		nginxPod       *corev1.Pod

tests/e2e/tests_suite_test.go

@@ -40,14 +40,30 @@ const (
 	k3sVersion    = "v1.36.2-k3s1"
 	k3sOldVersion = "v1.36.0-k3s1"
 
-	e2eTestLabel           = "e2e"
-	slowTestsLabel         = "slow"
+	// slowTestsLabel marks a top-level test container (a When/Describe node) whose
+	// end-to-end runtime in CI exceeds ~2 minutes. This is the natural cutoff in the
+	// suite: single-cluster feature checks run in ~70-115s, while nodes doing heavier
+	// lifecycle work (create/upgrade/scale/restart, HA persistence, multi-cluster
+	// networking, airgapped registry) run 140s+. It is orthogonal to the context
+	// labels below; `!slow` selects a fast local subset.
+	slowTestsLabel = "slow"
+
+	lifecycleTestsLabel    = "lifecycle"
+	kubeletTestsLabel      = "kubelet"
 	updateTestsLabel       = "update"
+	configTestsLabel       = "config"
+	upgradeTestsLabel      = "upgrade"
+	scalingTestsLabel      = "scaling"
+	podTestsLabel          = "pod"
+	appTestsLabel          = "app"
+	syncTestsLabel         = "sync"
+	policyTestsLabel       = "policy"
 	persistenceTestsLabel  = "persistence"
 	networkingTestsLabel   = "networking"
 	statusTestsLabel       = "status"
 	certificatesTestsLabel = "certificates"
 	registryTestsLabel     = "registry"
+	addonsTestsLabel       = "addons"
 
 	registryImage               = "registry:2"
 	registryCACertSecretName    = "private-registry-ca-cert"