Scale up/down tests for virtual and shared mode (#508)

* Scale up/down tests for virtual and shared mode Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com> * defer cleanup and more fixes Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com> * add labels to e2e tests and divide the workload Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com> * add labels to e2e tests and divide the workload Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com> * add validate job to e2e test Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com> * fix label filters for e2e tests Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com> * fix makefile Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com> * use constants for e2e tests labels Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com> * fix typo Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com> * fix labels Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com> --------- Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>
2026-08-18 03:46:31 +00:00 · 2025-10-15 17:01:14 +03:00
parent 7644406eeb
commit 89dc352bea
9 changed files with 580 additions and 113 deletions
@@ -0,0 +1,184 @@
+name: Tests E2E
+
+on:
+  push:
+  pull_request:
+  workflow_dispatch:
+
+permissions:
+    contents: read
+
+jobs:
+  validate:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - uses: actions/setup-go@v5
+      with:
+        go-version-file: go.mod
+
+    - name: Validate
+      run: make validate
+  tests-e2e:
+    runs-on: ubuntu-latest
+    needs: validate
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+        fetch-tags: true
+
+    - uses: actions/setup-go@v5
+      with:
+        go-version-file: go.mod
+
+    - name: Install Ginkgo
+      run: go install github.com/onsi/ginkgo/v2/ginkgo
+    
+    - name: Setup environment
+      run: |
+        mkdir ${{ github.workspace }}/covdata
+        
+        echo "COVERAGE=true" >> $GITHUB_ENV
+        echo "GOCOVERDIR=${{ github.workspace }}/covdata" >> $GITHUB_ENV
+        echo "REPO=ttl.sh/$(uuidgen)" >> $GITHUB_ENV
+        echo "VERSION=1h" >> $GITHUB_ENV
+        echo "K3S_HOST_VERSION=v1.32.1+k3s1 >> $GITHUB_ENV"
+
+    - name: Install k3s
+      run: |
+        curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=${{ env.K3S_HOST_VERSION }} INSTALL_K3S_EXEC="--write-kubeconfig-mode=777" sh -s - 
+
+    - name: Build and package and push dev images
+      env:
+        KUBECONFIG: /etc/rancher/k3s/k3s.yaml
+        REPO: ${{ env.REPO }}
+        VERSION: ${{ env.VERSION }}
+      run: |
+        make build
+        make package
+        make push
+        make install
+
+    - name: Run e2e tests
+      env:
+        KUBECONFIG: /etc/rancher/k3s/k3s.yaml
+        REPO: ${{ env.REPO }}
+        VERSION: ${{ env.VERSION }}
+      run:  make E2E_LABEL_FILTER="e2e && !slow" test-e2e
+
+    - name: Convert coverage data
+      run: go tool covdata textfmt -i=${GOCOVERDIR} -o ${GOCOVERDIR}/cover.out
+    
+    - name: Upload coverage reports to Codecov (controller)
+      uses: codecov/codecov-action@v5
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}
+        files: ${GOCOVERDIR}/cover.out
+        flags: controller
+
+    - name: Upload coverage reports to Codecov (e2e)
+      uses: codecov/codecov-action@v5
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}
+        files: ./cover.out
+        flags: e2e
+
+    - name: Archive k3s logs
+      uses: actions/upload-artifact@v4
+      if: always()
+      with:
+        name: e2e-k3s-logs
+        path: /tmp/k3s.log
+
+    - name: Archive k3k logs
+      uses: actions/upload-artifact@v4
+      if: always()
+      with:
+        name: e2e-k3k-logs
+        path: /tmp/k3k.log
+  tests-e2e-slow:
+    runs-on: ubuntu-latest
+    needs: validate
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+        fetch-tags: true
+
+    - uses: actions/setup-go@v5
+      with:
+        go-version-file: go.mod
+
+    - name: Install Ginkgo
+      run: go install github.com/onsi/ginkgo/v2/ginkgo
+    
+    - name: Setup environment
+      run: |
+        mkdir ${{ github.workspace }}/covdata
+        
+        echo "COVERAGE=true" >> $GITHUB_ENV
+        echo "GOCOVERDIR=${{ github.workspace }}/covdata" >> $GITHUB_ENV
+        echo "REPO=ttl.sh/$(uuidgen)" >> $GITHUB_ENV
+        echo "VERSION=1h" >> $GITHUB_ENV
+        echo "K3S_HOST_VERSION=v1.32.1+k3s1 >> $GITHUB_ENV"
+
+    - name: Install k3s
+      run: |
+        curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=${{ env.K3S_HOST_VERSION }} INSTALL_K3S_EXEC="--write-kubeconfig-mode=777" sh -s - 
+
+    - name: Build and package and push dev images
+      env:
+        KUBECONFIG: /etc/rancher/k3s/k3s.yaml
+        REPO: ${{ env.REPO }}
+        VERSION: ${{ env.VERSION }}
+      run: |
+        make build
+        make package
+        make push
+        make install
+
+    - name: Run e2e tests
+      env:
+        KUBECONFIG: /etc/rancher/k3s/k3s.yaml
+        REPO: ${{ env.REPO }}
+        VERSION: ${{ env.VERSION }}
+      run:  make E2E_LABEL_FILTER="e2e && slow" test-e2e
+
+    - name: Convert coverage data
+      run: go tool covdata textfmt -i=${GOCOVERDIR} -o ${GOCOVERDIR}/cover.out
+    
+    - name: Upload coverage reports to Codecov (controller)
+      uses: codecov/codecov-action@v5
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}
+        files: ${GOCOVERDIR}/cover.out
+        flags: controller
+
+    - name: Upload coverage reports to Codecov (e2e)
+      uses: codecov/codecov-action@v5
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}
+        files: ./cover.out
+        flags: e2e
+
+    - name: Archive k3s logs
+      uses: actions/upload-artifact@v4
+      if: always()
+      with:
+        name: e2e-k3s-logs
+        path: /tmp/k3s.log
+
+    - name: Archive k3k logs
+      uses: actions/upload-artifact@v4
+      if: always()
+      with:
+        name: e2e-k3k-logs
+        path: /tmp/k3k.log
@@ -62,93 +62,6 @@ jobs:
        files: ./cover.out
        flags: unit

-  tests-e2e:
-    runs-on: ubuntu-latest
-    needs: validate
-
-    steps:
-    - name: Checkout code
-      uses: actions/checkout@v4
-      with:
-        fetch-depth: 0
-        fetch-tags: true
-
-    - uses: actions/setup-go@v5
-      with:
-        go-version-file: go.mod
-
-    - name: Install Ginkgo
-      run: go install github.com/onsi/ginkgo/v2/ginkgo
-    
-    - name: Setup environment
-      run: |
-        mkdir ${{ github.workspace }}/covdata
-        
-        echo "COVERAGE=true" >> $GITHUB_ENV
-        echo "GOCOVERDIR=${{ github.workspace }}/covdata" >> $GITHUB_ENV
-        echo "REPO=ttl.sh/$(uuidgen)" >> $GITHUB_ENV
-        echo "VERSION=1h" >> $GITHUB_ENV
-        echo "K3S_HOST_VERSION=v1.32.1+k3s1 >> $GITHUB_ENV"
-
-    - name: Install k3s
-      run: |
-        curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=${{ env.K3S_HOST_VERSION }} INSTALL_K3S_EXEC="--write-kubeconfig-mode=777" sh -s - 
-
-    - name: Build and package and push dev images
-      env:
-        KUBECONFIG: /etc/rancher/k3s/k3s.yaml
-        REPO: ${{ env.REPO }}
-        VERSION: ${{ env.VERSION }}
-      run: |
-        make build
-        make package
-        make push
-        make install
-
-        # add k3kcli to $PATH
-        echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
-
-    - name: Check k3kcli
-      run: k3kcli -v
-
-    - name: Run e2e tests
-      env:
-        KUBECONFIG: /etc/rancher/k3s/k3s.yaml
-        REPO: ${{ env.REPO }}
-        VERSION: ${{ env.VERSION }}
-      run:  make test-e2e
-
-    - name: Convert coverage data
-      run: go tool covdata textfmt -i=${GOCOVERDIR} -o ${GOCOVERDIR}/cover.out
-    
-    - name: Upload coverage reports to Codecov (controller)
-      uses: codecov/codecov-action@v5
-      with:
-        token: ${{ secrets.CODECOV_TOKEN }}
-        files: ${GOCOVERDIR}/cover.out
-        flags: controller
-
-    - name: Upload coverage reports to Codecov (e2e)
-      uses: codecov/codecov-action@v5
-      with:
-        token: ${{ secrets.CODECOV_TOKEN }}
-        files: ./cover.out
-        flags: e2e
-
-    - name: Archive k3s logs
-      uses: actions/upload-artifact@v4
-      if: always()
-      with:
-        name: e2e-k3s-logs
-        path: /tmp/k3s.log
-
-    - name: Archive k3k logs
-      uses: actions/upload-artifact@v4
-      if: always()
-      with:
-        name: e2e-k3k-logs
-        path: /tmp/k3k.log
-
  tests-cli:
    runs-on: ubuntu-latest
    needs: validate
@@ -18,6 +18,9 @@ CRD_REF_DOCS := go run github.com/elastic/crd-ref-docs@$(CRD_REF_DOCS_VER)

 ENVTEST ?= go run sigs.k8s.io/controller-runtime/tools/setup-envtest@$(ENVTEST_VERSION)
 ENVTEST_DIR ?= $(shell pwd)/.envtest
+
+E2E_LABEL_FILTER ?= "e2e"
+
 export KUBEBUILDER_ASSETS ?= $(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(ENVTEST_DIR) -p path)


@@ -69,7 +72,7 @@ test-kubelet-controller:	## Run the controller tests (pkg/controller)

 .PHONY: test-e2e
 test-e2e:	## Run the e2e tests
-	$(GINKGO) $(GINKGO_FLAGS) --label-filter=e2e tests
+	$(GINKGO) $(GINKGO_FLAGS) --label-filter="$(E2E_LABEL_FILTER)" tests

 .PHONY: test-cli
 test-cli:	## Run the cli tests
@@ -13,7 +13,7 @@ import (
 	. "github.com/onsi/gomega"
 )

-var _ = When("a cluster with custom certificates is installed with individual cert secrets", Label("e2e"), func() {
+var _ = When("a cluster with custom certificates is installed with individual cert secrets", Label("e2e"), Label(certificatesTestsLabel), func() {
 	var virtualCluster *VirtualCluster

 	BeforeEach(func() {
@@ -5,7 +5,7 @@ import (
 	. "github.com/onsi/gomega"
 )

-var _ = When("two virtual clusters are installed", Label("e2e"), func() {
+var _ = When("two virtual clusters are installed", Label("e2e"), Label(networkingTestsLabel), func() {
 	var (
 		cluster1 *VirtualCluster
 		cluster2 *VirtualCluster
@@ -20,7 +20,7 @@ import (
 	. "github.com/onsi/gomega"
 )

-var _ = When("an ephemeral cluster is installed", Label("e2e"), func() {
+var _ = When("an ephemeral cluster is installed", Label("e2e"), Label(persistenceTestsLabel), func() {
 	var virtualCluster *VirtualCluster

 	BeforeEach(func() {
@@ -111,7 +111,7 @@ var _ = When("an ephemeral cluster is installed", Label("e2e"), func() {
 	})
 })

-var _ = When("a dynamic cluster is installed", Label("e2e"), func() {
+var _ = When("a dynamic cluster is installed", Label("e2e"), Label(persistenceTestsLabel), func() {
 	var virtualCluster *VirtualCluster

 	BeforeEach(func() {
@@ -18,7 +18,7 @@ import (
 	. "github.com/onsi/gomega"
 )

-var _ = When("a cluster's status is tracked", Label("e2e"), func() {
+var _ = When("a cluster's status is tracked", Label("e2e"), Label(statusTestsLabel), func() {
 	var (
 		namespace *corev1.Namespace
 		vcp       *v1beta1.VirtualClusterPolicy
@@ -18,12 +18,16 @@ import (
 	. "github.com/onsi/gomega"
 )

-var _ = When("a shared mode cluster update its envs", Label("e2e"), func() {
+var _ = When("a shared mode cluster update its envs", Label("e2e"), Label(updateTestsLabel), Label(slowTestsLabel), func() {
 	var virtualCluster *VirtualCluster
 	ctx := context.Background()
 	BeforeEach(func() {
 		namespace := NewNamespace()

+		DeferCleanup(func() {
+			DeleteNamespaces(namespace.Name)
+		})
+
 		cluster := NewCluster(namespace.Name)

 		// Add initial environment variables for server
@@ -152,12 +156,16 @@ var _ = When("a shared mode cluster update its envs", Label("e2e"), func() {
 	})
 })

-var _ = When("a shared mode cluster update its server args", Label("e2e"), func() {
+var _ = When("a shared mode cluster update its server args", Label("e2e"), Label(updateTestsLabel), Label(slowTestsLabel), func() {
 	var virtualCluster *VirtualCluster
 	ctx := context.Background()
 	BeforeEach(func() {
 		namespace := NewNamespace()

+		DeferCleanup(func() {
+			DeleteNamespaces(namespace.Name)
+		})
+
 		cluster := NewCluster(namespace.Name)

 		// Add initial args for server
@@ -207,12 +215,16 @@ var _ = When("a shared mode cluster update its server args", Label("e2e"), func(
 	})
 })

-var _ = When("a virtual mode cluster update its envs", Label("e2e"), func() {
+var _ = When("a virtual mode cluster update its envs", Label("e2e"), Label(updateTestsLabel), Label(slowTestsLabel), func() {
 	var virtualCluster *VirtualCluster
 	ctx := context.Background()
 	BeforeEach(func() {
 		namespace := NewNamespace()

+		DeferCleanup(func() {
+			DeleteNamespaces(namespace.Name)
+		})
+
 		cluster := NewCluster(namespace.Name)

 		// Add initial environment variables for server
@@ -239,7 +251,7 @@ var _ = When("a virtual mode cluster update its envs", Label("e2e"), func() {
 		}

 		cluster.Spec.Mode = v1beta1.VirtualClusterMode
-		cluster.Spec.Agents = ptr.To(int32(1))
+		cluster.Spec.Agents = ptr.To[int32](1)

 		CreateCluster(cluster)

@@ -344,7 +356,7 @@ var _ = When("a virtual mode cluster update its envs", Label("e2e"), func() {
 	})
 })

-var _ = When("a virtual mode cluster update its server args", Label("e2e"), func() {
+var _ = When("a virtual mode cluster update its server args", Label("e2e"), Label(updateTestsLabel), Label(slowTestsLabel), func() {
 	var virtualCluster *VirtualCluster
 	ctx := context.Background()
 	BeforeEach(func() {
@@ -358,7 +370,7 @@ var _ = When("a virtual mode cluster update its server args", Label("e2e"), func
 		}

 		cluster.Spec.Mode = v1beta1.VirtualClusterMode
-		cluster.Spec.Agents = ptr.To(int32(1))
+		cluster.Spec.Agents = ptr.To[int32](1)

 		CreateCluster(cluster)

@@ -402,7 +414,7 @@ var _ = When("a virtual mode cluster update its server args", Label("e2e"), func
 	})
 })

-var _ = When("a shared mode cluster update its version", Label("e2e"), func() {
+var _ = When("a shared mode cluster update its version", Label("e2e"), Label(updateTestsLabel), Label(slowTestsLabel), func() {
 	var (
 		virtualCluster *VirtualCluster
 		nginxPod       *v1.Pod
@@ -411,6 +423,10 @@ var _ = When("a shared mode cluster update its version", Label("e2e"), func() {
 		ctx := context.Background()
 		namespace := NewNamespace()

+		DeferCleanup(func() {
+			DeleteNamespaces(namespace.Name)
+		})
+
 		cluster := NewCluster(namespace.Name)

 		// Add initial version
@@ -462,8 +478,7 @@ var _ = When("a shared mode cluster update its version", Label("e2e"), func() {
 			g.Expect(len(serverPods)).To(Equal(1))

 			serverPod := serverPods[0]
-			condIndex, cond := pod.GetPodCondition(&serverPod.Status, v1.PodReady)
-			g.Expect(condIndex).NotTo(Equal(-1))
+			_, cond := pod.GetPodCondition(&serverPod.Status, v1.PodReady)
 			g.Expect(cond).NotTo(BeNil())
 			g.Expect(cond.Status).To(BeEquivalentTo(metav1.ConditionTrue))

@@ -475,9 +490,7 @@ var _ = When("a shared mode cluster update its version", Label("e2e"), func() {

 			nginxPod, err = virtualCluster.Client.CoreV1().Pods(nginxPod.Namespace).Get(ctx, nginxPod.Name, metav1.GetOptions{})
 			g.Expect(err).To(BeNil())
-
-			condIndex, cond = pod.GetPodCondition(&nginxPod.Status, v1.PodReady)
-			g.Expect(condIndex).NotTo(Equal(-1))
+			_, cond = pod.GetPodCondition(&nginxPod.Status, v1.PodReady)
 			g.Expect(cond).NotTo(BeNil())
 			g.Expect(cond.Status).To(BeEquivalentTo(metav1.ConditionTrue))
 		}).
@@ -487,7 +500,7 @@ var _ = When("a shared mode cluster update its version", Label("e2e"), func() {
 	})
 })

-var _ = When("a virtual mode cluster update its version", Label("e2e"), func() {
+var _ = When("a virtual mode cluster update its version", Label("e2e"), Label(updateTestsLabel), Label(slowTestsLabel), func() {
 	var (
 		virtualCluster *VirtualCluster
 		nginxPod       *v1.Pod
@@ -496,13 +509,17 @@ var _ = When("a virtual mode cluster update its version", Label("e2e"), func() {
 		ctx := context.Background()
 		namespace := NewNamespace()

+		DeferCleanup(func() {
+			DeleteNamespaces(namespace.Name)
+		})
+
 		cluster := NewCluster(namespace.Name)

 		// Add initial version
 		cluster.Spec.Version = "v1.31.13-k3s1"

 		cluster.Spec.Mode = v1beta1.VirtualClusterMode
-		cluster.Spec.Agents = ptr.To(int32(1))
+		cluster.Spec.Agents = ptr.To[int32](1)

 		// need to enable persistence for this
 		cluster.Spec.Persistence = v1beta1.PersistenceConfig{
@@ -550,8 +567,7 @@ var _ = When("a virtual mode cluster update its version", Label("e2e"), func() {
 			g.Expect(len(serverPods)).To(Equal(1))

 			serverPod := serverPods[0]
-			condIndex, cond := pod.GetPodCondition(&serverPod.Status, v1.PodReady)
-			g.Expect(condIndex).NotTo(Equal(-1))
+			_, cond := pod.GetPodCondition(&serverPod.Status, v1.PodReady)
 			g.Expect(cond).NotTo(BeNil())
 			g.Expect(cond.Status).To(BeEquivalentTo(metav1.ConditionTrue))

@@ -562,8 +578,7 @@ var _ = When("a virtual mode cluster update its version", Label("e2e"), func() {
 			g.Expect(len(agentPods)).To(Equal(1))

 			agentPod := agentPods[0]
-			condIndex, cond = pod.GetPodCondition(&agentPod.Status, v1.PodReady)
-			g.Expect(condIndex).NotTo(Equal(-1))
+			_, cond = pod.GetPodCondition(&agentPod.Status, v1.PodReady)
 			g.Expect(cond).NotTo(BeNil())
 			g.Expect(cond.Status).To(BeEquivalentTo(metav1.ConditionTrue))

@@ -576,8 +591,7 @@ var _ = When("a virtual mode cluster update its version", Label("e2e"), func() {
 			nginxPod, err = virtualCluster.Client.CoreV1().Pods(nginxPod.Namespace).Get(ctx, nginxPod.Name, metav1.GetOptions{})
 			g.Expect(err).To(BeNil())

-			condIndex, cond = pod.GetPodCondition(&nginxPod.Status, v1.PodReady)
-			g.Expect(condIndex).NotTo(Equal(-1))
+			_, cond = pod.GetPodCondition(&nginxPod.Status, v1.PodReady)
 			g.Expect(cond).NotTo(BeNil())
 			g.Expect(cond.Status).To(BeEquivalentTo(metav1.ConditionTrue))
 		}).
@@ -586,3 +600,349 @@ var _ = When("a virtual mode cluster update its version", Label("e2e"), func() {
 			Should(Succeed())
 	})
 })
+
+var _ = When("a shared mode cluster scales up servers", Label("e2e"), Label(updateTestsLabel), Label(slowTestsLabel), func() {
+	var (
+		virtualCluster *VirtualCluster
+		nginxPod       *v1.Pod
+	)
+	BeforeEach(func() {
+		ctx := context.Background()
+		namespace := NewNamespace()
+
+		DeferCleanup(func() {
+			DeleteNamespaces(namespace.Name)
+		})
+
+		cluster := NewCluster(namespace.Name)
+
+		// need to enable persistence for this
+		cluster.Spec.Persistence = v1beta1.PersistenceConfig{
+			Type: v1beta1.DynamicPersistenceMode,
+		}
+
+		CreateCluster(cluster)
+
+		client, restConfig := NewVirtualK8sClientAndConfig(cluster)
+
+		virtualCluster = &VirtualCluster{
+			Cluster:    cluster,
+			RestConfig: restConfig,
+			Client:     client,
+		}
+		sPods := listServerPods(ctx, virtualCluster)
+		Expect(len(sPods)).To(Equal(1))
+
+		Eventually(func(g Gomega) {
+			// since there is no way to check nodes in shared mode
+			// we can check if the endpoints are registered to N nodes
+			k8sEndpointSlices, err := virtualCluster.Client.DiscoveryV1().EndpointSlices("default").Get(ctx, "kubernetes", metav1.GetOptions{})
+			g.Expect(err).ToNot(HaveOccurred())
+			g.Expect(len(k8sEndpointSlices.Endpoints)).To(Equal(1))
+		}).
+			WithPolling(time.Second * 2).
+			WithTimeout(time.Minute * 3).
+			Should(Succeed())
+
+		nginxPod, _ = virtualCluster.NewNginxPod("")
+	})
+	It("will scale up server pods", func() {
+		var cluster v1beta1.Cluster
+		ctx := context.Background()
+
+		err := k8sClient.Get(ctx, ctrlruntimeclient.ObjectKeyFromObject(virtualCluster.Cluster), &cluster)
+		Expect(err).NotTo(HaveOccurred())
+
+		// scale cluster servers to 3 nodes
+		cluster.Spec.Servers = ptr.To[int32](3)
+
+		err = k8sClient.Update(ctx, &cluster)
+		Expect(err).NotTo(HaveOccurred())
+
+		Eventually(func(g Gomega) {
+			// server pods
+			serverPods := listServerPods(ctx, virtualCluster)
+			g.Expect(len(serverPods)).To(Equal(3))
+
+			for _, serverPod := range serverPods {
+				_, cond := pod.GetPodCondition(&serverPod.Status, v1.PodReady)
+				g.Expect(cond).NotTo(BeNil())
+				g.Expect(cond.Status).To(BeEquivalentTo(metav1.ConditionTrue))
+			}
+
+			k8sEndpointSlices, err := virtualCluster.Client.DiscoveryV1().EndpointSlices("default").Get(ctx, "kubernetes", metav1.GetOptions{})
+			g.Expect(err).ToNot(HaveOccurred())
+			g.Expect(len(k8sEndpointSlices.Endpoints)).To(Equal(3))
+
+			nginxPod, err = virtualCluster.Client.CoreV1().Pods(nginxPod.Namespace).Get(ctx, nginxPod.Name, metav1.GetOptions{})
+			g.Expect(err).To(BeNil())
+			_, cond := pod.GetPodCondition(&nginxPod.Status, v1.PodReady)
+			g.Expect(cond).NotTo(BeNil())
+			g.Expect(cond.Status).To(BeEquivalentTo(metav1.ConditionTrue))
+		}).
+			WithPolling(time.Second * 2).
+			WithTimeout(time.Minute * 3).
+			Should(Succeed())
+	})
+})
+
+var _ = When("a shared mode cluster scales down servers", Label("e2e"), Label(updateTestsLabel), Label(slowTestsLabel), func() {
+	var (
+		virtualCluster *VirtualCluster
+		nginxPod       *v1.Pod
+	)
+	BeforeEach(func() {
+		ctx := context.Background()
+		namespace := NewNamespace()
+
+		DeferCleanup(func() {
+			DeleteNamespaces(namespace.Name)
+		})
+
+		cluster := NewCluster(namespace.Name)
+
+		// start cluster with 3 servers
+		cluster.Spec.Servers = ptr.To[int32](3)
+
+		// need to enable persistence for this
+		cluster.Spec.Persistence = v1beta1.PersistenceConfig{
+			Type: v1beta1.DynamicPersistenceMode,
+		}
+
+		CreateCluster(cluster)
+
+		client, restConfig := NewVirtualK8sClientAndConfig(cluster)
+
+		virtualCluster = &VirtualCluster{
+			Cluster:    cluster,
+			RestConfig: restConfig,
+			Client:     client,
+		}
+		// no need to check servers status since createCluster() will wait until all servers are in ready state
+		sPods := listServerPods(ctx, virtualCluster)
+		Expect(len(sPods)).To(Equal(3))
+
+		Eventually(func(g Gomega) {
+			// since there is no way to check nodes in shared mode
+			// we can check if the endpoints are registered to N nodes
+			k8sEndpointSlices, err := virtualCluster.Client.DiscoveryV1().EndpointSlices("default").Get(ctx, "kubernetes", metav1.GetOptions{})
+			g.Expect(err).ToNot(HaveOccurred())
+			g.Expect(len(k8sEndpointSlices.Endpoints)).To(Equal(3))
+		}).
+			WithPolling(time.Second * 2).
+			WithTimeout(time.Minute * 3).
+			Should(Succeed())
+
+		nginxPod, _ = virtualCluster.NewNginxPod("")
+	})
+	It("will scale down server pods", func() {
+		var cluster v1beta1.Cluster
+		ctx := context.Background()
+
+		err := k8sClient.Get(ctx, ctrlruntimeclient.ObjectKeyFromObject(virtualCluster.Cluster), &cluster)
+		Expect(err).NotTo(HaveOccurred())
+
+		// scale down cluster servers to 1 node
+		cluster.Spec.Servers = ptr.To[int32](1)
+
+		err = k8sClient.Update(ctx, &cluster)
+		Expect(err).NotTo(HaveOccurred())
+
+		Eventually(func(g Gomega) {
+			// server pods
+			serverPods := listServerPods(ctx, virtualCluster)
+			g.Expect(len(serverPods)).To(Equal(1))
+
+			_, cond := pod.GetPodCondition(&serverPods[0].Status, v1.PodReady)
+			g.Expect(cond).NotTo(BeNil())
+			g.Expect(cond.Status).To(BeEquivalentTo(metav1.ConditionTrue))
+
+			k8sEndpointSlices, err := virtualCluster.Client.DiscoveryV1().EndpointSlices("default").Get(ctx, "kubernetes", metav1.GetOptions{})
+			g.Expect(err).ToNot(HaveOccurred())
+			g.Expect(len(k8sEndpointSlices.Endpoints)).To(Equal(1))
+
+			nginxPod, err = virtualCluster.Client.CoreV1().Pods(nginxPod.Namespace).Get(ctx, nginxPod.Name, metav1.GetOptions{})
+			g.Expect(err).To(BeNil())
+			_, cond = pod.GetPodCondition(&nginxPod.Status, v1.PodReady)
+			g.Expect(cond).NotTo(BeNil())
+			g.Expect(cond.Status).To(BeEquivalentTo(metav1.ConditionTrue))
+		}).
+			WithPolling(time.Second * 2).
+			WithTimeout(time.Minute * 3).
+			Should(Succeed())
+	})
+})
+
+var _ = When("a virtual mode cluster scales up servers", Label("e2e"), Label(updateTestsLabel), Label(slowTestsLabel), func() {
+	var (
+		virtualCluster *VirtualCluster
+		nginxPod       *v1.Pod
+	)
+	BeforeEach(func() {
+		ctx := context.Background()
+		namespace := NewNamespace()
+
+		DeferCleanup(func() {
+			DeleteNamespaces(namespace.Name)
+		})
+
+		cluster := NewCluster(namespace.Name)
+
+		cluster.Spec.Mode = v1beta1.VirtualClusterMode
+
+		// need to enable persistence for this
+		cluster.Spec.Persistence = v1beta1.PersistenceConfig{
+			Type: v1beta1.DynamicPersistenceMode,
+		}
+
+		CreateCluster(cluster)
+
+		client, restConfig := NewVirtualK8sClientAndConfig(cluster)
+
+		virtualCluster = &VirtualCluster{
+			Cluster:    cluster,
+			RestConfig: restConfig,
+			Client:     client,
+		}
+		sPods := listServerPods(ctx, virtualCluster)
+		Expect(len(sPods)).To(Equal(1))
+
+		Eventually(func(g Gomega) {
+			nodes, err := virtualCluster.Client.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
+			g.Expect(err).NotTo(HaveOccurred())
+			g.Expect(len(nodes.Items)).To(Equal(1))
+		}).
+			WithPolling(time.Second * 2).
+			WithTimeout(time.Minute * 5).
+			Should(Succeed())
+
+		nginxPod, _ = virtualCluster.NewNginxPod("")
+	})
+	It("will scale up server pods", func() {
+		var cluster v1beta1.Cluster
+		ctx := context.Background()
+
+		err := k8sClient.Get(ctx, ctrlruntimeclient.ObjectKeyFromObject(virtualCluster.Cluster), &cluster)
+		Expect(err).NotTo(HaveOccurred())
+
+		// scale cluster servers to 3 nodes
+		cluster.Spec.Servers = ptr.To[int32](3)
+
+		err = k8sClient.Update(ctx, &cluster)
+		Expect(err).NotTo(HaveOccurred())
+
+		Eventually(func(g Gomega) {
+			// server pods
+			serverPods := listServerPods(ctx, virtualCluster)
+			g.Expect(len(serverPods)).To(Equal(3))
+
+			for _, serverPod := range serverPods {
+				_, cond := pod.GetPodCondition(&serverPod.Status, v1.PodReady)
+				g.Expect(cond).NotTo(BeNil())
+				g.Expect(cond.Status).To(BeEquivalentTo(metav1.ConditionTrue))
+			}
+
+			nodes, err := virtualCluster.Client.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
+			g.Expect(err).NotTo(HaveOccurred())
+			g.Expect(len(nodes.Items)).To(Equal(3))
+
+			nginxPod, err = virtualCluster.Client.CoreV1().Pods(nginxPod.Namespace).Get(ctx, nginxPod.Name, metav1.GetOptions{})
+			g.Expect(err).To(BeNil())
+			_, cond := pod.GetPodCondition(&nginxPod.Status, v1.PodReady)
+			g.Expect(cond).NotTo(BeNil())
+			g.Expect(cond.Status).To(BeEquivalentTo(metav1.ConditionTrue))
+		}).
+			WithPolling(time.Second * 2).
+			WithTimeout(time.Minute * 5).
+			Should(Succeed())
+	})
+})
+
+var _ = When("a virtual mode cluster scales down servers", Label("e2e"), Label(updateTestsLabel), Label(slowTestsLabel), func() {
+	var (
+		virtualCluster *VirtualCluster
+		nginxPod       *v1.Pod
+	)
+	BeforeEach(func() {
+		ctx := context.Background()
+		namespace := NewNamespace()
+
+		DeferCleanup(func() {
+			DeleteNamespaces(namespace.Name)
+		})
+
+		cluster := NewCluster(namespace.Name)
+
+		cluster.Spec.Mode = v1beta1.VirtualClusterMode
+
+		// start cluster with 3 servers
+		cluster.Spec.Servers = ptr.To[int32](3)
+
+		// need to enable persistence for this
+		cluster.Spec.Persistence = v1beta1.PersistenceConfig{
+			Type: v1beta1.DynamicPersistenceMode,
+		}
+
+		CreateCluster(cluster)
+
+		client, restConfig := NewVirtualK8sClientAndConfig(cluster)
+
+		virtualCluster = &VirtualCluster{
+			Cluster:    cluster,
+			RestConfig: restConfig,
+			Client:     client,
+		}
+		// no need to check servers status since createCluster() will wait until all servers are in ready state
+		sPods := listServerPods(ctx, virtualCluster)
+		Expect(len(sPods)).To(Equal(3))
+
+		Eventually(func(g Gomega) {
+			nodes, err := virtualCluster.Client.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
+			g.Expect(err).NotTo(HaveOccurred())
+			g.Expect(len(nodes.Items)).To(Equal(3))
+		}).
+			WithPolling(time.Second * 2).
+			WithTimeout(time.Minute * 5).
+			Should(Succeed())
+
+		nginxPod, _ = virtualCluster.NewNginxPod("")
+	})
+	It("will scale down server pods", func() {
+		var cluster v1beta1.Cluster
+		ctx := context.Background()
+
+		err := k8sClient.Get(ctx, ctrlruntimeclient.ObjectKeyFromObject(virtualCluster.Cluster), &cluster)
+		Expect(err).NotTo(HaveOccurred())
+
+		// scale down cluster servers to 1 node
+		cluster.Spec.Servers = ptr.To[int32](1)
+
+		err = k8sClient.Update(ctx, &cluster)
+		Expect(err).NotTo(HaveOccurred())
+
+		Eventually(func(g Gomega) {
+			// server pods
+			serverPods := listServerPods(ctx, virtualCluster)
+			g.Expect(len(serverPods)).To(Equal(1))
+
+			_, cond := pod.GetPodCondition(&serverPods[0].Status, v1.PodReady)
+			g.Expect(cond).NotTo(BeNil())
+			g.Expect(cond.Status).To(BeEquivalentTo(metav1.ConditionTrue))
+
+			// we can't check for number of nodes in scale down because the nodes will be there but in a non-ready state
+			k8sEndpointSlices, err := virtualCluster.Client.DiscoveryV1().EndpointSlices("default").Get(ctx, "kubernetes", metav1.GetOptions{})
+			g.Expect(err).ToNot(HaveOccurred())
+			g.Expect(len(k8sEndpointSlices.Endpoints)).To(Equal(1))
+
+			nginxPod, err = virtualCluster.Client.CoreV1().Pods(nginxPod.Namespace).Get(ctx, nginxPod.Name, metav1.GetOptions{})
+			g.Expect(err).To(BeNil())
+
+			_, cond = pod.GetPodCondition(&nginxPod.Status, v1.PodReady)
+			g.Expect(cond).NotTo(BeNil())
+			g.Expect(cond.Status).To(BeEquivalentTo(metav1.ConditionTrue))
+		}).
+			WithPolling(time.Second * 2).
+			WithTimeout(time.Minute * 5).
+			Should(Succeed())
+	})
+})
@@ -42,6 +42,13 @@ import (
 const (
 	k3kNamespace = "k3k-system"
 	k3kName      = "k3k"
+
+	slowTestsLabel         = "slow"
+	updateTestsLabel       = "update"
+	persistenceTestsLabel  = "persistence"
+	networkingTestsLabel   = "networking"
+	statusTestsLabel       = "status"
+	certificatesTestsLabel = "certificates"
 )

 func TestTests(t *testing.T) {