From 89dc352bea24621ec5c5c4424e65af1c8f3c967b Mon Sep 17 00:00:00 2001 From: Hussein Galal Date: Wed, 15 Oct 2025 17:01:14 +0300 Subject: [PATCH] Scale up/down tests for virtual and shared mode (#508) * Scale up/down tests for virtual and shared mode Signed-off-by: galal-hussein * defer cleanup and more fixes Signed-off-by: galal-hussein * add labels to e2e tests and divide the workload Signed-off-by: galal-hussein * add labels to e2e tests and divide the workload Signed-off-by: galal-hussein * add validate job to e2e test Signed-off-by: galal-hussein * fix label filters for e2e tests Signed-off-by: galal-hussein * fix makefile Signed-off-by: galal-hussein * use constants for e2e tests labels Signed-off-by: galal-hussein * fix typo Signed-off-by: galal-hussein * fix labels Signed-off-by: galal-hussein --------- Signed-off-by: galal-hussein --- .github/workflows/test-e2e.yaml | 184 ++++++++++++++ .github/workflows/test.yaml | 87 ------- Makefile | 5 +- tests/cluster_certs_test.go | 2 +- tests/cluster_network_test.go | 2 +- tests/cluster_persistence_test.go | 4 +- tests/cluster_status_test.go | 2 +- tests/cluster_update_test.go | 400 ++++++++++++++++++++++++++++-- tests/tests_suite_test.go | 7 + 9 files changed, 580 insertions(+), 113 deletions(-) create mode 100644 .github/workflows/test-e2e.yaml diff --git a/.github/workflows/test-e2e.yaml b/.github/workflows/test-e2e.yaml new file mode 100644 index 00000000..f5f46e0b --- /dev/null +++ b/.github/workflows/test-e2e.yaml @@ -0,0 +1,184 @@ +name: Tests E2E + +on: + push: + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + validate: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + + - name: Validate + run: make validate + tests-e2e: + runs-on: ubuntu-latest + needs: validate + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + + - name: Install Ginkgo + run: go install github.com/onsi/ginkgo/v2/ginkgo + + - name: Setup environment + run: | + mkdir ${{ github.workspace }}/covdata + + echo "COVERAGE=true" >> $GITHUB_ENV + echo "GOCOVERDIR=${{ github.workspace }}/covdata" >> $GITHUB_ENV + echo "REPO=ttl.sh/$(uuidgen)" >> $GITHUB_ENV + echo "VERSION=1h" >> $GITHUB_ENV + echo "K3S_HOST_VERSION=v1.32.1+k3s1 >> $GITHUB_ENV" + + - name: Install k3s + run: | + curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=${{ env.K3S_HOST_VERSION }} INSTALL_K3S_EXEC="--write-kubeconfig-mode=777" sh -s - + + - name: Build and package and push dev images + env: + KUBECONFIG: /etc/rancher/k3s/k3s.yaml + REPO: ${{ env.REPO }} + VERSION: ${{ env.VERSION }} + run: | + make build + make package + make push + make install + + - name: Run e2e tests + env: + KUBECONFIG: /etc/rancher/k3s/k3s.yaml + REPO: ${{ env.REPO }} + VERSION: ${{ env.VERSION }} + run: make E2E_LABEL_FILTER="e2e && !slow" test-e2e + + - name: Convert coverage data + run: go tool covdata textfmt -i=${GOCOVERDIR} -o ${GOCOVERDIR}/cover.out + + - name: Upload coverage reports to Codecov (controller) + uses: codecov/codecov-action@v5 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: ${GOCOVERDIR}/cover.out + flags: controller + + - name: Upload coverage reports to Codecov (e2e) + uses: codecov/codecov-action@v5 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: ./cover.out + flags: e2e + + - name: Archive k3s logs + uses: actions/upload-artifact@v4 + if: always() + with: + name: e2e-k3s-logs + path: /tmp/k3s.log + + - name: Archive k3k logs + uses: actions/upload-artifact@v4 + if: always() + with: + name: e2e-k3k-logs + path: /tmp/k3k.log + tests-e2e-slow: + runs-on: ubuntu-latest + needs: validate + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + + - name: Install Ginkgo + run: go install github.com/onsi/ginkgo/v2/ginkgo + + - name: Setup environment + run: | + mkdir ${{ github.workspace }}/covdata + + echo "COVERAGE=true" >> $GITHUB_ENV + echo "GOCOVERDIR=${{ github.workspace }}/covdata" >> $GITHUB_ENV + echo "REPO=ttl.sh/$(uuidgen)" >> $GITHUB_ENV + echo "VERSION=1h" >> $GITHUB_ENV + echo "K3S_HOST_VERSION=v1.32.1+k3s1 >> $GITHUB_ENV" + + - name: Install k3s + run: | + curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=${{ env.K3S_HOST_VERSION }} INSTALL_K3S_EXEC="--write-kubeconfig-mode=777" sh -s - + + - name: Build and package and push dev images + env: + KUBECONFIG: /etc/rancher/k3s/k3s.yaml + REPO: ${{ env.REPO }} + VERSION: ${{ env.VERSION }} + run: | + make build + make package + make push + make install + + - name: Run e2e tests + env: + KUBECONFIG: /etc/rancher/k3s/k3s.yaml + REPO: ${{ env.REPO }} + VERSION: ${{ env.VERSION }} + run: make E2E_LABEL_FILTER="e2e && slow" test-e2e + + - name: Convert coverage data + run: go tool covdata textfmt -i=${GOCOVERDIR} -o ${GOCOVERDIR}/cover.out + + - name: Upload coverage reports to Codecov (controller) + uses: codecov/codecov-action@v5 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: ${GOCOVERDIR}/cover.out + flags: controller + + - name: Upload coverage reports to Codecov (e2e) + uses: codecov/codecov-action@v5 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: ./cover.out + flags: e2e + + - name: Archive k3s logs + uses: actions/upload-artifact@v4 + if: always() + with: + name: e2e-k3s-logs + path: /tmp/k3s.log + + - name: Archive k3k logs + uses: actions/upload-artifact@v4 + if: always() + with: + name: e2e-k3k-logs + path: /tmp/k3k.log \ No newline at end of file diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index f5a30a25..9d504975 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -62,93 +62,6 @@ jobs: files: ./cover.out flags: unit - tests-e2e: - runs-on: ubuntu-latest - needs: validate - - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 - fetch-tags: true - - - uses: actions/setup-go@v5 - with: - go-version-file: go.mod - - - name: Install Ginkgo - run: go install github.com/onsi/ginkgo/v2/ginkgo - - - name: Setup environment - run: | - mkdir ${{ github.workspace }}/covdata - - echo "COVERAGE=true" >> $GITHUB_ENV - echo "GOCOVERDIR=${{ github.workspace }}/covdata" >> $GITHUB_ENV - echo "REPO=ttl.sh/$(uuidgen)" >> $GITHUB_ENV - echo "VERSION=1h" >> $GITHUB_ENV - echo "K3S_HOST_VERSION=v1.32.1+k3s1 >> $GITHUB_ENV" - - - name: Install k3s - run: | - curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=${{ env.K3S_HOST_VERSION }} INSTALL_K3S_EXEC="--write-kubeconfig-mode=777" sh -s - - - - name: Build and package and push dev images - env: - KUBECONFIG: /etc/rancher/k3s/k3s.yaml - REPO: ${{ env.REPO }} - VERSION: ${{ env.VERSION }} - run: | - make build - make package - make push - make install - - # add k3kcli to $PATH - echo "${{ github.workspace }}/bin" >> $GITHUB_PATH - - - name: Check k3kcli - run: k3kcli -v - - - name: Run e2e tests - env: - KUBECONFIG: /etc/rancher/k3s/k3s.yaml - REPO: ${{ env.REPO }} - VERSION: ${{ env.VERSION }} - run: make test-e2e - - - name: Convert coverage data - run: go tool covdata textfmt -i=${GOCOVERDIR} -o ${GOCOVERDIR}/cover.out - - - name: Upload coverage reports to Codecov (controller) - uses: codecov/codecov-action@v5 - with: - token: ${{ secrets.CODECOV_TOKEN }} - files: ${GOCOVERDIR}/cover.out - flags: controller - - - name: Upload coverage reports to Codecov (e2e) - uses: codecov/codecov-action@v5 - with: - token: ${{ secrets.CODECOV_TOKEN }} - files: ./cover.out - flags: e2e - - - name: Archive k3s logs - uses: actions/upload-artifact@v4 - if: always() - with: - name: e2e-k3s-logs - path: /tmp/k3s.log - - - name: Archive k3k logs - uses: actions/upload-artifact@v4 - if: always() - with: - name: e2e-k3k-logs - path: /tmp/k3k.log - tests-cli: runs-on: ubuntu-latest needs: validate diff --git a/Makefile b/Makefile index 9a8bd9da..2c11f178 100644 --- a/Makefile +++ b/Makefile @@ -18,6 +18,9 @@ CRD_REF_DOCS := go run github.com/elastic/crd-ref-docs@$(CRD_REF_DOCS_VER) ENVTEST ?= go run sigs.k8s.io/controller-runtime/tools/setup-envtest@$(ENVTEST_VERSION) ENVTEST_DIR ?= $(shell pwd)/.envtest + +E2E_LABEL_FILTER ?= "e2e" + export KUBEBUILDER_ASSETS ?= $(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(ENVTEST_DIR) -p path) @@ -69,7 +72,7 @@ test-kubelet-controller: ## Run the controller tests (pkg/controller) .PHONY: test-e2e test-e2e: ## Run the e2e tests - $(GINKGO) $(GINKGO_FLAGS) --label-filter=e2e tests + $(GINKGO) $(GINKGO_FLAGS) --label-filter="$(E2E_LABEL_FILTER)" tests .PHONY: test-cli test-cli: ## Run the cli tests diff --git a/tests/cluster_certs_test.go b/tests/cluster_certs_test.go index 7e16722f..dc629df8 100644 --- a/tests/cluster_certs_test.go +++ b/tests/cluster_certs_test.go @@ -13,7 +13,7 @@ import ( . "github.com/onsi/gomega" ) -var _ = When("a cluster with custom certificates is installed with individual cert secrets", Label("e2e"), func() { +var _ = When("a cluster with custom certificates is installed with individual cert secrets", Label("e2e"), Label(certificatesTestsLabel), func() { var virtualCluster *VirtualCluster BeforeEach(func() { diff --git a/tests/cluster_network_test.go b/tests/cluster_network_test.go index 18e854aa..b097c58a 100644 --- a/tests/cluster_network_test.go +++ b/tests/cluster_network_test.go @@ -5,7 +5,7 @@ import ( . "github.com/onsi/gomega" ) -var _ = When("two virtual clusters are installed", Label("e2e"), func() { +var _ = When("two virtual clusters are installed", Label("e2e"), Label(networkingTestsLabel), func() { var ( cluster1 *VirtualCluster cluster2 *VirtualCluster diff --git a/tests/cluster_persistence_test.go b/tests/cluster_persistence_test.go index 9091ed47..ff89cab7 100644 --- a/tests/cluster_persistence_test.go +++ b/tests/cluster_persistence_test.go @@ -20,7 +20,7 @@ import ( . "github.com/onsi/gomega" ) -var _ = When("an ephemeral cluster is installed", Label("e2e"), func() { +var _ = When("an ephemeral cluster is installed", Label("e2e"), Label(persistenceTestsLabel), func() { var virtualCluster *VirtualCluster BeforeEach(func() { @@ -111,7 +111,7 @@ var _ = When("an ephemeral cluster is installed", Label("e2e"), func() { }) }) -var _ = When("a dynamic cluster is installed", Label("e2e"), func() { +var _ = When("a dynamic cluster is installed", Label("e2e"), Label(persistenceTestsLabel), func() { var virtualCluster *VirtualCluster BeforeEach(func() { diff --git a/tests/cluster_status_test.go b/tests/cluster_status_test.go index afe85be5..e8e5635e 100644 --- a/tests/cluster_status_test.go +++ b/tests/cluster_status_test.go @@ -18,7 +18,7 @@ import ( . "github.com/onsi/gomega" ) -var _ = When("a cluster's status is tracked", Label("e2e"), func() { +var _ = When("a cluster's status is tracked", Label("e2e"), Label(statusTestsLabel), func() { var ( namespace *corev1.Namespace vcp *v1beta1.VirtualClusterPolicy diff --git a/tests/cluster_update_test.go b/tests/cluster_update_test.go index ec736c2f..00f5b399 100644 --- a/tests/cluster_update_test.go +++ b/tests/cluster_update_test.go @@ -18,12 +18,16 @@ import ( . "github.com/onsi/gomega" ) -var _ = When("a shared mode cluster update its envs", Label("e2e"), func() { +var _ = When("a shared mode cluster update its envs", Label("e2e"), Label(updateTestsLabel), Label(slowTestsLabel), func() { var virtualCluster *VirtualCluster ctx := context.Background() BeforeEach(func() { namespace := NewNamespace() + DeferCleanup(func() { + DeleteNamespaces(namespace.Name) + }) + cluster := NewCluster(namespace.Name) // Add initial environment variables for server @@ -152,12 +156,16 @@ var _ = When("a shared mode cluster update its envs", Label("e2e"), func() { }) }) -var _ = When("a shared mode cluster update its server args", Label("e2e"), func() { +var _ = When("a shared mode cluster update its server args", Label("e2e"), Label(updateTestsLabel), Label(slowTestsLabel), func() { var virtualCluster *VirtualCluster ctx := context.Background() BeforeEach(func() { namespace := NewNamespace() + DeferCleanup(func() { + DeleteNamespaces(namespace.Name) + }) + cluster := NewCluster(namespace.Name) // Add initial args for server @@ -207,12 +215,16 @@ var _ = When("a shared mode cluster update its server args", Label("e2e"), func( }) }) -var _ = When("a virtual mode cluster update its envs", Label("e2e"), func() { +var _ = When("a virtual mode cluster update its envs", Label("e2e"), Label(updateTestsLabel), Label(slowTestsLabel), func() { var virtualCluster *VirtualCluster ctx := context.Background() BeforeEach(func() { namespace := NewNamespace() + DeferCleanup(func() { + DeleteNamespaces(namespace.Name) + }) + cluster := NewCluster(namespace.Name) // Add initial environment variables for server @@ -239,7 +251,7 @@ var _ = When("a virtual mode cluster update its envs", Label("e2e"), func() { } cluster.Spec.Mode = v1beta1.VirtualClusterMode - cluster.Spec.Agents = ptr.To(int32(1)) + cluster.Spec.Agents = ptr.To[int32](1) CreateCluster(cluster) @@ -344,7 +356,7 @@ var _ = When("a virtual mode cluster update its envs", Label("e2e"), func() { }) }) -var _ = When("a virtual mode cluster update its server args", Label("e2e"), func() { +var _ = When("a virtual mode cluster update its server args", Label("e2e"), Label(updateTestsLabel), Label(slowTestsLabel), func() { var virtualCluster *VirtualCluster ctx := context.Background() BeforeEach(func() { @@ -358,7 +370,7 @@ var _ = When("a virtual mode cluster update its server args", Label("e2e"), func } cluster.Spec.Mode = v1beta1.VirtualClusterMode - cluster.Spec.Agents = ptr.To(int32(1)) + cluster.Spec.Agents = ptr.To[int32](1) CreateCluster(cluster) @@ -402,7 +414,7 @@ var _ = When("a virtual mode cluster update its server args", Label("e2e"), func }) }) -var _ = When("a shared mode cluster update its version", Label("e2e"), func() { +var _ = When("a shared mode cluster update its version", Label("e2e"), Label(updateTestsLabel), Label(slowTestsLabel), func() { var ( virtualCluster *VirtualCluster nginxPod *v1.Pod @@ -411,6 +423,10 @@ var _ = When("a shared mode cluster update its version", Label("e2e"), func() { ctx := context.Background() namespace := NewNamespace() + DeferCleanup(func() { + DeleteNamespaces(namespace.Name) + }) + cluster := NewCluster(namespace.Name) // Add initial version @@ -462,8 +478,7 @@ var _ = When("a shared mode cluster update its version", Label("e2e"), func() { g.Expect(len(serverPods)).To(Equal(1)) serverPod := serverPods[0] - condIndex, cond := pod.GetPodCondition(&serverPod.Status, v1.PodReady) - g.Expect(condIndex).NotTo(Equal(-1)) + _, cond := pod.GetPodCondition(&serverPod.Status, v1.PodReady) g.Expect(cond).NotTo(BeNil()) g.Expect(cond.Status).To(BeEquivalentTo(metav1.ConditionTrue)) @@ -475,9 +490,7 @@ var _ = When("a shared mode cluster update its version", Label("e2e"), func() { nginxPod, err = virtualCluster.Client.CoreV1().Pods(nginxPod.Namespace).Get(ctx, nginxPod.Name, metav1.GetOptions{}) g.Expect(err).To(BeNil()) - - condIndex, cond = pod.GetPodCondition(&nginxPod.Status, v1.PodReady) - g.Expect(condIndex).NotTo(Equal(-1)) + _, cond = pod.GetPodCondition(&nginxPod.Status, v1.PodReady) g.Expect(cond).NotTo(BeNil()) g.Expect(cond.Status).To(BeEquivalentTo(metav1.ConditionTrue)) }). @@ -487,7 +500,7 @@ var _ = When("a shared mode cluster update its version", Label("e2e"), func() { }) }) -var _ = When("a virtual mode cluster update its version", Label("e2e"), func() { +var _ = When("a virtual mode cluster update its version", Label("e2e"), Label(updateTestsLabel), Label(slowTestsLabel), func() { var ( virtualCluster *VirtualCluster nginxPod *v1.Pod @@ -496,13 +509,17 @@ var _ = When("a virtual mode cluster update its version", Label("e2e"), func() { ctx := context.Background() namespace := NewNamespace() + DeferCleanup(func() { + DeleteNamespaces(namespace.Name) + }) + cluster := NewCluster(namespace.Name) // Add initial version cluster.Spec.Version = "v1.31.13-k3s1" cluster.Spec.Mode = v1beta1.VirtualClusterMode - cluster.Spec.Agents = ptr.To(int32(1)) + cluster.Spec.Agents = ptr.To[int32](1) // need to enable persistence for this cluster.Spec.Persistence = v1beta1.PersistenceConfig{ @@ -550,8 +567,7 @@ var _ = When("a virtual mode cluster update its version", Label("e2e"), func() { g.Expect(len(serverPods)).To(Equal(1)) serverPod := serverPods[0] - condIndex, cond := pod.GetPodCondition(&serverPod.Status, v1.PodReady) - g.Expect(condIndex).NotTo(Equal(-1)) + _, cond := pod.GetPodCondition(&serverPod.Status, v1.PodReady) g.Expect(cond).NotTo(BeNil()) g.Expect(cond.Status).To(BeEquivalentTo(metav1.ConditionTrue)) @@ -562,8 +578,7 @@ var _ = When("a virtual mode cluster update its version", Label("e2e"), func() { g.Expect(len(agentPods)).To(Equal(1)) agentPod := agentPods[0] - condIndex, cond = pod.GetPodCondition(&agentPod.Status, v1.PodReady) - g.Expect(condIndex).NotTo(Equal(-1)) + _, cond = pod.GetPodCondition(&agentPod.Status, v1.PodReady) g.Expect(cond).NotTo(BeNil()) g.Expect(cond.Status).To(BeEquivalentTo(metav1.ConditionTrue)) @@ -576,8 +591,7 @@ var _ = When("a virtual mode cluster update its version", Label("e2e"), func() { nginxPod, err = virtualCluster.Client.CoreV1().Pods(nginxPod.Namespace).Get(ctx, nginxPod.Name, metav1.GetOptions{}) g.Expect(err).To(BeNil()) - condIndex, cond = pod.GetPodCondition(&nginxPod.Status, v1.PodReady) - g.Expect(condIndex).NotTo(Equal(-1)) + _, cond = pod.GetPodCondition(&nginxPod.Status, v1.PodReady) g.Expect(cond).NotTo(BeNil()) g.Expect(cond.Status).To(BeEquivalentTo(metav1.ConditionTrue)) }). @@ -586,3 +600,349 @@ var _ = When("a virtual mode cluster update its version", Label("e2e"), func() { Should(Succeed()) }) }) + +var _ = When("a shared mode cluster scales up servers", Label("e2e"), Label(updateTestsLabel), Label(slowTestsLabel), func() { + var ( + virtualCluster *VirtualCluster + nginxPod *v1.Pod + ) + BeforeEach(func() { + ctx := context.Background() + namespace := NewNamespace() + + DeferCleanup(func() { + DeleteNamespaces(namespace.Name) + }) + + cluster := NewCluster(namespace.Name) + + // need to enable persistence for this + cluster.Spec.Persistence = v1beta1.PersistenceConfig{ + Type: v1beta1.DynamicPersistenceMode, + } + + CreateCluster(cluster) + + client, restConfig := NewVirtualK8sClientAndConfig(cluster) + + virtualCluster = &VirtualCluster{ + Cluster: cluster, + RestConfig: restConfig, + Client: client, + } + sPods := listServerPods(ctx, virtualCluster) + Expect(len(sPods)).To(Equal(1)) + + Eventually(func(g Gomega) { + // since there is no way to check nodes in shared mode + // we can check if the endpoints are registered to N nodes + k8sEndpointSlices, err := virtualCluster.Client.DiscoveryV1().EndpointSlices("default").Get(ctx, "kubernetes", metav1.GetOptions{}) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(len(k8sEndpointSlices.Endpoints)).To(Equal(1)) + }). + WithPolling(time.Second * 2). + WithTimeout(time.Minute * 3). + Should(Succeed()) + + nginxPod, _ = virtualCluster.NewNginxPod("") + }) + It("will scale up server pods", func() { + var cluster v1beta1.Cluster + ctx := context.Background() + + err := k8sClient.Get(ctx, ctrlruntimeclient.ObjectKeyFromObject(virtualCluster.Cluster), &cluster) + Expect(err).NotTo(HaveOccurred()) + + // scale cluster servers to 3 nodes + cluster.Spec.Servers = ptr.To[int32](3) + + err = k8sClient.Update(ctx, &cluster) + Expect(err).NotTo(HaveOccurred()) + + Eventually(func(g Gomega) { + // server pods + serverPods := listServerPods(ctx, virtualCluster) + g.Expect(len(serverPods)).To(Equal(3)) + + for _, serverPod := range serverPods { + _, cond := pod.GetPodCondition(&serverPod.Status, v1.PodReady) + g.Expect(cond).NotTo(BeNil()) + g.Expect(cond.Status).To(BeEquivalentTo(metav1.ConditionTrue)) + } + + k8sEndpointSlices, err := virtualCluster.Client.DiscoveryV1().EndpointSlices("default").Get(ctx, "kubernetes", metav1.GetOptions{}) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(len(k8sEndpointSlices.Endpoints)).To(Equal(3)) + + nginxPod, err = virtualCluster.Client.CoreV1().Pods(nginxPod.Namespace).Get(ctx, nginxPod.Name, metav1.GetOptions{}) + g.Expect(err).To(BeNil()) + _, cond := pod.GetPodCondition(&nginxPod.Status, v1.PodReady) + g.Expect(cond).NotTo(BeNil()) + g.Expect(cond.Status).To(BeEquivalentTo(metav1.ConditionTrue)) + }). + WithPolling(time.Second * 2). + WithTimeout(time.Minute * 3). + Should(Succeed()) + }) +}) + +var _ = When("a shared mode cluster scales down servers", Label("e2e"), Label(updateTestsLabel), Label(slowTestsLabel), func() { + var ( + virtualCluster *VirtualCluster + nginxPod *v1.Pod + ) + BeforeEach(func() { + ctx := context.Background() + namespace := NewNamespace() + + DeferCleanup(func() { + DeleteNamespaces(namespace.Name) + }) + + cluster := NewCluster(namespace.Name) + + // start cluster with 3 servers + cluster.Spec.Servers = ptr.To[int32](3) + + // need to enable persistence for this + cluster.Spec.Persistence = v1beta1.PersistenceConfig{ + Type: v1beta1.DynamicPersistenceMode, + } + + CreateCluster(cluster) + + client, restConfig := NewVirtualK8sClientAndConfig(cluster) + + virtualCluster = &VirtualCluster{ + Cluster: cluster, + RestConfig: restConfig, + Client: client, + } + // no need to check servers status since createCluster() will wait until all servers are in ready state + sPods := listServerPods(ctx, virtualCluster) + Expect(len(sPods)).To(Equal(3)) + + Eventually(func(g Gomega) { + // since there is no way to check nodes in shared mode + // we can check if the endpoints are registered to N nodes + k8sEndpointSlices, err := virtualCluster.Client.DiscoveryV1().EndpointSlices("default").Get(ctx, "kubernetes", metav1.GetOptions{}) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(len(k8sEndpointSlices.Endpoints)).To(Equal(3)) + }). + WithPolling(time.Second * 2). + WithTimeout(time.Minute * 3). + Should(Succeed()) + + nginxPod, _ = virtualCluster.NewNginxPod("") + }) + It("will scale down server pods", func() { + var cluster v1beta1.Cluster + ctx := context.Background() + + err := k8sClient.Get(ctx, ctrlruntimeclient.ObjectKeyFromObject(virtualCluster.Cluster), &cluster) + Expect(err).NotTo(HaveOccurred()) + + // scale down cluster servers to 1 node + cluster.Spec.Servers = ptr.To[int32](1) + + err = k8sClient.Update(ctx, &cluster) + Expect(err).NotTo(HaveOccurred()) + + Eventually(func(g Gomega) { + // server pods + serverPods := listServerPods(ctx, virtualCluster) + g.Expect(len(serverPods)).To(Equal(1)) + + _, cond := pod.GetPodCondition(&serverPods[0].Status, v1.PodReady) + g.Expect(cond).NotTo(BeNil()) + g.Expect(cond.Status).To(BeEquivalentTo(metav1.ConditionTrue)) + + k8sEndpointSlices, err := virtualCluster.Client.DiscoveryV1().EndpointSlices("default").Get(ctx, "kubernetes", metav1.GetOptions{}) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(len(k8sEndpointSlices.Endpoints)).To(Equal(1)) + + nginxPod, err = virtualCluster.Client.CoreV1().Pods(nginxPod.Namespace).Get(ctx, nginxPod.Name, metav1.GetOptions{}) + g.Expect(err).To(BeNil()) + _, cond = pod.GetPodCondition(&nginxPod.Status, v1.PodReady) + g.Expect(cond).NotTo(BeNil()) + g.Expect(cond.Status).To(BeEquivalentTo(metav1.ConditionTrue)) + }). + WithPolling(time.Second * 2). + WithTimeout(time.Minute * 3). + Should(Succeed()) + }) +}) + +var _ = When("a virtual mode cluster scales up servers", Label("e2e"), Label(updateTestsLabel), Label(slowTestsLabel), func() { + var ( + virtualCluster *VirtualCluster + nginxPod *v1.Pod + ) + BeforeEach(func() { + ctx := context.Background() + namespace := NewNamespace() + + DeferCleanup(func() { + DeleteNamespaces(namespace.Name) + }) + + cluster := NewCluster(namespace.Name) + + cluster.Spec.Mode = v1beta1.VirtualClusterMode + + // need to enable persistence for this + cluster.Spec.Persistence = v1beta1.PersistenceConfig{ + Type: v1beta1.DynamicPersistenceMode, + } + + CreateCluster(cluster) + + client, restConfig := NewVirtualK8sClientAndConfig(cluster) + + virtualCluster = &VirtualCluster{ + Cluster: cluster, + RestConfig: restConfig, + Client: client, + } + sPods := listServerPods(ctx, virtualCluster) + Expect(len(sPods)).To(Equal(1)) + + Eventually(func(g Gomega) { + nodes, err := virtualCluster.Client.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(len(nodes.Items)).To(Equal(1)) + }). + WithPolling(time.Second * 2). + WithTimeout(time.Minute * 5). + Should(Succeed()) + + nginxPod, _ = virtualCluster.NewNginxPod("") + }) + It("will scale up server pods", func() { + var cluster v1beta1.Cluster + ctx := context.Background() + + err := k8sClient.Get(ctx, ctrlruntimeclient.ObjectKeyFromObject(virtualCluster.Cluster), &cluster) + Expect(err).NotTo(HaveOccurred()) + + // scale cluster servers to 3 nodes + cluster.Spec.Servers = ptr.To[int32](3) + + err = k8sClient.Update(ctx, &cluster) + Expect(err).NotTo(HaveOccurred()) + + Eventually(func(g Gomega) { + // server pods + serverPods := listServerPods(ctx, virtualCluster) + g.Expect(len(serverPods)).To(Equal(3)) + + for _, serverPod := range serverPods { + _, cond := pod.GetPodCondition(&serverPod.Status, v1.PodReady) + g.Expect(cond).NotTo(BeNil()) + g.Expect(cond.Status).To(BeEquivalentTo(metav1.ConditionTrue)) + } + + nodes, err := virtualCluster.Client.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(len(nodes.Items)).To(Equal(3)) + + nginxPod, err = virtualCluster.Client.CoreV1().Pods(nginxPod.Namespace).Get(ctx, nginxPod.Name, metav1.GetOptions{}) + g.Expect(err).To(BeNil()) + _, cond := pod.GetPodCondition(&nginxPod.Status, v1.PodReady) + g.Expect(cond).NotTo(BeNil()) + g.Expect(cond.Status).To(BeEquivalentTo(metav1.ConditionTrue)) + }). + WithPolling(time.Second * 2). + WithTimeout(time.Minute * 5). + Should(Succeed()) + }) +}) + +var _ = When("a virtual mode cluster scales down servers", Label("e2e"), Label(updateTestsLabel), Label(slowTestsLabel), func() { + var ( + virtualCluster *VirtualCluster + nginxPod *v1.Pod + ) + BeforeEach(func() { + ctx := context.Background() + namespace := NewNamespace() + + DeferCleanup(func() { + DeleteNamespaces(namespace.Name) + }) + + cluster := NewCluster(namespace.Name) + + cluster.Spec.Mode = v1beta1.VirtualClusterMode + + // start cluster with 3 servers + cluster.Spec.Servers = ptr.To[int32](3) + + // need to enable persistence for this + cluster.Spec.Persistence = v1beta1.PersistenceConfig{ + Type: v1beta1.DynamicPersistenceMode, + } + + CreateCluster(cluster) + + client, restConfig := NewVirtualK8sClientAndConfig(cluster) + + virtualCluster = &VirtualCluster{ + Cluster: cluster, + RestConfig: restConfig, + Client: client, + } + // no need to check servers status since createCluster() will wait until all servers are in ready state + sPods := listServerPods(ctx, virtualCluster) + Expect(len(sPods)).To(Equal(3)) + + Eventually(func(g Gomega) { + nodes, err := virtualCluster.Client.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(len(nodes.Items)).To(Equal(3)) + }). + WithPolling(time.Second * 2). + WithTimeout(time.Minute * 5). + Should(Succeed()) + + nginxPod, _ = virtualCluster.NewNginxPod("") + }) + It("will scale down server pods", func() { + var cluster v1beta1.Cluster + ctx := context.Background() + + err := k8sClient.Get(ctx, ctrlruntimeclient.ObjectKeyFromObject(virtualCluster.Cluster), &cluster) + Expect(err).NotTo(HaveOccurred()) + + // scale down cluster servers to 1 node + cluster.Spec.Servers = ptr.To[int32](1) + + err = k8sClient.Update(ctx, &cluster) + Expect(err).NotTo(HaveOccurred()) + + Eventually(func(g Gomega) { + // server pods + serverPods := listServerPods(ctx, virtualCluster) + g.Expect(len(serverPods)).To(Equal(1)) + + _, cond := pod.GetPodCondition(&serverPods[0].Status, v1.PodReady) + g.Expect(cond).NotTo(BeNil()) + g.Expect(cond.Status).To(BeEquivalentTo(metav1.ConditionTrue)) + + // we can't check for number of nodes in scale down because the nodes will be there but in a non-ready state + k8sEndpointSlices, err := virtualCluster.Client.DiscoveryV1().EndpointSlices("default").Get(ctx, "kubernetes", metav1.GetOptions{}) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(len(k8sEndpointSlices.Endpoints)).To(Equal(1)) + + nginxPod, err = virtualCluster.Client.CoreV1().Pods(nginxPod.Namespace).Get(ctx, nginxPod.Name, metav1.GetOptions{}) + g.Expect(err).To(BeNil()) + + _, cond = pod.GetPodCondition(&nginxPod.Status, v1.PodReady) + g.Expect(cond).NotTo(BeNil()) + g.Expect(cond.Status).To(BeEquivalentTo(metav1.ConditionTrue)) + }). + WithPolling(time.Second * 2). + WithTimeout(time.Minute * 5). + Should(Succeed()) + }) +}) diff --git a/tests/tests_suite_test.go b/tests/tests_suite_test.go index 4a9b37c4..9b656e0d 100644 --- a/tests/tests_suite_test.go +++ b/tests/tests_suite_test.go @@ -42,6 +42,13 @@ import ( const ( k3kNamespace = "k3k-system" k3kName = "k3k" + + slowTestsLabel = "slow" + updateTestsLabel = "update" + persistenceTestsLabel = "persistence" + networkingTestsLabel = "networking" + statusTestsLabel = "status" + certificatesTestsLabel = "certificates" ) func TestTests(t *testing.T) {