From 80037e815fc7a312fbfa9e838d5ba20d271c285f Mon Sep 17 00:00:00 2001 From: Hussein Galal Date: Thu, 2 Oct 2025 14:53:08 +0300 Subject: [PATCH] Adding upgrade path tests (#481) * Adding upgrade path tests Signed-off-by: galal-hussein * wsl Signed-off-by: galal-hussein * fixes Signed-off-by: galal-hussein * fixes Signed-off-by: galal-hussein * Remove update label Signed-off-by: galal-hussein --------- Signed-off-by: galal-hussein --- tests/cluster_upgrade_test.go | 203 ++++++++++++++++++++++++++++++++++ tests/common_test.go | 58 +++++++++- 2 files changed, 255 insertions(+), 6 deletions(-) create mode 100644 tests/cluster_upgrade_test.go diff --git a/tests/cluster_upgrade_test.go b/tests/cluster_upgrade_test.go new file mode 100644 index 00000000..6da07909 --- /dev/null +++ b/tests/cluster_upgrade_test.go @@ -0,0 +1,203 @@ +package k3k_test + +import ( + "context" + "time" + + corev1 "k8s.io/api/core/v1" + ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = When("a shared mode cluster update its envs", Label("e2e"), func() { + var virtualCluster *VirtualCluster + ctx := context.Background() + BeforeEach(func() { + namespace := NewNamespace() + + cluster := NewCluster(namespace.Name) + + // Add initial environment variables for server + cluster.Spec.ServerEnvs = []corev1.EnvVar{ + { + Name: "TEST_SERVER_ENV_1", + Value: "not_upgraded", + }, + { + Name: "TEST_SERVER_ENV_2", + Value: "toBeRemoved", + }, + } + // Add initial environment variables for agent + cluster.Spec.AgentEnvs = []corev1.EnvVar{ + { + Name: "TEST_AGENT_ENV_1", + Value: "not_upgraded", + }, + { + Name: "TEST_AGENT_ENV_2", + Value: "toBeRemoved", + }, + } + + CreateCluster(cluster) + + client, restConfig := NewVirtualK8sClientAndConfig(cluster) + + virtualCluster = &VirtualCluster{ + Cluster: cluster, + RestConfig: restConfig, + Client: client, + } + sPods := listServerPods(ctx, virtualCluster) + Expect(len(sPods)).To(Equal(1)) + + serverPod := sPods[0] + + serverEnv1, ok := getEnv(&serverPod, "TEST_SERVER_ENV_1") + Expect(ok).To(BeTrue()) + Expect(serverEnv1).To(Equal("not_upgraded")) + + serverEnv2, ok := getEnv(&serverPod, "TEST_SERVER_ENV_2") + Expect(ok).To(BeTrue()) + Expect(serverEnv2).To(Equal("toBeRemoved")) + + aPods := listAgentPods(ctx, virtualCluster) + Expect(len(aPods)).To(Equal(1)) + + agentPod := aPods[0] + + agentEnv1, ok := getEnv(&agentPod, "TEST_AGENT_ENV_1") + Expect(ok).To(BeTrue()) + Expect(agentEnv1).To(Equal("not_upgraded")) + + agentEnv2, ok := getEnv(&agentPod, "TEST_AGENT_ENV_2") + Expect(ok).To(BeTrue()) + Expect(agentEnv2).To(Equal("toBeRemoved")) + }) + It("will update server and agent envs when cluster is updated", func() { + Eventually(func(g Gomega) { + var cluster v1alpha1.Cluster + + err := k8sClient.Get(ctx, ctrlruntimeclient.ObjectKeyFromObject(virtualCluster.Cluster), &cluster) + g.Expect(err).NotTo(HaveOccurred()) + + // update both agent and server envs + cluster.Spec.ServerEnvs = []corev1.EnvVar{ + { + Name: "TEST_SERVER_ENV_1", + Value: "upgraded", + }, + { + Name: "TEST_SERVER_ENV_3", + Value: "new", + }, + } + cluster.Spec.AgentEnvs = []corev1.EnvVar{ + { + Name: "TEST_AGENT_ENV_1", + Value: "upgraded", + }, + { + Name: "TEST_AGENT_ENV_3", + Value: "new", + }, + } + + err = k8sClient.Update(ctx, &cluster) + g.Expect(err).NotTo(HaveOccurred()) + + // server pods + serverPods := listServerPods(ctx, virtualCluster) + g.Expect(len(serverPods)).To(Equal(1)) + + serverEnv1, ok := getEnv(&serverPods[0], "TEST_SERVER_ENV_1") + g.Expect(ok).To(BeTrue()) + g.Expect(serverEnv1).To(Equal("upgraded")) + + _, ok = getEnv(&serverPods[0], "TEST_SERVER_ENV_2") + g.Expect(ok).To(BeFalse()) + + serverEnv3, ok := getEnv(&serverPods[0], "TEST_SERVER_ENV_3") + g.Expect(ok).To(BeTrue()) + g.Expect(serverEnv3).To(Equal("new")) + + // agent pods + aPods := listAgentPods(ctx, virtualCluster) + g.Expect(len(aPods)).To(Equal(1)) + + agentEnv1, ok := getEnv(&aPods[0], "TEST_AGENT_ENV_1") + g.Expect(ok).To(BeTrue()) + g.Expect(agentEnv1).To(Equal("upgraded")) + + _, ok = getEnv(&aPods[0], "TEST_AGENT_ENV_2") + g.Expect(ok).To(BeFalse()) + + agentEnv3, ok := getEnv(&aPods[0], "TEST_AGENT_ENV_3") + g.Expect(ok).To(BeTrue()) + g.Expect(agentEnv3).To(Equal("new")) + }). + WithPolling(time.Second * 2). + WithTimeout(time.Minute * 2). + Should(Succeed()) + }) +}) + +var _ = When("a shared mode cluster update its server args", Label("e2e"), func() { + var virtualCluster *VirtualCluster + ctx := context.Background() + BeforeEach(func() { + namespace := NewNamespace() + + cluster := NewCluster(namespace.Name) + + // Add initial args for server + cluster.Spec.ServerArgs = []string{ + "--node-label=test_server=not_upgraded", + } + + CreateCluster(cluster) + + client, restConfig := NewVirtualK8sClientAndConfig(cluster) + + virtualCluster = &VirtualCluster{ + Cluster: cluster, + RestConfig: restConfig, + Client: client, + } + sPods := listServerPods(ctx, virtualCluster) + Expect(len(sPods)).To(Equal(1)) + + serverPod := sPods[0] + + Expect(isArgFound(&serverPod, "--node-label=test_server=not_upgraded")).To(BeTrue()) + }) + It("will update server args", func() { + Eventually(func(g Gomega) { + var cluster v1alpha1.Cluster + + err := k8sClient.Get(ctx, ctrlruntimeclient.ObjectKeyFromObject(virtualCluster.Cluster), &cluster) + g.Expect(err).NotTo(HaveOccurred()) + + cluster.Spec.ServerArgs = []string{ + "--node-label=test_server=upgraded", + } + + err = k8sClient.Update(ctx, &cluster) + g.Expect(err).NotTo(HaveOccurred()) + + // server pods + sPods := listServerPods(ctx, virtualCluster) + g.Expect(len(sPods)).To(Equal(1)) + + g.Expect(isArgFound(&sPods[0], "--node-label=test_server=upgraded")).To(BeTrue()) + }). + WithPolling(time.Second * 2). + WithTimeout(time.Minute * 2). + Should(Succeed()) + }) +}) diff --git a/tests/common_test.go b/tests/common_test.go index c7f87271..3e080713 100644 --- a/tests/common_test.go +++ b/tests/common_test.go @@ -154,17 +154,21 @@ func CreateCluster(cluster *v1alpha1.Cluster) { Expect(err).To(Not(HaveOccurred())) serverRunning := false - kubeletRunning := false + agentRunning := false for _, pod := range podList.Items { - imageName := pod.Spec.Containers[0].Image - if strings.Contains(imageName, "rancher/k3s") { + if pod.Labels == nil { + continue + } + roleLabel := pod.Labels["role"] + typeLabel := pod.Labels["type"] + if roleLabel == "server" { serverRunning = pod.Status.Phase == corev1.PodRunning - } else if strings.Contains(imageName, "rancher/k3k-kubelet") { - kubeletRunning = pod.Status.Phase == corev1.PodRunning + } else if typeLabel == "agent" { + agentRunning = pod.Status.Phase == corev1.PodRunning } - if serverRunning && kubeletRunning { + if serverRunning && agentRunning { return true } } @@ -332,3 +336,45 @@ func restartServerPod(ctx context.Context, virtualCluster *VirtualCluster) { return serverPods.Items[0].DeletionTimestamp }).WithTimeout(60 * time.Second).WithPolling(time.Second * 5).Should(BeNil()) } + +func listServerPods(ctx context.Context, virtualCluster *VirtualCluster) []corev1.Pod { + labelSelector := "cluster=" + virtualCluster.Cluster.Name + ",role=server" + + serverPods, err := k8s.CoreV1().Pods(virtualCluster.Cluster.Namespace).List(ctx, v1.ListOptions{LabelSelector: labelSelector}) + Expect(err).To(Not(HaveOccurred())) + + return serverPods.Items +} + +func listAgentPods(ctx context.Context, virtualCluster *VirtualCluster) []corev1.Pod { + labelSelector := fmt.Sprintf("cluster=%s,type=agent,mode=%s", virtualCluster.Cluster.Name, virtualCluster.Cluster.Spec.Mode) + + agentPods, err := k8s.CoreV1().Pods(virtualCluster.Cluster.Namespace).List(ctx, v1.ListOptions{LabelSelector: labelSelector}) + Expect(err).To(Not(HaveOccurred())) + + return agentPods.Items +} + +// getEnv will get an environment variable from a pod it will return empty string if not found +func getEnv(pod *corev1.Pod, envName string) (string, bool) { + container := pod.Spec.Containers[0] + for _, envVar := range container.Env { + if envVar.Name == envName { + return envVar.Value, true + } + } + + return "", false +} + +// isArgFound will return true if the argument passed to the function is found in container args +func isArgFound(pod *corev1.Pod, arg string) bool { + container := pod.Spec.Containers[0] + for _, cmd := range container.Command { + if strings.Contains(cmd, arg) { + return true + } + } + + return false +}