mirror of
https://github.com/rancher/k3k.git
synced 2026-08-19 04:16:16 +00:00
Adding upgrade path tests (#481)
* Adding upgrade path tests Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com> * wsl Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com> * fixes Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com> * fixes Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com> * Remove update label Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com> --------- Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>
This commit is contained in:
Hussein Galal
@@ -0,0 +1,203 @@
|
||||
package k3k_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client"
|
||||
|
||||
"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
var _ = When("a shared mode cluster update its envs", Label("e2e"), func() {
|
||||
var virtualCluster *VirtualCluster
|
||||
ctx := context.Background()
|
||||
BeforeEach(func() {
|
||||
namespace := NewNamespace()
|
||||
|
||||
cluster := NewCluster(namespace.Name)
|
||||
|
||||
// Add initial environment variables for server
|
||||
cluster.Spec.ServerEnvs = []corev1.EnvVar{
|
||||
{
|
||||
Name: "TEST_SERVER_ENV_1",
|
||||
Value: "not_upgraded",
|
||||
},
|
||||
{
|
||||
Name: "TEST_SERVER_ENV_2",
|
||||
Value: "toBeRemoved",
|
||||
},
|
||||
}
|
||||
// Add initial environment variables for agent
|
||||
cluster.Spec.AgentEnvs = []corev1.EnvVar{
|
||||
{
|
||||
Name: "TEST_AGENT_ENV_1",
|
||||
Value: "not_upgraded",
|
||||
},
|
||||
{
|
||||
Name: "TEST_AGENT_ENV_2",
|
||||
Value: "toBeRemoved",
|
||||
},
|
||||
}
|
||||
|
||||
CreateCluster(cluster)
|
||||
|
||||
client, restConfig := NewVirtualK8sClientAndConfig(cluster)
|
||||
|
||||
virtualCluster = &VirtualCluster{
|
||||
Cluster: cluster,
|
||||
RestConfig: restConfig,
|
||||
Client: client,
|
||||
}
|
||||
sPods := listServerPods(ctx, virtualCluster)
|
||||
Expect(len(sPods)).To(Equal(1))
|
||||
|
||||
serverPod := sPods[0]
|
||||
|
||||
serverEnv1, ok := getEnv(&serverPod, "TEST_SERVER_ENV_1")
|
||||
Expect(ok).To(BeTrue())
|
||||
Expect(serverEnv1).To(Equal("not_upgraded"))
|
||||
|
||||
serverEnv2, ok := getEnv(&serverPod, "TEST_SERVER_ENV_2")
|
||||
Expect(ok).To(BeTrue())
|
||||
Expect(serverEnv2).To(Equal("toBeRemoved"))
|
||||
|
||||
aPods := listAgentPods(ctx, virtualCluster)
|
||||
Expect(len(aPods)).To(Equal(1))
|
||||
|
||||
agentPod := aPods[0]
|
||||
|
||||
agentEnv1, ok := getEnv(&agentPod, "TEST_AGENT_ENV_1")
|
||||
Expect(ok).To(BeTrue())
|
||||
Expect(agentEnv1).To(Equal("not_upgraded"))
|
||||
|
||||
agentEnv2, ok := getEnv(&agentPod, "TEST_AGENT_ENV_2")
|
||||
Expect(ok).To(BeTrue())
|
||||
Expect(agentEnv2).To(Equal("toBeRemoved"))
|
||||
})
|
||||
It("will update server and agent envs when cluster is updated", func() {
|
||||
Eventually(func(g Gomega) {
|
||||
var cluster v1alpha1.Cluster
|
||||
|
||||
err := k8sClient.Get(ctx, ctrlruntimeclient.ObjectKeyFromObject(virtualCluster.Cluster), &cluster)
|
||||
g.Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
// update both agent and server envs
|
||||
cluster.Spec.ServerEnvs = []corev1.EnvVar{
|
||||
{
|
||||
Name: "TEST_SERVER_ENV_1",
|
||||
Value: "upgraded",
|
||||
},
|
||||
{
|
||||
Name: "TEST_SERVER_ENV_3",
|
||||
Value: "new",
|
||||
},
|
||||
}
|
||||
cluster.Spec.AgentEnvs = []corev1.EnvVar{
|
||||
{
|
||||
Name: "TEST_AGENT_ENV_1",
|
||||
Value: "upgraded",
|
||||
},
|
||||
{
|
||||
Name: "TEST_AGENT_ENV_3",
|
||||
Value: "new",
|
||||
},
|
||||
}
|
||||
|
||||
err = k8sClient.Update(ctx, &cluster)
|
||||
g.Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
// server pods
|
||||
serverPods := listServerPods(ctx, virtualCluster)
|
||||
g.Expect(len(serverPods)).To(Equal(1))
|
||||
|
||||
serverEnv1, ok := getEnv(&serverPods[0], "TEST_SERVER_ENV_1")
|
||||
g.Expect(ok).To(BeTrue())
|
||||
g.Expect(serverEnv1).To(Equal("upgraded"))
|
||||
|
||||
_, ok = getEnv(&serverPods[0], "TEST_SERVER_ENV_2")
|
||||
g.Expect(ok).To(BeFalse())
|
||||
|
||||
serverEnv3, ok := getEnv(&serverPods[0], "TEST_SERVER_ENV_3")
|
||||
g.Expect(ok).To(BeTrue())
|
||||
g.Expect(serverEnv3).To(Equal("new"))
|
||||
|
||||
// agent pods
|
||||
aPods := listAgentPods(ctx, virtualCluster)
|
||||
g.Expect(len(aPods)).To(Equal(1))
|
||||
|
||||
agentEnv1, ok := getEnv(&aPods[0], "TEST_AGENT_ENV_1")
|
||||
g.Expect(ok).To(BeTrue())
|
||||
g.Expect(agentEnv1).To(Equal("upgraded"))
|
||||
|
||||
_, ok = getEnv(&aPods[0], "TEST_AGENT_ENV_2")
|
||||
g.Expect(ok).To(BeFalse())
|
||||
|
||||
agentEnv3, ok := getEnv(&aPods[0], "TEST_AGENT_ENV_3")
|
||||
g.Expect(ok).To(BeTrue())
|
||||
g.Expect(agentEnv3).To(Equal("new"))
|
||||
}).
|
||||
WithPolling(time.Second * 2).
|
||||
WithTimeout(time.Minute * 2).
|
||||
Should(Succeed())
|
||||
})
|
||||
})
|
||||
|
||||
var _ = When("a shared mode cluster update its server args", Label("e2e"), func() {
|
||||
var virtualCluster *VirtualCluster
|
||||
ctx := context.Background()
|
||||
BeforeEach(func() {
|
||||
namespace := NewNamespace()
|
||||
|
||||
cluster := NewCluster(namespace.Name)
|
||||
|
||||
// Add initial args for server
|
||||
cluster.Spec.ServerArgs = []string{
|
||||
"--node-label=test_server=not_upgraded",
|
||||
}
|
||||
|
||||
CreateCluster(cluster)
|
||||
|
||||
client, restConfig := NewVirtualK8sClientAndConfig(cluster)
|
||||
|
||||
virtualCluster = &VirtualCluster{
|
||||
Cluster: cluster,
|
||||
RestConfig: restConfig,
|
||||
Client: client,
|
||||
}
|
||||
sPods := listServerPods(ctx, virtualCluster)
|
||||
Expect(len(sPods)).To(Equal(1))
|
||||
|
||||
serverPod := sPods[0]
|
||||
|
||||
Expect(isArgFound(&serverPod, "--node-label=test_server=not_upgraded")).To(BeTrue())
|
||||
})
|
||||
It("will update server args", func() {
|
||||
Eventually(func(g Gomega) {
|
||||
var cluster v1alpha1.Cluster
|
||||
|
||||
err := k8sClient.Get(ctx, ctrlruntimeclient.ObjectKeyFromObject(virtualCluster.Cluster), &cluster)
|
||||
g.Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
cluster.Spec.ServerArgs = []string{
|
||||
"--node-label=test_server=upgraded",
|
||||
}
|
||||
|
||||
err = k8sClient.Update(ctx, &cluster)
|
||||
g.Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
// server pods
|
||||
sPods := listServerPods(ctx, virtualCluster)
|
||||
g.Expect(len(sPods)).To(Equal(1))
|
||||
|
||||
g.Expect(isArgFound(&sPods[0], "--node-label=test_server=upgraded")).To(BeTrue())
|
||||
}).
|
||||
WithPolling(time.Second * 2).
|
||||
WithTimeout(time.Minute * 2).
|
||||
Should(Succeed())
|
||||
})
|
||||
})
|
||||
+52
-6
@@ -154,17 +154,21 @@ func CreateCluster(cluster *v1alpha1.Cluster) {
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
|
||||
serverRunning := false
|
||||
kubeletRunning := false
|
||||
agentRunning := false
|
||||
|
||||
for _, pod := range podList.Items {
|
||||
imageName := pod.Spec.Containers[0].Image
|
||||
if strings.Contains(imageName, "rancher/k3s") {
|
||||
if pod.Labels == nil {
|
||||
continue
|
||||
}
|
||||
roleLabel := pod.Labels["role"]
|
||||
typeLabel := pod.Labels["type"]
|
||||
if roleLabel == "server" {
|
||||
serverRunning = pod.Status.Phase == corev1.PodRunning
|
||||
} else if strings.Contains(imageName, "rancher/k3k-kubelet") {
|
||||
kubeletRunning = pod.Status.Phase == corev1.PodRunning
|
||||
} else if typeLabel == "agent" {
|
||||
agentRunning = pod.Status.Phase == corev1.PodRunning
|
||||
}
|
||||
|
||||
if serverRunning && kubeletRunning {
|
||||
if serverRunning && agentRunning {
|
||||
return true
|
||||
}
|
||||
}
|
||||
@@ -332,3 +336,45 @@ func restartServerPod(ctx context.Context, virtualCluster *VirtualCluster) {
|
||||
return serverPods.Items[0].DeletionTimestamp
|
||||
}).WithTimeout(60 * time.Second).WithPolling(time.Second * 5).Should(BeNil())
|
||||
}
|
||||
|
||||
func listServerPods(ctx context.Context, virtualCluster *VirtualCluster) []corev1.Pod {
|
||||
labelSelector := "cluster=" + virtualCluster.Cluster.Name + ",role=server"
|
||||
|
||||
serverPods, err := k8s.CoreV1().Pods(virtualCluster.Cluster.Namespace).List(ctx, v1.ListOptions{LabelSelector: labelSelector})
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
|
||||
return serverPods.Items
|
||||
}
|
||||
|
||||
func listAgentPods(ctx context.Context, virtualCluster *VirtualCluster) []corev1.Pod {
|
||||
labelSelector := fmt.Sprintf("cluster=%s,type=agent,mode=%s", virtualCluster.Cluster.Name, virtualCluster.Cluster.Spec.Mode)
|
||||
|
||||
agentPods, err := k8s.CoreV1().Pods(virtualCluster.Cluster.Namespace).List(ctx, v1.ListOptions{LabelSelector: labelSelector})
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
|
||||
return agentPods.Items
|
||||
}
|
||||
|
||||
// getEnv will get an environment variable from a pod it will return empty string if not found
|
||||
func getEnv(pod *corev1.Pod, envName string) (string, bool) {
|
||||
container := pod.Spec.Containers[0]
|
||||
for _, envVar := range container.Env {
|
||||
if envVar.Name == envName {
|
||||
return envVar.Value, true
|
||||
}
|
||||
}
|
||||
|
||||
return "", false
|
||||
}
|
||||
|
||||
// isArgFound will return true if the argument passed to the function is found in container args
|
||||
func isArgFound(pod *corev1.Pod, arg string) bool {
|
||||
container := pod.Spec.Containers[0]
|
||||
for _, cmd := range container.Command {
|
||||
if strings.Contains(cmd, arg) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user