Default to dynamic persistence and fix HA restarts (#250)

* Default to dynamic persistence and fix HA restarts Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com> --------- Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>
2026-05-03 16:06:49 +00:00 · 2025-02-14 14:26:10 +02:00
parent 51a8fd8a8d
commit fa553d25d4
15 changed files with 429 additions and 150 deletions
--- a/tests/cluster_test.go
+++ b/tests/cluster_test.go
@@ -42,7 +42,7 @@ var _ = When("k3k is installed", func() {
 	})
 })

-var _ = When("a cluster is installed", func() {
+var _ = When("a ephemeral cluster is installed", func() {

 	var namespace string

@@ -66,6 +66,9 @@ var _ = When("a cluster is installed", func() {
 				Expose: &v1alpha1.ExposeConfig{
 					NodePort: &v1alpha1.NodePortConfig{},
 				},
+				Persistence: v1alpha1.PersistenceConfig{
+					Type: v1alpha1.EphemeralNodeType,
+				},
 			},
 		}

@@ -130,6 +133,9 @@ var _ = When("a cluster is installed", func() {
 				Expose: &v1alpha1.ExposeConfig{
 					NodePort: &v1alpha1.NodePortConfig{},
 				},
+				Persistence: v1alpha1.PersistenceConfig{
+					Type: v1alpha1.EphemeralNodeType,
+				},
 			},
 		}

@@ -191,3 +197,148 @@ var _ = When("a cluster is installed", func() {
 			Should(BeNil())
 	})
 })
+
+var _ = When("a dynamic cluster is installed", func() {
+
+	var namespace string
+
+	BeforeEach(func() {
+		createdNS := &corev1.Namespace{ObjectMeta: v1.ObjectMeta{GenerateName: "ns-"}}
+		createdNS, err := k8s.CoreV1().Namespaces().Create(context.Background(), createdNS, v1.CreateOptions{})
+		Expect(err).To(Not(HaveOccurred()))
+		namespace = createdNS.Name
+	})
+
+	It("can create a nginx pod", func() {
+		ctx := context.Background()
+
+		cluster := v1alpha1.Cluster{
+			ObjectMeta: v1.ObjectMeta{
+				Name:      "mycluster",
+				Namespace: namespace,
+			},
+			Spec: v1alpha1.ClusterSpec{
+				TLSSANs: []string{hostIP},
+				Expose: &v1alpha1.ExposeConfig{
+					NodePort: &v1alpha1.NodePortConfig{},
+				},
+				Persistence: v1alpha1.PersistenceConfig{
+					Type: v1alpha1.DynamicNodesType,
+				},
+			},
+		}
+
+		By(fmt.Sprintf("Creating virtual cluster %s/%s", cluster.Namespace, cluster.Name))
+		NewVirtualCluster(cluster)
+
+		By("Waiting to get a kubernetes client for the virtual cluster")
+		virtualK8sClient := NewVirtualK8sClient(cluster)
+
+		nginxPod := &corev1.Pod{
+			ObjectMeta: v1.ObjectMeta{
+				Name:      "nginx",
+				Namespace: "default",
+			},
+			Spec: corev1.PodSpec{
+				Containers: []corev1.Container{{
+					Name:  "nginx",
+					Image: "nginx",
+				}},
+			},
+		}
+		nginxPod, err := virtualK8sClient.CoreV1().Pods(nginxPod.Namespace).Create(ctx, nginxPod, v1.CreateOptions{})
+		Expect(err).To(Not(HaveOccurred()))
+
+		// check that the nginx Pod is up and running in the host cluster
+		Eventually(func() bool {
+			//labelSelector := fmt.Sprintf("%s=%s", translate.ClusterNameLabel, cluster.Namespace)
+			podList, err := k8s.CoreV1().Pods(namespace).List(ctx, v1.ListOptions{})
+			Expect(err).To(Not(HaveOccurred()))
+
+			for _, pod := range podList.Items {
+				resourceName := pod.Annotations[translate.ResourceNameAnnotation]
+				resourceNamespace := pod.Annotations[translate.ResourceNamespaceAnnotation]
+
+				if resourceName == nginxPod.Name && resourceNamespace == nginxPod.Namespace {
+					fmt.Fprintf(GinkgoWriter,
+						"pod=%s resource=%s/%s status=%s\n",
+						pod.Name, resourceNamespace, resourceName, pod.Status.Phase,
+					)
+
+					return pod.Status.Phase == corev1.PodRunning
+				}
+			}
+
+			return false
+		}).
+			WithTimeout(time.Minute).
+			WithPolling(time.Second * 5).
+			Should(BeTrue())
+	})
+
+	It("use the same bootstrap secret after a restart", func() {
+		ctx := context.Background()
+
+		cluster := v1alpha1.Cluster{
+			ObjectMeta: v1.ObjectMeta{
+				Name:      "mycluster",
+				Namespace: namespace,
+			},
+			Spec: v1alpha1.ClusterSpec{
+				TLSSANs: []string{hostIP},
+				Expose: &v1alpha1.ExposeConfig{
+					NodePort: &v1alpha1.NodePortConfig{},
+				},
+				Persistence: v1alpha1.PersistenceConfig{
+					Type: v1alpha1.DynamicNodesType,
+				},
+			},
+		}
+
+		By(fmt.Sprintf("Creating virtual cluster %s/%s", cluster.Namespace, cluster.Name))
+		NewVirtualCluster(cluster)
+
+		By("Waiting to get a kubernetes client for the virtual cluster")
+		virtualK8sClient := NewVirtualK8sClient(cluster)
+
+		_, err := virtualK8sClient.DiscoveryClient.ServerVersion()
+		Expect(err).To(Not(HaveOccurred()))
+
+		labelSelector := "cluster=" + cluster.Name + ",role=server"
+		serverPods, err := k8s.CoreV1().Pods(namespace).List(ctx, v1.ListOptions{LabelSelector: labelSelector})
+		Expect(err).To(Not(HaveOccurred()))
+
+		Expect(len(serverPods.Items)).To(Equal(1))
+		serverPod := serverPods.Items[0]
+
+		fmt.Fprintf(GinkgoWriter, "deleting pod %s/%s\n", serverPod.Namespace, serverPod.Name)
+		err = k8s.CoreV1().Pods(namespace).Delete(ctx, serverPod.Name, v1.DeleteOptions{})
+		Expect(err).To(Not(HaveOccurred()))
+
+		By("Deleting server pod")
+
+		// check that the server pods restarted
+		Eventually(func() any {
+			serverPods, err = k8s.CoreV1().Pods(namespace).List(ctx, v1.ListOptions{LabelSelector: labelSelector})
+			Expect(err).To(Not(HaveOccurred()))
+			Expect(len(serverPods.Items)).To(Equal(1))
+			return serverPods.Items[0].DeletionTimestamp
+		}).
+			WithTimeout(30 * time.Second).
+			WithPolling(time.Second * 5).
+			Should(BeNil())
+
+		By("Server pod up and running again")
+
+		By("Using old k8s client configuration should succeed")
+
+		Eventually(func() error {
+			virtualK8sClient = NewVirtualK8sClient(cluster)
+			_, err = virtualK8sClient.DiscoveryClient.ServerVersion()
+			return err
+		}).
+			WithTimeout(2 * time.Minute).
+			WithPolling(time.Second * 5).
+			Should(BeNil())
+	})
+})
--- a/tests/tests_suite_test.go
+++ b/tests/tests_suite_test.go
@@ -20,6 +20,7 @@ import (
 	"helm.sh/helm/v3/pkg/action"
 	"helm.sh/helm/v3/pkg/chart/loader"
 	corev1 "k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/clientcmd"
@@ -137,6 +138,14 @@ var _ = AfterSuite(func() {

 	fmt.Fprintln(GinkgoWriter, "k3s logs written to: "+logfile)

+	// dump k3k controller logs
+	readCloser, err = k3sContainer.Logs(context.Background())
+	Expect(err).To(Not(HaveOccurred()))
+	writeLogs("k3s.log", readCloser)
+
+	// dump k3k logs
+	writeK3kLogs()
+
 	testcontainers.CleanupContainer(GinkgoTB(), k3sContainer)
 })

@@ -150,3 +159,28 @@ func buildScheme() *runtime.Scheme {

 	return scheme
 }
+
+func writeK3kLogs() {
+	var err error
+	var podList v1.PodList
+
+	ctx := context.Background()
+	err = k8sClient.List(ctx, &podList, &client.ListOptions{Namespace: "k3k-system"})
+	Expect(err).To(Not(HaveOccurred()))
+
+	k3kPod := podList.Items[0]
+	req := k8s.CoreV1().Pods(k3kPod.Namespace).GetLogs(k3kPod.Name, &corev1.PodLogOptions{})
+	podLogs, err := req.Stream(ctx)
+	Expect(err).To(Not(HaveOccurred()))
+	writeLogs("k3k.log", podLogs)
+}
+
+func writeLogs(filename string, logs io.ReadCloser) {
+	logsStr, err := io.ReadAll(logs)
+	Expect(err).To(Not(HaveOccurred()))
+	defer logs.Close()
+	tempfile := path.Join(os.TempDir(), filename)
+	err = os.WriteFile(tempfile, []byte(logsStr), 0644)
+	Expect(err).To(Not(HaveOccurred()))
+	fmt.Fprintln(GinkgoWriter, "logs written to: "+filename)
+}