test(e2e): testing different datastores and migration

e2e/suite_test.go

@@ -34,6 +34,11 @@ func TestAPIs(t *testing.T) {
 	RunSpecs(t, "Controller Suite")
 }
 
+var _ = AfterEach(func() {
+	PrintTenantControlPlaneInfo()
+	PrintKamajiLogs()
+})
+
 var _ = BeforeSuite(func() {
 	logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true)))
 

e2e/tcp_migration_test.go

@@ -0,0 +1,121 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package e2e
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/rand"
+	"k8s.io/client-go/tools/clientcmd"
+	ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
+
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+	"github.com/clastix/kamaji/internal/utilities"
+)
+
+var _ = Describe("When migrating a Tenant Control Plane to another datastore", func() {
+	var tcp *kamajiv1alpha1.TenantControlPlane
+	// Create a TenantControlPlane resource into the cluster
+	JustBeforeEach(func() {
+		// Fill TenantControlPlane object
+		tcp = &kamajiv1alpha1.TenantControlPlane{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      fmt.Sprintf("migrating-%s-etcd", rand.String(5)),
+				Namespace: "default",
+			},
+			Spec: kamajiv1alpha1.TenantControlPlaneSpec{
+				DataStore: "etcd-bronze",
+				ControlPlane: kamajiv1alpha1.ControlPlane{
+					Deployment: kamajiv1alpha1.DeploymentSpec{
+						Replicas: 1,
+					},
+					Service: kamajiv1alpha1.ServiceSpec{
+						ServiceType: "NodePort",
+					},
+				},
+				NetworkProfile: kamajiv1alpha1.NetworkProfileSpec{
+					Address: GetKindIPAddress(),
+					Port:    int32(rand.Int63nRange(31000, 32000)),
+				},
+				Kubernetes: kamajiv1alpha1.KubernetesSpec{
+					Version: "v1.23.6",
+					Kubelet: kamajiv1alpha1.KubeletSpec{
+						CGroupFS: "cgroupfs",
+					},
+				},
+			},
+		}
+		Expect(k8sClient.Create(context.Background(), tcp)).NotTo(HaveOccurred())
+		StatusMustEqualTo(tcp, kamajiv1alpha1.VersionReady)
+	})
+	// Delete the TenantControlPlane resource after test is finished
+	JustAfterEach(func() {
+		Expect(k8sClient.Delete(context.Background(), tcp)).Should(Succeed())
+	})
+	// Check if TenantControlPlane resource has been created
+	It("Should contain all the migrated data", func() {
+		By("getting TCP rest.Config")
+		config, err := utilities.GetTenantKubeconfig(context.Background(), k8sClient, tcp)
+		Expect(err).ToNot(HaveOccurred())
+
+		b, err := utilities.EncodeToYaml(config)
+		Expect(err).ToNot(HaveOccurred())
+
+		clientCfg, err := clientcmd.NewClientConfigFromBytes(b)
+		Expect(err).ToNot(HaveOccurred())
+
+		restConfig, err := clientCfg.ClientConfig()
+		Expect(err).ToNot(HaveOccurred())
+
+		tcpClient, err := ctrlclient.New(restConfig, ctrlclient.Options{})
+		Expect(err).ToNot(HaveOccurred())
+
+		ns := &corev1.Namespace{}
+		ns.SetName("kamaji-test")
+		Expect(tcpClient.Create(context.Background(), ns)).ToNot(HaveOccurred())
+
+		By("start migration to a new DataStore")
+		Eventually(func() error {
+			if err := k8sClient.Get(context.Background(), types.NamespacedName{Namespace: tcp.GetNamespace(), Name: tcp.GetName()}, tcp); err != nil {
+				return err
+			}
+
+			tcp.Spec.DataStore = "etcd-silver"
+
+			return k8sClient.Update(context.Background(), tcp)
+		}, time.Minute, time.Second).ShouldNot(HaveOccurred())
+
+		By("waiting for the migrating status")
+		StatusMustEqualTo(tcp, kamajiv1alpha1.VersionMigrating)
+
+		By("ensuring changes are not allowed")
+		Consistently(func() error {
+			return tcpClient.Delete(context.Background(), ns)
+		}, 10*time.Second, time.Second).Should(HaveOccurred())
+
+		By("waiting for completion of migration")
+		StatusMustEqualTo(tcp, kamajiv1alpha1.VersionReady)
+
+		By("checking the DataStore of the TCP")
+		Eventually(func() string {
+			if err := k8sClient.Get(context.Background(), types.NamespacedName{Name: tcp.GetName(), Namespace: tcp.GetNamespace()}, tcp); err != nil {
+				return ""
+			}
+
+			return tcp.Status.Storage.DataStoreName
+		}, time.Minute, time.Second).Should(BeEquivalentTo("etcd-silver"))
+
+		By("checking the presence of the previous Namespace")
+		Eventually(func() error {
+			return tcpClient.Get(context.Background(), types.NamespacedName{Name: ns.GetName()}, &corev1.Namespace{})
+		}).ShouldNot(HaveOccurred())
+	})
+})

e2e/tcp_mysql_ready_test.go

@@ -0,0 +1,53 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package e2e
+
+import (
+	"context"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+)
+
+var _ = Describe("Deploy a TenantControlPlane resource with the MySQL driver", func() {
+	// Fill TenantControlPlane object
+	tcp := &kamajiv1alpha1.TenantControlPlane{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "mysql",
+			Namespace: "default",
+		},
+		Spec: kamajiv1alpha1.TenantControlPlaneSpec{
+			DataStore: "mysql-bronze",
+			ControlPlane: kamajiv1alpha1.ControlPlane{
+				Deployment: kamajiv1alpha1.DeploymentSpec{
+					Replicas: 1,
+				},
+				Service: kamajiv1alpha1.ServiceSpec{
+					ServiceType: "ClusterIP",
+				},
+			},
+			Kubernetes: kamajiv1alpha1.KubernetesSpec{
+				Version: "v1.23.6",
+				Kubelet: kamajiv1alpha1.KubeletSpec{
+					CGroupFS: "cgroupfs",
+				},
+			},
+		},
+	}
+	// Create a TenantControlPlane resource into the cluster
+	JustBeforeEach(func() {
+		Expect(k8sClient.Create(context.Background(), tcp)).NotTo(HaveOccurred())
+	})
+	// Delete the TenantControlPlane resource after test is finished
+	JustAfterEach(func() {
+		Expect(k8sClient.Delete(context.Background(), tcp)).Should(Succeed())
+	})
+	// Check if TenantControlPlane resource has been created
+	It("Should be Ready", func() {
+		StatusMustEqualTo(tcp, kamajiv1alpha1.VersionReady)
+	})
+})

e2e/tcp_postgres_ready_test.go

@@ -0,0 +1,53 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package e2e
+
+import (
+	"context"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+)
+
+var _ = Describe("Deploy a TenantControlPlane resource with the PostgreSQL driver", func() {
+	// Fill TenantControlPlane object
+	tcp := &kamajiv1alpha1.TenantControlPlane{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "postgresql",
+			Namespace: "default",
+		},
+		Spec: kamajiv1alpha1.TenantControlPlaneSpec{
+			DataStore: "postgresql-bronze",
+			ControlPlane: kamajiv1alpha1.ControlPlane{
+				Deployment: kamajiv1alpha1.DeploymentSpec{
+					Replicas: 1,
+				},
+				Service: kamajiv1alpha1.ServiceSpec{
+					ServiceType: "ClusterIP",
+				},
+			},
+			Kubernetes: kamajiv1alpha1.KubernetesSpec{
+				Version: "v1.23.6",
+				Kubelet: kamajiv1alpha1.KubeletSpec{
+					CGroupFS: "cgroupfs",
+				},
+			},
+		},
+	}
+	// Create a TenantControlPlane resource into the cluster
+	JustBeforeEach(func() {
+		Expect(k8sClient.Create(context.Background(), tcp)).NotTo(HaveOccurred())
+	})
+	// Delete the TenantControlPlane resource after test is finished
+	JustAfterEach(func() {
+		Expect(k8sClient.Delete(context.Background(), tcp)).Should(Succeed())
+	})
+	// Check if TenantControlPlane resource has been created
+	It("Should be Ready", func() {
+		StatusMustEqualTo(tcp, kamajiv1alpha1.VersionReady)
+	})
+})

e2e/tenant_control_plane_ready_test.go

@@ -5,19 +5,17 @@ package e2e
 
 import (
 	"context"
-	"time"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/types"
 
 	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
 )
 
 var _ = Describe("Deploy a TenantControlPlane resource", func() {
 	// Fill TenantControlPlane object
-	tcp := kamajiv1alpha1.TenantControlPlane{
+	tcp := &kamajiv1alpha1.TenantControlPlane{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      "tcp-clusterip",
 			Namespace: "default",
@@ -50,33 +48,15 @@ var _ = Describe("Deploy a TenantControlPlane resource", func() {
 
 	// Create a TenantControlPlane resource into the cluster
 	JustBeforeEach(func() {
-		Expect(k8sClient.Create(context.Background(), &tcp)).NotTo(HaveOccurred())
+		Expect(k8sClient.Create(context.Background(), tcp)).NotTo(HaveOccurred())
 	})
 
 	// Delete the TenantControlPlane resource after test is finished
 	JustAfterEach(func() {
-		PrintTenantControlPlaneInfo(&tcp)
-		PrintKamajiLogs()
-		Expect(k8sClient.Delete(context.Background(), &tcp)).Should(Succeed())
+		Expect(k8sClient.Delete(context.Background(), tcp)).Should(Succeed())
 	})
-
 	// Check if TenantControlPlane resource has been created
 	It("Should be Ready", func() {
-		Eventually(func() kamajiv1alpha1.KubernetesVersionStatus {
-			err := k8sClient.Get(context.Background(), types.NamespacedName{
-				Name:      tcp.GetName(),
-				Namespace: tcp.GetNamespace(),
-			}, &tcp)
-			if err != nil {
-				return ""
-			}
-
-			// Check if Status field has been created on TenantControlPlane struct
-			if tcp.Status.Kubernetes.Version.Status == nil {
-				return ""
-			}
-
-			return *tcp.Status.Kubernetes.Version.Status
-		}, 5*time.Minute, time.Second).Should(Equal(kamajiv1alpha1.VersionReady))
+		StatusMustEqualTo(tcp, kamajiv1alpha1.VersionReady)
 	})
 })

e2e/utils_test.go

@@ -9,6 +9,7 @@ import (
 	"fmt"
 	"io"
 	"os/exec"
+	"time"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -27,7 +28,16 @@ func GetKindIPAddress() string {
 	return ep.Subsets[0].Addresses[0].IP
 }
 
-func PrintTenantControlPlaneInfo(tcp *kamajiv1alpha1.TenantControlPlane) {
+func PrintTenantControlPlaneInfo() {
+	tcpList := &kamajiv1alpha1.TenantControlPlaneList{}
+	Expect(k8sClient.List(context.Background(), tcpList)).ToNot(HaveOccurred())
+
+	if len(tcpList.Items) == 0 {
+		return
+	}
+
+	tcp := tcpList.Items[0]
+
 	kubectlExec := func(args ...string) {
 		cmd := exec.Command("kubectl")
 
@@ -108,3 +118,21 @@ func PrintKamajiLogs() {
 		_, _ = fmt.Fprintln(GinkgoWriter, "DEBUG: end of Kamaji Pod logs")
 	}
 }
+
+func StatusMustEqualTo(tcp *kamajiv1alpha1.TenantControlPlane, status kamajiv1alpha1.KubernetesVersionStatus) {
+	Eventually(func() kamajiv1alpha1.KubernetesVersionStatus {
+		err := k8sClient.Get(context.Background(), types.NamespacedName{
+			Name:      tcp.GetName(),
+			Namespace: tcp.GetNamespace(),
+		}, tcp)
+		if err != nil {
+			return ""
+		}
+		// Check if Status field has been created on TenantControlPlane struct
+		if tcp.Status.Kubernetes.Version.Status == nil {
+			return ""
+		}
+
+		return *tcp.Status.Kubernetes.Version.Status
+	}, 5*time.Minute, time.Second).Should(Equal(status))
+}

e2e/worker_kubeadm_join_test.go

@@ -87,8 +87,6 @@ var _ = Describe("starting a kind worker with kubeadm", func() {
 	})
 
 	JustAfterEach(func() {
-		PrintTenantControlPlaneInfo(&tcp)
-		PrintKamajiLogs()
 		Expect(workerContainer.Terminate(ctx)).ToNot(HaveOccurred())
 		Expect(k8sClient.Delete(ctx, &tcp)).Should(Succeed())
 		Expect(os.Remove(kubeconfigFile.Name())).ToNot(HaveOccurred())

e2e/worker_tcp_change_port_test.go

@@ -70,8 +70,6 @@ var _ = Describe("validating kubeconfig", func() {
 	})
 
 	JustAfterEach(func() {
-		PrintKamajiLogs()
-		PrintTenantControlPlaneInfo(tcp)
 		Expect(k8sClient.Delete(ctx, tcp)).Should(Succeed())
 		Expect(os.Remove(kubeconfigFile.Name())).ToNot(HaveOccurred())
 	})