diff --git a/e2e/suite_test.go b/e2e/suite_test.go index efdfd06..400819f 100644 --- a/e2e/suite_test.go +++ b/e2e/suite_test.go @@ -34,6 +34,11 @@ func TestAPIs(t *testing.T) { RunSpecs(t, "Controller Suite") } +var _ = AfterEach(func() { + PrintTenantControlPlaneInfo() + PrintKamajiLogs() +}) + var _ = BeforeSuite(func() { logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) diff --git a/e2e/tcp_migration_test.go b/e2e/tcp_migration_test.go new file mode 100644 index 0000000..b4168cd --- /dev/null +++ b/e2e/tcp_migration_test.go @@ -0,0 +1,121 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package e2e + +import ( + "context" + "fmt" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/rand" + "k8s.io/client-go/tools/clientcmd" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" + "github.com/clastix/kamaji/internal/utilities" +) + +var _ = Describe("When migrating a Tenant Control Plane to another datastore", func() { + var tcp *kamajiv1alpha1.TenantControlPlane + // Create a TenantControlPlane resource into the cluster + JustBeforeEach(func() { + // Fill TenantControlPlane object + tcp = &kamajiv1alpha1.TenantControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("migrating-%s-etcd", rand.String(5)), + Namespace: "default", + }, + Spec: kamajiv1alpha1.TenantControlPlaneSpec{ + DataStore: "etcd-bronze", + ControlPlane: kamajiv1alpha1.ControlPlane{ + Deployment: kamajiv1alpha1.DeploymentSpec{ + Replicas: 1, + }, + Service: kamajiv1alpha1.ServiceSpec{ + ServiceType: "NodePort", + }, + }, + NetworkProfile: kamajiv1alpha1.NetworkProfileSpec{ + Address: GetKindIPAddress(), + Port: int32(rand.Int63nRange(31000, 32000)), + }, + Kubernetes: kamajiv1alpha1.KubernetesSpec{ + Version: "v1.23.6", + Kubelet: kamajiv1alpha1.KubeletSpec{ + CGroupFS: "cgroupfs", + }, + }, + }, + } + Expect(k8sClient.Create(context.Background(), tcp)).NotTo(HaveOccurred()) + StatusMustEqualTo(tcp, kamajiv1alpha1.VersionReady) + }) + // Delete the TenantControlPlane resource after test is finished + JustAfterEach(func() { + Expect(k8sClient.Delete(context.Background(), tcp)).Should(Succeed()) + }) + // Check if TenantControlPlane resource has been created + It("Should contain all the migrated data", func() { + By("getting TCP rest.Config") + config, err := utilities.GetTenantKubeconfig(context.Background(), k8sClient, tcp) + Expect(err).ToNot(HaveOccurred()) + + b, err := utilities.EncodeToYaml(config) + Expect(err).ToNot(HaveOccurred()) + + clientCfg, err := clientcmd.NewClientConfigFromBytes(b) + Expect(err).ToNot(HaveOccurred()) + + restConfig, err := clientCfg.ClientConfig() + Expect(err).ToNot(HaveOccurred()) + + tcpClient, err := ctrlclient.New(restConfig, ctrlclient.Options{}) + Expect(err).ToNot(HaveOccurred()) + + ns := &corev1.Namespace{} + ns.SetName("kamaji-test") + Expect(tcpClient.Create(context.Background(), ns)).ToNot(HaveOccurred()) + + By("start migration to a new DataStore") + Eventually(func() error { + if err := k8sClient.Get(context.Background(), types.NamespacedName{Namespace: tcp.GetNamespace(), Name: tcp.GetName()}, tcp); err != nil { + return err + } + + tcp.Spec.DataStore = "etcd-silver" + + return k8sClient.Update(context.Background(), tcp) + }, time.Minute, time.Second).ShouldNot(HaveOccurred()) + + By("waiting for the migrating status") + StatusMustEqualTo(tcp, kamajiv1alpha1.VersionMigrating) + + By("ensuring changes are not allowed") + Consistently(func() error { + return tcpClient.Delete(context.Background(), ns) + }, 10*time.Second, time.Second).Should(HaveOccurred()) + + By("waiting for completion of migration") + StatusMustEqualTo(tcp, kamajiv1alpha1.VersionReady) + + By("checking the DataStore of the TCP") + Eventually(func() string { + if err := k8sClient.Get(context.Background(), types.NamespacedName{Name: tcp.GetName(), Namespace: tcp.GetNamespace()}, tcp); err != nil { + return "" + } + + return tcp.Status.Storage.DataStoreName + }, time.Minute, time.Second).Should(BeEquivalentTo("etcd-silver")) + + By("checking the presence of the previous Namespace") + Eventually(func() error { + return tcpClient.Get(context.Background(), types.NamespacedName{Name: ns.GetName()}, &corev1.Namespace{}) + }).ShouldNot(HaveOccurred()) + }) +}) diff --git a/e2e/tcp_mysql_ready_test.go b/e2e/tcp_mysql_ready_test.go new file mode 100644 index 0000000..678ca18 --- /dev/null +++ b/e2e/tcp_mysql_ready_test.go @@ -0,0 +1,53 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package e2e + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" +) + +var _ = Describe("Deploy a TenantControlPlane resource with the MySQL driver", func() { + // Fill TenantControlPlane object + tcp := &kamajiv1alpha1.TenantControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "mysql", + Namespace: "default", + }, + Spec: kamajiv1alpha1.TenantControlPlaneSpec{ + DataStore: "mysql-bronze", + ControlPlane: kamajiv1alpha1.ControlPlane{ + Deployment: kamajiv1alpha1.DeploymentSpec{ + Replicas: 1, + }, + Service: kamajiv1alpha1.ServiceSpec{ + ServiceType: "ClusterIP", + }, + }, + Kubernetes: kamajiv1alpha1.KubernetesSpec{ + Version: "v1.23.6", + Kubelet: kamajiv1alpha1.KubeletSpec{ + CGroupFS: "cgroupfs", + }, + }, + }, + } + // Create a TenantControlPlane resource into the cluster + JustBeforeEach(func() { + Expect(k8sClient.Create(context.Background(), tcp)).NotTo(HaveOccurred()) + }) + // Delete the TenantControlPlane resource after test is finished + JustAfterEach(func() { + Expect(k8sClient.Delete(context.Background(), tcp)).Should(Succeed()) + }) + // Check if TenantControlPlane resource has been created + It("Should be Ready", func() { + StatusMustEqualTo(tcp, kamajiv1alpha1.VersionReady) + }) +}) diff --git a/e2e/tcp_postgres_ready_test.go b/e2e/tcp_postgres_ready_test.go new file mode 100644 index 0000000..a41b94d --- /dev/null +++ b/e2e/tcp_postgres_ready_test.go @@ -0,0 +1,53 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package e2e + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" +) + +var _ = Describe("Deploy a TenantControlPlane resource with the PostgreSQL driver", func() { + // Fill TenantControlPlane object + tcp := &kamajiv1alpha1.TenantControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "postgresql", + Namespace: "default", + }, + Spec: kamajiv1alpha1.TenantControlPlaneSpec{ + DataStore: "postgresql-bronze", + ControlPlane: kamajiv1alpha1.ControlPlane{ + Deployment: kamajiv1alpha1.DeploymentSpec{ + Replicas: 1, + }, + Service: kamajiv1alpha1.ServiceSpec{ + ServiceType: "ClusterIP", + }, + }, + Kubernetes: kamajiv1alpha1.KubernetesSpec{ + Version: "v1.23.6", + Kubelet: kamajiv1alpha1.KubeletSpec{ + CGroupFS: "cgroupfs", + }, + }, + }, + } + // Create a TenantControlPlane resource into the cluster + JustBeforeEach(func() { + Expect(k8sClient.Create(context.Background(), tcp)).NotTo(HaveOccurred()) + }) + // Delete the TenantControlPlane resource after test is finished + JustAfterEach(func() { + Expect(k8sClient.Delete(context.Background(), tcp)).Should(Succeed()) + }) + // Check if TenantControlPlane resource has been created + It("Should be Ready", func() { + StatusMustEqualTo(tcp, kamajiv1alpha1.VersionReady) + }) +}) diff --git a/e2e/tenant_control_plane_ready_test.go b/e2e/tenant_control_plane_ready_test.go index f2a46c0..362b699 100644 --- a/e2e/tenant_control_plane_ready_test.go +++ b/e2e/tenant_control_plane_ready_test.go @@ -5,19 +5,17 @@ package e2e import ( "context" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" ) var _ = Describe("Deploy a TenantControlPlane resource", func() { // Fill TenantControlPlane object - tcp := kamajiv1alpha1.TenantControlPlane{ + tcp := &kamajiv1alpha1.TenantControlPlane{ ObjectMeta: metav1.ObjectMeta{ Name: "tcp-clusterip", Namespace: "default", @@ -50,33 +48,15 @@ var _ = Describe("Deploy a TenantControlPlane resource", func() { // Create a TenantControlPlane resource into the cluster JustBeforeEach(func() { - Expect(k8sClient.Create(context.Background(), &tcp)).NotTo(HaveOccurred()) + Expect(k8sClient.Create(context.Background(), tcp)).NotTo(HaveOccurred()) }) // Delete the TenantControlPlane resource after test is finished JustAfterEach(func() { - PrintTenantControlPlaneInfo(&tcp) - PrintKamajiLogs() - Expect(k8sClient.Delete(context.Background(), &tcp)).Should(Succeed()) + Expect(k8sClient.Delete(context.Background(), tcp)).Should(Succeed()) }) - // Check if TenantControlPlane resource has been created It("Should be Ready", func() { - Eventually(func() kamajiv1alpha1.KubernetesVersionStatus { - err := k8sClient.Get(context.Background(), types.NamespacedName{ - Name: tcp.GetName(), - Namespace: tcp.GetNamespace(), - }, &tcp) - if err != nil { - return "" - } - - // Check if Status field has been created on TenantControlPlane struct - if tcp.Status.Kubernetes.Version.Status == nil { - return "" - } - - return *tcp.Status.Kubernetes.Version.Status - }, 5*time.Minute, time.Second).Should(Equal(kamajiv1alpha1.VersionReady)) + StatusMustEqualTo(tcp, kamajiv1alpha1.VersionReady) }) }) diff --git a/e2e/utils_test.go b/e2e/utils_test.go index f53890d..05edd0f 100644 --- a/e2e/utils_test.go +++ b/e2e/utils_test.go @@ -9,6 +9,7 @@ import ( "fmt" "io" "os/exec" + "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -27,7 +28,16 @@ func GetKindIPAddress() string { return ep.Subsets[0].Addresses[0].IP } -func PrintTenantControlPlaneInfo(tcp *kamajiv1alpha1.TenantControlPlane) { +func PrintTenantControlPlaneInfo() { + tcpList := &kamajiv1alpha1.TenantControlPlaneList{} + Expect(k8sClient.List(context.Background(), tcpList)).ToNot(HaveOccurred()) + + if len(tcpList.Items) == 0 { + return + } + + tcp := tcpList.Items[0] + kubectlExec := func(args ...string) { cmd := exec.Command("kubectl") @@ -108,3 +118,21 @@ func PrintKamajiLogs() { _, _ = fmt.Fprintln(GinkgoWriter, "DEBUG: end of Kamaji Pod logs") } } + +func StatusMustEqualTo(tcp *kamajiv1alpha1.TenantControlPlane, status kamajiv1alpha1.KubernetesVersionStatus) { + Eventually(func() kamajiv1alpha1.KubernetesVersionStatus { + err := k8sClient.Get(context.Background(), types.NamespacedName{ + Name: tcp.GetName(), + Namespace: tcp.GetNamespace(), + }, tcp) + if err != nil { + return "" + } + // Check if Status field has been created on TenantControlPlane struct + if tcp.Status.Kubernetes.Version.Status == nil { + return "" + } + + return *tcp.Status.Kubernetes.Version.Status + }, 5*time.Minute, time.Second).Should(Equal(status)) +} diff --git a/e2e/worker_kubeadm_join_test.go b/e2e/worker_kubeadm_join_test.go index fb474db..15f705f 100644 --- a/e2e/worker_kubeadm_join_test.go +++ b/e2e/worker_kubeadm_join_test.go @@ -87,8 +87,6 @@ var _ = Describe("starting a kind worker with kubeadm", func() { }) JustAfterEach(func() { - PrintTenantControlPlaneInfo(&tcp) - PrintKamajiLogs() Expect(workerContainer.Terminate(ctx)).ToNot(HaveOccurred()) Expect(k8sClient.Delete(ctx, &tcp)).Should(Succeed()) Expect(os.Remove(kubeconfigFile.Name())).ToNot(HaveOccurred()) diff --git a/e2e/worker_tcp_change_port_test.go b/e2e/worker_tcp_change_port_test.go index 6b6f1d1..63617bc 100644 --- a/e2e/worker_tcp_change_port_test.go +++ b/e2e/worker_tcp_change_port_test.go @@ -70,8 +70,6 @@ var _ = Describe("validating kubeconfig", func() { }) JustAfterEach(func() { - PrintKamajiLogs() - PrintTenantControlPlaneInfo(tcp) Expect(k8sClient.Delete(ctx, tcp)).Should(Succeed()) Expect(os.Remove(kubeconfigFile.Name())).ToNot(HaveOccurred()) })