diff --git a/tests/cluster_certs_test.go b/tests/cluster_certs_test.go new file mode 100644 index 00000000..86c325b8 --- /dev/null +++ b/tests/cluster_certs_test.go @@ -0,0 +1,113 @@ +package k3k_test + +import ( + "context" + "os" + "strings" + + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = When("a cluster with custom certificates is installed with individual cert secrets", Label("e2e"), func() { + var virtualCluster *VirtualCluster + + BeforeEach(func() { + ctx := context.Background() + + namespace := NewNamespace() + + // create custom cert secret + customCertDir := "testdata/customcerts/" + + certList := []string{ + "server-ca", + "client-ca", + "request-header-ca", + "service", + "etcd-peer-ca", + "etcd-server-ca", + } + + for _, certName := range certList { + var cert, key []byte + var err error + filePathPrefix := "" + certfile := certName + if strings.HasPrefix(certName, "etcd") { + filePathPrefix = "etcd/" + certfile = strings.TrimPrefix(certName, "etcd-") + } + if !strings.Contains(certName, "service") { + cert, err = os.ReadFile(customCertDir + filePathPrefix + certfile + ".crt") + Expect(err).To(Not(HaveOccurred())) + } + key, err = os.ReadFile(customCertDir + filePathPrefix + certfile + ".key") + Expect(err).To(Not(HaveOccurred())) + + certSecret := caCertSecret(certName, namespace.Name, cert, key) + err = k8sClient.Create(ctx, certSecret) + Expect(err).To(Not(HaveOccurred())) + } + + cluster := NewCluster(namespace.Name) + + cluster.Spec.CustomCAs = v1alpha1.CustomCAs{ + Enabled: true, + Sources: v1alpha1.CredentialSources{ + ServerCA: v1alpha1.CredentialSource{ + SecretName: "server-ca", + }, + ClientCA: v1alpha1.CredentialSource{ + SecretName: "client-ca", + }, + ETCDServerCA: v1alpha1.CredentialSource{ + SecretName: "etcd-server-ca", + }, + ETCDPeerCA: v1alpha1.CredentialSource{ + SecretName: "etcd-peer-ca", + }, + RequestHeaderCA: v1alpha1.CredentialSource{ + SecretName: "request-header-ca", + }, + ServiceAccountToken: v1alpha1.CredentialSource{ + SecretName: "service", + }, + }, + } + + CreateCluster(cluster) + + client, restConfig := NewVirtualK8sClientAndConfig(cluster) + + virtualCluster = &VirtualCluster{ + Cluster: cluster, + RestConfig: restConfig, + Client: client, + } + }) + + It("will load the custom certs in the server pod", func() { + ctx := context.Background() + + labelSelector := "cluster=" + virtualCluster.Cluster.Name + ",role=server" + serverPods, err := k8s.CoreV1().Pods(virtualCluster.Cluster.Namespace).List(ctx, v1.ListOptions{LabelSelector: labelSelector}) + Expect(err).To(Not(HaveOccurred())) + + Expect(len(serverPods.Items)).To(Equal(1)) + serverPod := serverPods.Items[0] + + // check server-ca.crt + serverCACrtPath := "/var/lib/rancher/k3s/server/tls/server-ca.crt" + serverCACrt, err := readFileWithinPod(ctx, k8s, restcfg, serverPod.Name, serverPod.Namespace, serverCACrtPath) + Expect(err).To(Not(HaveOccurred())) + + serverCACrtTestFile, err := os.ReadFile("testdata/customcerts/server-ca.crt") + Expect(err).To(Not(HaveOccurred())) + Expect(serverCACrt).To(Equal(serverCACrtTestFile)) + }) +}) diff --git a/tests/cluster_persistence_test.go b/tests/cluster_persistence_test.go new file mode 100644 index 00000000..4a2cb50f --- /dev/null +++ b/tests/cluster_persistence_test.go @@ -0,0 +1,124 @@ +package k3k_test + +import ( + "context" + "crypto/x509" + "errors" + "time" + + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = When("an ephemeral cluster is installed", Label("e2e"), func() { + var virtualCluster *VirtualCluster + + BeforeEach(func() { + virtualCluster = NewVirtualCluster() + }) + + AfterEach(func() { + DeleteNamespaces(virtualCluster.Cluster.Namespace) + }) + + It("can create a nginx pod", func() { + _, _ = virtualCluster.NewNginxPod("") + }) + + It("regenerates the bootstrap secret after a restart", func() { + ctx := context.Background() + + _, err := virtualCluster.Client.ServerVersion() + Expect(err).To(Not(HaveOccurred())) + + labelSelector := "cluster=" + virtualCluster.Cluster.Name + ",role=server" + serverPods, err := k8s.CoreV1().Pods(virtualCluster.Cluster.Namespace).List(ctx, v1.ListOptions{LabelSelector: labelSelector}) + Expect(err).To(Not(HaveOccurred())) + + Expect(len(serverPods.Items)).To(Equal(1)) + serverPod := serverPods.Items[0] + + GinkgoWriter.Printf("deleting pod %s/%s\n", serverPod.Namespace, serverPod.Name) + + err = k8s.CoreV1().Pods(virtualCluster.Cluster.Namespace).Delete(ctx, serverPod.Name, v1.DeleteOptions{}) + Expect(err).To(Not(HaveOccurred())) + + By("Deleting server pod") + + // check that the server pods restarted + Eventually(func() any { + serverPods, err = k8s.CoreV1().Pods(virtualCluster.Cluster.Namespace).List(ctx, v1.ListOptions{LabelSelector: labelSelector}) + Expect(err).To(Not(HaveOccurred())) + Expect(len(serverPods.Items)).To(Equal(1)) + return serverPods.Items[0].DeletionTimestamp + }). + WithTimeout(time.Minute). + WithPolling(time.Second * 5). + Should(BeNil()) + + By("Server pod up and running again") + + By("Using old k8s client configuration should fail") + + Eventually(func() bool { + _, err = virtualCluster.Client.DiscoveryClient.ServerVersion() + var unknownAuthorityErr x509.UnknownAuthorityError + return errors.As(err, &unknownAuthorityErr) + }). + WithTimeout(time.Minute * 2). + WithPolling(time.Second * 5). + Should(BeTrue()) + + By("Recover new config should succeed") + + Eventually(func() error { + virtualCluster.Client, virtualCluster.RestConfig = NewVirtualK8sClientAndConfig(virtualCluster.Cluster) + _, err = virtualCluster.Client.DiscoveryClient.ServerVersion() + return err + }). + WithTimeout(time.Minute * 2). + WithPolling(time.Second * 5). + Should(BeNil()) + }) +}) + +var _ = When("a dynamic cluster is installed", Label("e2e"), func() { + var virtualCluster *VirtualCluster + + BeforeEach(func() { + virtualCluster = NewVirtualClusterWithType(v1alpha1.DynamicPersistenceMode) + }) + + AfterEach(func() { + DeleteNamespaces(virtualCluster.Cluster.Namespace) + }) + + It("can create a nginx pod", func() { + _, _ = virtualCluster.NewNginxPod("") + }) + + It("uses the same bootstrap secret after a restart", func() { + ctx := context.Background() + + _, err := virtualCluster.Client.ServerVersion() + Expect(err).To(Not(HaveOccurred())) + + restartServerPod(ctx, virtualCluster) + + By("Server pod up and running again") + + By("Using old k8s client configuration should succeed") + + Eventually(func() error { + _, err = virtualCluster.Client.DiscoveryClient.ServerVersion() + return err + }). + WithTimeout(2 * time.Minute). + WithPolling(time.Second * 5). + Should(BeNil()) + }) +}) diff --git a/tests/cluster_test.go b/tests/cluster_test.go deleted file mode 100644 index 44136964..00000000 --- a/tests/cluster_test.go +++ /dev/null @@ -1,277 +0,0 @@ -package k3k_test - -import ( - "context" - "crypto/x509" - "errors" - "os" - "strings" - "time" - - corev1 "k8s.io/api/core/v1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -var _ = When("k3k is installed", Label("e2e"), func() { - It("is in Running status", func() { - // check that the controller is running - Eventually(func() bool { - opts := v1.ListOptions{LabelSelector: "app.kubernetes.io/name=k3k"} - podList, err := k8s.CoreV1().Pods("k3k-system").List(context.Background(), opts) - - Expect(err).To(Not(HaveOccurred())) - Expect(podList.Items).To(Not(BeEmpty())) - - var isRunning bool - for _, pod := range podList.Items { - if pod.Status.Phase == corev1.PodRunning { - isRunning = true - break - } - } - - return isRunning - }). - WithTimeout(time.Second * 10). - WithPolling(time.Second). - Should(BeTrue()) - }) -}) - -var _ = When("a ephemeral cluster is installed", Label("e2e"), func() { - var virtualCluster *VirtualCluster - - BeforeEach(func() { - virtualCluster = NewVirtualCluster() - }) - - AfterEach(func() { - DeleteNamespaces(virtualCluster.Cluster.Namespace) - }) - - It("can create a nginx pod", func() { - _, _ = virtualCluster.NewNginxPod("") - }) - - It("regenerates the bootstrap secret after a restart", func() { - ctx := context.Background() - - _, err := virtualCluster.Client.ServerVersion() - Expect(err).To(Not(HaveOccurred())) - - labelSelector := "cluster=" + virtualCluster.Cluster.Name + ",role=server" - serverPods, err := k8s.CoreV1().Pods(virtualCluster.Cluster.Namespace).List(ctx, v1.ListOptions{LabelSelector: labelSelector}) - Expect(err).To(Not(HaveOccurred())) - - Expect(len(serverPods.Items)).To(Equal(1)) - serverPod := serverPods.Items[0] - - GinkgoWriter.Printf("deleting pod %s/%s\n", serverPod.Namespace, serverPod.Name) - - err = k8s.CoreV1().Pods(virtualCluster.Cluster.Namespace).Delete(ctx, serverPod.Name, v1.DeleteOptions{}) - Expect(err).To(Not(HaveOccurred())) - - By("Deleting server pod") - - // check that the server pods restarted - Eventually(func() any { - serverPods, err = k8s.CoreV1().Pods(virtualCluster.Cluster.Namespace).List(ctx, v1.ListOptions{LabelSelector: labelSelector}) - Expect(err).To(Not(HaveOccurred())) - Expect(len(serverPods.Items)).To(Equal(1)) - return serverPods.Items[0].DeletionTimestamp - }). - WithTimeout(time.Minute). - WithPolling(time.Second * 5). - Should(BeNil()) - - By("Server pod up and running again") - - By("Using old k8s client configuration should fail") - - Eventually(func() bool { - _, err = virtualCluster.Client.DiscoveryClient.ServerVersion() - var unknownAuthorityErr x509.UnknownAuthorityError - return errors.As(err, &unknownAuthorityErr) - }). - WithTimeout(time.Minute * 2). - WithPolling(time.Second * 5). - Should(BeTrue()) - - By("Recover new config should succeed") - - Eventually(func() error { - virtualCluster.Client, virtualCluster.RestConfig = NewVirtualK8sClientAndConfig(virtualCluster.Cluster) - _, err = virtualCluster.Client.DiscoveryClient.ServerVersion() - return err - }). - WithTimeout(time.Minute * 2). - WithPolling(time.Second * 5). - Should(BeNil()) - }) -}) - -var _ = When("a dynamic cluster is installed", func() { - var virtualCluster *VirtualCluster - - BeforeEach(func() { - namespace := NewNamespace() - cluster := NewCluster(namespace.Name) - cluster.Spec.Persistence.Type = v1alpha1.DynamicPersistenceMode - CreateCluster(cluster) - client, restConfig := NewVirtualK8sClientAndConfig(cluster) - - virtualCluster = &VirtualCluster{ - Cluster: cluster, - RestConfig: restConfig, - Client: client, - } - }) - - AfterEach(func() { - DeleteNamespaces(virtualCluster.Cluster.Namespace) - }) - - It("can create a nginx pod", func() { - _, _ = virtualCluster.NewNginxPod("") - }) - - It("use the same bootstrap secret after a restart", func() { - ctx := context.Background() - - _, err := virtualCluster.Client.ServerVersion() - Expect(err).To(Not(HaveOccurred())) - - labelSelector := "cluster=" + virtualCluster.Cluster.Name + ",role=server" - serverPods, err := k8s.CoreV1().Pods(virtualCluster.Cluster.Namespace).List(ctx, v1.ListOptions{LabelSelector: labelSelector}) - Expect(err).To(Not(HaveOccurred())) - - Expect(len(serverPods.Items)).To(Equal(1)) - serverPod := serverPods.Items[0] - - GinkgoWriter.Printf("deleting pod %s/%s\n", serverPod.Namespace, serverPod.Name) - - err = k8s.CoreV1().Pods(virtualCluster.Cluster.Namespace).Delete(ctx, serverPod.Name, v1.DeleteOptions{}) - Expect(err).To(Not(HaveOccurred())) - - By("Deleting server pod") - - // check that the server pods restarted - Eventually(func() any { - serverPods, err = k8s.CoreV1().Pods(virtualCluster.Cluster.Namespace).List(ctx, v1.ListOptions{LabelSelector: labelSelector}) - Expect(err).To(Not(HaveOccurred())) - Expect(len(serverPods.Items)).To(Equal(1)) - return serverPods.Items[0].DeletionTimestamp - }). - WithTimeout(60 * time.Second). - WithPolling(time.Second * 5). - Should(BeNil()) - - By("Server pod up and running again") - - By("Using old k8s client configuration should succeed") - - Eventually(func() error { - _, err = virtualCluster.Client.DiscoveryClient.ServerVersion() - return err - }). - WithTimeout(2 * time.Minute). - WithPolling(time.Second * 5). - Should(BeNil()) - }) -}) - -var _ = When("a cluster with custom certificates is installed with individual cert secrets", Label("e2e"), func() { - ctx := context.Background() - var virtualCluster *VirtualCluster - BeforeEach(func() { - namespace := NewNamespace() - // create custom cert secret - customCertDir := "testdata/customcerts/" - certList := []string{ - "server-ca", - "client-ca", - "request-header-ca", - "service", - "etcd-peer-ca", - "etcd-server-ca", - } - for _, certName := range certList { - var cert, key []byte - var err error - filePathPrefix := "" - certfile := certName - if strings.HasPrefix(certName, "etcd") { - filePathPrefix = "etcd/" - certfile = strings.TrimPrefix(certName, "etcd-") - } - if !strings.Contains(certName, "service") { - cert, err = os.ReadFile(customCertDir + filePathPrefix + certfile + ".crt") - Expect(err).To(Not(HaveOccurred())) - } - key, err = os.ReadFile(customCertDir + filePathPrefix + certfile + ".key") - Expect(err).To(Not(HaveOccurred())) - - certSecret := caCertSecret(certName, namespace.Name, cert, key) - err = k8sClient.Create(ctx, certSecret) - Expect(err).To(Not(HaveOccurred())) - } - - cluster := NewCluster(namespace.Name) - cluster.Spec.CustomCAs = v1alpha1.CustomCAs{ - Enabled: true, - Sources: v1alpha1.CredentialSources{ - ServerCA: v1alpha1.CredentialSource{ - SecretName: "server-ca", - }, - ClientCA: v1alpha1.CredentialSource{ - SecretName: "client-ca", - }, - ETCDServerCA: v1alpha1.CredentialSource{ - SecretName: "etcd-server-ca", - }, - ETCDPeerCA: v1alpha1.CredentialSource{ - SecretName: "etcd-peer-ca", - }, - RequestHeaderCA: v1alpha1.CredentialSource{ - SecretName: "request-header-ca", - }, - ServiceAccountToken: v1alpha1.CredentialSource{ - SecretName: "service", - }, - }, - } - CreateCluster(cluster) - client, restConfig := NewVirtualK8sClientAndConfig(cluster) - - virtualCluster = &VirtualCluster{ - Cluster: cluster, - RestConfig: restConfig, - Client: client, - } - }) - It("will load the custom certs in the server pod", func() { - _, _ = virtualCluster.NewNginxPod("") - - labelSelector := "cluster=" + virtualCluster.Cluster.Name + ",role=server" - serverPods, err := k8s.CoreV1().Pods(virtualCluster.Cluster.Namespace).List(ctx, v1.ListOptions{LabelSelector: labelSelector}) - Expect(err).To(Not(HaveOccurred())) - - Expect(len(serverPods.Items)).To(Equal(1)) - serverPod := serverPods.Items[0] - - // check server-ca.crt - serverCACrtPath := "/var/lib/rancher/k3s/server/tls/server-ca.crt" - serverCACrt, err := readFileWithinPod(ctx, k8s, restcfg, serverPod.Name, serverPod.Namespace, serverCACrtPath) - Expect(err).To(Not(HaveOccurred())) - - serverCACrtTestFile, err := os.ReadFile("testdata/customcerts/server-ca.crt") - Expect(err).To(Not(HaveOccurred())) - - Expect(serverCACrt).To(Equal(serverCACrtTestFile)) - }) -}) diff --git a/tests/common_test.go b/tests/common_test.go index 2900fcf5..c7f87271 100644 --- a/tests/common_test.go +++ b/tests/common_test.go @@ -34,13 +34,20 @@ type VirtualCluster struct { Client *kubernetes.Clientset } -func NewVirtualCluster() *VirtualCluster { +func NewVirtualCluster() *VirtualCluster { // By default, create an ephemeral cluster + return NewVirtualClusterWithType(v1alpha1.EphemeralPersistenceMode) +} + +func NewVirtualClusterWithType(persistenceType v1alpha1.PersistenceMode) *VirtualCluster { GinkgoHelper() namespace := NewNamespace() By(fmt.Sprintf("Creating new virtual cluster in namespace %s", namespace.Name)) + cluster := NewCluster(namespace.Name) + cluster.Spec.Persistence.Type = persistenceType + CreateCluster(cluster) client, restConfig := NewVirtualK8sClientAndConfig(cluster) @@ -299,3 +306,29 @@ func (c *VirtualCluster) ExecCmd(pod *corev1.Pod, command string) (string, strin return stdout.String(), stderr.String(), err } + +func restartServerPod(ctx context.Context, virtualCluster *VirtualCluster) { + GinkgoHelper() + + labelSelector := "cluster=" + virtualCluster.Cluster.Name + ",role=server" + serverPods, err := k8s.CoreV1().Pods(virtualCluster.Cluster.Namespace).List(ctx, v1.ListOptions{LabelSelector: labelSelector}) + Expect(err).To(Not(HaveOccurred())) + + Expect(len(serverPods.Items)).To(Equal(1)) + serverPod := serverPods.Items[0] + + GinkgoWriter.Printf("deleting pod %s/%s\n", serverPod.Namespace, serverPod.Name) + + err = k8s.CoreV1().Pods(virtualCluster.Cluster.Namespace).Delete(ctx, serverPod.Name, v1.DeleteOptions{}) + Expect(err).To(Not(HaveOccurred())) + + By("Deleting server pod") + + // check that the server pods restarted + Eventually(func() any { + serverPods, err = k8s.CoreV1().Pods(virtualCluster.Cluster.Namespace).List(ctx, v1.ListOptions{LabelSelector: labelSelector}) + Expect(err).To(Not(HaveOccurred())) + Expect(len(serverPods.Items)).To(Equal(1)) + return serverPods.Items[0].DeletionTimestamp + }).WithTimeout(60 * time.Second).WithPolling(time.Second * 5).Should(BeNil()) +} diff --git a/tests/installation_test.go b/tests/installation_test.go new file mode 100644 index 00000000..446252e8 --- /dev/null +++ b/tests/installation_test.go @@ -0,0 +1,35 @@ +package k3k_test + +import ( + "context" + "time" + + corev1 "k8s.io/api/core/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = When("k3k is installed", Label("e2e"), func() { + It("is in Running status", func() { + // check that the controller is running + Eventually(func() bool { + opts := v1.ListOptions{LabelSelector: "app.kubernetes.io/name=k3k"} + podList, err := k8s.CoreV1().Pods("k3k-system").List(context.Background(), opts) + + Expect(err).To(Not(HaveOccurred())) + Expect(podList.Items).To(Not(BeEmpty())) + + for _, pod := range podList.Items { + if pod.Status.Phase == corev1.PodRunning { + return true + } + } + return false + }). + WithTimeout(time.Second * 10). + WithPolling(time.Second). + Should(BeTrue()) + }) +}) diff --git a/tests/tests_suite_test.go b/tests/tests_suite_test.go index 1370e8fe..4e71c0c1 100644 --- a/tests/tests_suite_test.go +++ b/tests/tests_suite_test.go @@ -275,7 +275,7 @@ var _ = AfterSuite(func() { goCoverDir := os.Getenv("GOCOVERDIR") if goCoverDir == "" { goCoverDir = path.Join(os.TempDir(), "covdata") - Expect(os.Mkdir(goCoverDir, 0o755)).To(Succeed()) + Expect(os.MkdirAll(goCoverDir, 0o755)).To(Succeed()) } dumpK3kCoverageData(ctx, goCoverDir)