splitting tests (#461)

tests/cluster_certs_test.go

@@ -0,0 +1,113 @@
+package k3k_test
+
+import (
+	"context"
+	"os"
+	"strings"
+
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = When("a cluster with custom certificates is installed with individual cert secrets", Label("e2e"), func() {
+	var virtualCluster *VirtualCluster
+
+	BeforeEach(func() {
+		ctx := context.Background()
+
+		namespace := NewNamespace()
+
+		// create custom cert secret
+		customCertDir := "testdata/customcerts/"
+
+		certList := []string{
+			"server-ca",
+			"client-ca",
+			"request-header-ca",
+			"service",
+			"etcd-peer-ca",
+			"etcd-server-ca",
+		}
+
+		for _, certName := range certList {
+			var cert, key []byte
+			var err error
+			filePathPrefix := ""
+			certfile := certName
+			if strings.HasPrefix(certName, "etcd") {
+				filePathPrefix = "etcd/"
+				certfile = strings.TrimPrefix(certName, "etcd-")
+			}
+			if !strings.Contains(certName, "service") {
+				cert, err = os.ReadFile(customCertDir + filePathPrefix + certfile + ".crt")
+				Expect(err).To(Not(HaveOccurred()))
+			}
+			key, err = os.ReadFile(customCertDir + filePathPrefix + certfile + ".key")
+			Expect(err).To(Not(HaveOccurred()))
+
+			certSecret := caCertSecret(certName, namespace.Name, cert, key)
+			err = k8sClient.Create(ctx, certSecret)
+			Expect(err).To(Not(HaveOccurred()))
+		}
+
+		cluster := NewCluster(namespace.Name)
+
+		cluster.Spec.CustomCAs = v1alpha1.CustomCAs{
+			Enabled: true,
+			Sources: v1alpha1.CredentialSources{
+				ServerCA: v1alpha1.CredentialSource{
+					SecretName: "server-ca",
+				},
+				ClientCA: v1alpha1.CredentialSource{
+					SecretName: "client-ca",
+				},
+				ETCDServerCA: v1alpha1.CredentialSource{
+					SecretName: "etcd-server-ca",
+				},
+				ETCDPeerCA: v1alpha1.CredentialSource{
+					SecretName: "etcd-peer-ca",
+				},
+				RequestHeaderCA: v1alpha1.CredentialSource{
+					SecretName: "request-header-ca",
+				},
+				ServiceAccountToken: v1alpha1.CredentialSource{
+					SecretName: "service",
+				},
+			},
+		}
+
+		CreateCluster(cluster)
+
+		client, restConfig := NewVirtualK8sClientAndConfig(cluster)
+
+		virtualCluster = &VirtualCluster{
+			Cluster:    cluster,
+			RestConfig: restConfig,
+			Client:     client,
+		}
+	})
+
+	It("will load the custom certs in the server pod", func() {
+		ctx := context.Background()
+
+		labelSelector := "cluster=" + virtualCluster.Cluster.Name + ",role=server"
+		serverPods, err := k8s.CoreV1().Pods(virtualCluster.Cluster.Namespace).List(ctx, v1.ListOptions{LabelSelector: labelSelector})
+		Expect(err).To(Not(HaveOccurred()))
+
+		Expect(len(serverPods.Items)).To(Equal(1))
+		serverPod := serverPods.Items[0]
+
+		// check server-ca.crt
+		serverCACrtPath := "/var/lib/rancher/k3s/server/tls/server-ca.crt"
+		serverCACrt, err := readFileWithinPod(ctx, k8s, restcfg, serverPod.Name, serverPod.Namespace, serverCACrtPath)
+		Expect(err).To(Not(HaveOccurred()))
+
+		serverCACrtTestFile, err := os.ReadFile("testdata/customcerts/server-ca.crt")
+		Expect(err).To(Not(HaveOccurred()))
+		Expect(serverCACrt).To(Equal(serverCACrtTestFile))
+	})
+})

tests/cluster_persistence_test.go

@@ -0,0 +1,124 @@
+package k3k_test
+
+import (
+	"context"
+	"crypto/x509"
+	"errors"
+	"time"
+
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = When("an ephemeral cluster is installed", Label("e2e"), func() {
+	var virtualCluster *VirtualCluster
+
+	BeforeEach(func() {
+		virtualCluster = NewVirtualCluster()
+	})
+
+	AfterEach(func() {
+		DeleteNamespaces(virtualCluster.Cluster.Namespace)
+	})
+
+	It("can create a nginx pod", func() {
+		_, _ = virtualCluster.NewNginxPod("")
+	})
+
+	It("regenerates the bootstrap secret after a restart", func() {
+		ctx := context.Background()
+
+		_, err := virtualCluster.Client.ServerVersion()
+		Expect(err).To(Not(HaveOccurred()))
+
+		labelSelector := "cluster=" + virtualCluster.Cluster.Name + ",role=server"
+		serverPods, err := k8s.CoreV1().Pods(virtualCluster.Cluster.Namespace).List(ctx, v1.ListOptions{LabelSelector: labelSelector})
+		Expect(err).To(Not(HaveOccurred()))
+
+		Expect(len(serverPods.Items)).To(Equal(1))
+		serverPod := serverPods.Items[0]
+
+		GinkgoWriter.Printf("deleting pod %s/%s\n", serverPod.Namespace, serverPod.Name)
+
+		err = k8s.CoreV1().Pods(virtualCluster.Cluster.Namespace).Delete(ctx, serverPod.Name, v1.DeleteOptions{})
+		Expect(err).To(Not(HaveOccurred()))
+
+		By("Deleting server pod")
+
+		// check that the server pods restarted
+		Eventually(func() any {
+			serverPods, err = k8s.CoreV1().Pods(virtualCluster.Cluster.Namespace).List(ctx, v1.ListOptions{LabelSelector: labelSelector})
+			Expect(err).To(Not(HaveOccurred()))
+			Expect(len(serverPods.Items)).To(Equal(1))
+			return serverPods.Items[0].DeletionTimestamp
+		}).
+			WithTimeout(time.Minute).
+			WithPolling(time.Second * 5).
+			Should(BeNil())
+
+		By("Server pod up and running again")
+
+		By("Using old k8s client configuration should fail")
+
+		Eventually(func() bool {
+			_, err = virtualCluster.Client.DiscoveryClient.ServerVersion()
+			var unknownAuthorityErr x509.UnknownAuthorityError
+			return errors.As(err, &unknownAuthorityErr)
+		}).
+			WithTimeout(time.Minute * 2).
+			WithPolling(time.Second * 5).
+			Should(BeTrue())
+
+		By("Recover new config should succeed")
+
+		Eventually(func() error {
+			virtualCluster.Client, virtualCluster.RestConfig = NewVirtualK8sClientAndConfig(virtualCluster.Cluster)
+			_, err = virtualCluster.Client.DiscoveryClient.ServerVersion()
+			return err
+		}).
+			WithTimeout(time.Minute * 2).
+			WithPolling(time.Second * 5).
+			Should(BeNil())
+	})
+})
+
+var _ = When("a dynamic cluster is installed", Label("e2e"), func() {
+	var virtualCluster *VirtualCluster
+
+	BeforeEach(func() {
+		virtualCluster = NewVirtualClusterWithType(v1alpha1.DynamicPersistenceMode)
+	})
+
+	AfterEach(func() {
+		DeleteNamespaces(virtualCluster.Cluster.Namespace)
+	})
+
+	It("can create a nginx pod", func() {
+		_, _ = virtualCluster.NewNginxPod("")
+	})
+
+	It("uses the same bootstrap secret after a restart", func() {
+		ctx := context.Background()
+
+		_, err := virtualCluster.Client.ServerVersion()
+		Expect(err).To(Not(HaveOccurred()))
+
+		restartServerPod(ctx, virtualCluster)
+
+		By("Server pod up and running again")
+
+		By("Using old k8s client configuration should succeed")
+
+		Eventually(func() error {
+			_, err = virtualCluster.Client.DiscoveryClient.ServerVersion()
+			return err
+		}).
+			WithTimeout(2 * time.Minute).
+			WithPolling(time.Second * 5).
+			Should(BeNil())
+	})
+})

tests/cluster_test.go

@@ -1,277 +0,0 @@
-package k3k_test
-
-import (
-	"context"
-	"crypto/x509"
-	"errors"
-	"os"
-	"strings"
-	"time"
-
-	corev1 "k8s.io/api/core/v1"
-	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-
-	"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
-
-	. "github.com/onsi/ginkgo/v2"
-	. "github.com/onsi/gomega"
-)
-
-var _ = When("k3k is installed", Label("e2e"), func() {
-	It("is in Running status", func() {
-		// check that the controller is running
-		Eventually(func() bool {
-			opts := v1.ListOptions{LabelSelector: "app.kubernetes.io/name=k3k"}
-			podList, err := k8s.CoreV1().Pods("k3k-system").List(context.Background(), opts)
-
-			Expect(err).To(Not(HaveOccurred()))
-			Expect(podList.Items).To(Not(BeEmpty()))
-
-			var isRunning bool
-			for _, pod := range podList.Items {
-				if pod.Status.Phase == corev1.PodRunning {
-					isRunning = true
-					break
-				}
-			}
-
-			return isRunning
-		}).
-			WithTimeout(time.Second * 10).
-			WithPolling(time.Second).
-			Should(BeTrue())
-	})
-})
-
-var _ = When("a ephemeral cluster is installed", Label("e2e"), func() {
-	var virtualCluster *VirtualCluster
-
-	BeforeEach(func() {
-		virtualCluster = NewVirtualCluster()
-	})
-
-	AfterEach(func() {
-		DeleteNamespaces(virtualCluster.Cluster.Namespace)
-	})
-
-	It("can create a nginx pod", func() {
-		_, _ = virtualCluster.NewNginxPod("")
-	})
-
-	It("regenerates the bootstrap secret after a restart", func() {
-		ctx := context.Background()
-
-		_, err := virtualCluster.Client.ServerVersion()
-		Expect(err).To(Not(HaveOccurred()))
-
-		labelSelector := "cluster=" + virtualCluster.Cluster.Name + ",role=server"
-		serverPods, err := k8s.CoreV1().Pods(virtualCluster.Cluster.Namespace).List(ctx, v1.ListOptions{LabelSelector: labelSelector})
-		Expect(err).To(Not(HaveOccurred()))
-
-		Expect(len(serverPods.Items)).To(Equal(1))
-		serverPod := serverPods.Items[0]
-
-		GinkgoWriter.Printf("deleting pod %s/%s\n", serverPod.Namespace, serverPod.Name)
-
-		err = k8s.CoreV1().Pods(virtualCluster.Cluster.Namespace).Delete(ctx, serverPod.Name, v1.DeleteOptions{})
-		Expect(err).To(Not(HaveOccurred()))
-
-		By("Deleting server pod")
-
-		// check that the server pods restarted
-		Eventually(func() any {
-			serverPods, err = k8s.CoreV1().Pods(virtualCluster.Cluster.Namespace).List(ctx, v1.ListOptions{LabelSelector: labelSelector})
-			Expect(err).To(Not(HaveOccurred()))
-			Expect(len(serverPods.Items)).To(Equal(1))
-			return serverPods.Items[0].DeletionTimestamp
-		}).
-			WithTimeout(time.Minute).
-			WithPolling(time.Second * 5).
-			Should(BeNil())
-
-		By("Server pod up and running again")
-
-		By("Using old k8s client configuration should fail")
-
-		Eventually(func() bool {
-			_, err = virtualCluster.Client.DiscoveryClient.ServerVersion()
-			var unknownAuthorityErr x509.UnknownAuthorityError
-			return errors.As(err, &unknownAuthorityErr)
-		}).
-			WithTimeout(time.Minute * 2).
-			WithPolling(time.Second * 5).
-			Should(BeTrue())
-
-		By("Recover new config should succeed")
-
-		Eventually(func() error {
-			virtualCluster.Client, virtualCluster.RestConfig = NewVirtualK8sClientAndConfig(virtualCluster.Cluster)
-			_, err = virtualCluster.Client.DiscoveryClient.ServerVersion()
-			return err
-		}).
-			WithTimeout(time.Minute * 2).
-			WithPolling(time.Second * 5).
-			Should(BeNil())
-	})
-})
-
-var _ = When("a dynamic cluster is installed", func() {
-	var virtualCluster *VirtualCluster
-
-	BeforeEach(func() {
-		namespace := NewNamespace()
-		cluster := NewCluster(namespace.Name)
-		cluster.Spec.Persistence.Type = v1alpha1.DynamicPersistenceMode
-		CreateCluster(cluster)
-		client, restConfig := NewVirtualK8sClientAndConfig(cluster)
-
-		virtualCluster = &VirtualCluster{
-			Cluster:    cluster,
-			RestConfig: restConfig,
-			Client:     client,
-		}
-	})
-
-	AfterEach(func() {
-		DeleteNamespaces(virtualCluster.Cluster.Namespace)
-	})
-
-	It("can create a nginx pod", func() {
-		_, _ = virtualCluster.NewNginxPod("")
-	})
-
-	It("use the same bootstrap secret after a restart", func() {
-		ctx := context.Background()
-
-		_, err := virtualCluster.Client.ServerVersion()
-		Expect(err).To(Not(HaveOccurred()))
-
-		labelSelector := "cluster=" + virtualCluster.Cluster.Name + ",role=server"
-		serverPods, err := k8s.CoreV1().Pods(virtualCluster.Cluster.Namespace).List(ctx, v1.ListOptions{LabelSelector: labelSelector})
-		Expect(err).To(Not(HaveOccurred()))
-
-		Expect(len(serverPods.Items)).To(Equal(1))
-		serverPod := serverPods.Items[0]
-
-		GinkgoWriter.Printf("deleting pod %s/%s\n", serverPod.Namespace, serverPod.Name)
-
-		err = k8s.CoreV1().Pods(virtualCluster.Cluster.Namespace).Delete(ctx, serverPod.Name, v1.DeleteOptions{})
-		Expect(err).To(Not(HaveOccurred()))
-
-		By("Deleting server pod")
-
-		// check that the server pods restarted
-		Eventually(func() any {
-			serverPods, err = k8s.CoreV1().Pods(virtualCluster.Cluster.Namespace).List(ctx, v1.ListOptions{LabelSelector: labelSelector})
-			Expect(err).To(Not(HaveOccurred()))
-			Expect(len(serverPods.Items)).To(Equal(1))
-			return serverPods.Items[0].DeletionTimestamp
-		}).
-			WithTimeout(60 * time.Second).
-			WithPolling(time.Second * 5).
-			Should(BeNil())
-
-		By("Server pod up and running again")
-
-		By("Using old k8s client configuration should succeed")
-
-		Eventually(func() error {
-			_, err = virtualCluster.Client.DiscoveryClient.ServerVersion()
-			return err
-		}).
-			WithTimeout(2 * time.Minute).
-			WithPolling(time.Second * 5).
-			Should(BeNil())
-	})
-})
-
-var _ = When("a cluster with custom certificates is installed with individual cert secrets", Label("e2e"), func() {
-	ctx := context.Background()
-	var virtualCluster *VirtualCluster
-	BeforeEach(func() {
-		namespace := NewNamespace()
-		// create custom cert secret
-		customCertDir := "testdata/customcerts/"
-		certList := []string{
-			"server-ca",
-			"client-ca",
-			"request-header-ca",
-			"service",
-			"etcd-peer-ca",
-			"etcd-server-ca",
-		}
-		for _, certName := range certList {
-			var cert, key []byte
-			var err error
-			filePathPrefix := ""
-			certfile := certName
-			if strings.HasPrefix(certName, "etcd") {
-				filePathPrefix = "etcd/"
-				certfile = strings.TrimPrefix(certName, "etcd-")
-			}
-			if !strings.Contains(certName, "service") {
-				cert, err = os.ReadFile(customCertDir + filePathPrefix + certfile + ".crt")
-				Expect(err).To(Not(HaveOccurred()))
-			}
-			key, err = os.ReadFile(customCertDir + filePathPrefix + certfile + ".key")
-			Expect(err).To(Not(HaveOccurred()))
-
-			certSecret := caCertSecret(certName, namespace.Name, cert, key)
-			err = k8sClient.Create(ctx, certSecret)
-			Expect(err).To(Not(HaveOccurred()))
-		}
-
-		cluster := NewCluster(namespace.Name)
-		cluster.Spec.CustomCAs = v1alpha1.CustomCAs{
-			Enabled: true,
-			Sources: v1alpha1.CredentialSources{
-				ServerCA: v1alpha1.CredentialSource{
-					SecretName: "server-ca",
-				},
-				ClientCA: v1alpha1.CredentialSource{
-					SecretName: "client-ca",
-				},
-				ETCDServerCA: v1alpha1.CredentialSource{
-					SecretName: "etcd-server-ca",
-				},
-				ETCDPeerCA: v1alpha1.CredentialSource{
-					SecretName: "etcd-peer-ca",
-				},
-				RequestHeaderCA: v1alpha1.CredentialSource{
-					SecretName: "request-header-ca",
-				},
-				ServiceAccountToken: v1alpha1.CredentialSource{
-					SecretName: "service",
-				},
-			},
-		}
-		CreateCluster(cluster)
-		client, restConfig := NewVirtualK8sClientAndConfig(cluster)
-
-		virtualCluster = &VirtualCluster{
-			Cluster:    cluster,
-			RestConfig: restConfig,
-			Client:     client,
-		}
-	})
-	It("will load the custom certs in the server pod", func() {
-		_, _ = virtualCluster.NewNginxPod("")
-
-		labelSelector := "cluster=" + virtualCluster.Cluster.Name + ",role=server"
-		serverPods, err := k8s.CoreV1().Pods(virtualCluster.Cluster.Namespace).List(ctx, v1.ListOptions{LabelSelector: labelSelector})
-		Expect(err).To(Not(HaveOccurred()))
-
-		Expect(len(serverPods.Items)).To(Equal(1))
-		serverPod := serverPods.Items[0]
-
-		// check server-ca.crt
-		serverCACrtPath := "/var/lib/rancher/k3s/server/tls/server-ca.crt"
-		serverCACrt, err := readFileWithinPod(ctx, k8s, restcfg, serverPod.Name, serverPod.Namespace, serverCACrtPath)
-		Expect(err).To(Not(HaveOccurred()))
-
-		serverCACrtTestFile, err := os.ReadFile("testdata/customcerts/server-ca.crt")
-		Expect(err).To(Not(HaveOccurred()))
-
-		Expect(serverCACrt).To(Equal(serverCACrtTestFile))
-	})
-})

tests/common_test.go

@@ -34,13 +34,20 @@ type VirtualCluster struct {
 	Client     *kubernetes.Clientset
 }
 
-func NewVirtualCluster() *VirtualCluster {
+func NewVirtualCluster() *VirtualCluster { // By default, create an ephemeral cluster
+	return NewVirtualClusterWithType(v1alpha1.EphemeralPersistenceMode)
+}
+
+func NewVirtualClusterWithType(persistenceType v1alpha1.PersistenceMode) *VirtualCluster {
 	GinkgoHelper()
 
 	namespace := NewNamespace()
 
 	By(fmt.Sprintf("Creating new virtual cluster in namespace %s", namespace.Name))
+
 	cluster := NewCluster(namespace.Name)
+	cluster.Spec.Persistence.Type = persistenceType
+
 	CreateCluster(cluster)
 
 	client, restConfig := NewVirtualK8sClientAndConfig(cluster)
@@ -299,3 +306,29 @@ func (c *VirtualCluster) ExecCmd(pod *corev1.Pod, command string) (string, strin
 
 	return stdout.String(), stderr.String(), err
 }
+
+func restartServerPod(ctx context.Context, virtualCluster *VirtualCluster) {
+	GinkgoHelper()
+
+	labelSelector := "cluster=" + virtualCluster.Cluster.Name + ",role=server"
+	serverPods, err := k8s.CoreV1().Pods(virtualCluster.Cluster.Namespace).List(ctx, v1.ListOptions{LabelSelector: labelSelector})
+	Expect(err).To(Not(HaveOccurred()))
+
+	Expect(len(serverPods.Items)).To(Equal(1))
+	serverPod := serverPods.Items[0]
+
+	GinkgoWriter.Printf("deleting pod %s/%s\n", serverPod.Namespace, serverPod.Name)
+
+	err = k8s.CoreV1().Pods(virtualCluster.Cluster.Namespace).Delete(ctx, serverPod.Name, v1.DeleteOptions{})
+	Expect(err).To(Not(HaveOccurred()))
+
+	By("Deleting server pod")
+
+	// check that the server pods restarted
+	Eventually(func() any {
+		serverPods, err = k8s.CoreV1().Pods(virtualCluster.Cluster.Namespace).List(ctx, v1.ListOptions{LabelSelector: labelSelector})
+		Expect(err).To(Not(HaveOccurred()))
+		Expect(len(serverPods.Items)).To(Equal(1))
+		return serverPods.Items[0].DeletionTimestamp
+	}).WithTimeout(60 * time.Second).WithPolling(time.Second * 5).Should(BeNil())
+}

tests/installation_test.go

@@ -0,0 +1,35 @@
+package k3k_test
+
+import (
+	"context"
+	"time"
+
+	corev1 "k8s.io/api/core/v1"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = When("k3k is installed", Label("e2e"), func() {
+	It("is in Running status", func() {
+		// check that the controller is running
+		Eventually(func() bool {
+			opts := v1.ListOptions{LabelSelector: "app.kubernetes.io/name=k3k"}
+			podList, err := k8s.CoreV1().Pods("k3k-system").List(context.Background(), opts)
+
+			Expect(err).To(Not(HaveOccurred()))
+			Expect(podList.Items).To(Not(BeEmpty()))
+
+			for _, pod := range podList.Items {
+				if pod.Status.Phase == corev1.PodRunning {
+					return true
+				}
+			}
+			return false
+		}).
+			WithTimeout(time.Second * 10).
+			WithPolling(time.Second).
+			Should(BeTrue())
+	})
+})

tests/tests_suite_test.go

@@ -275,7 +275,7 @@ var _ = AfterSuite(func() {
 	goCoverDir := os.Getenv("GOCOVERDIR")
 	if goCoverDir == "" {
 		goCoverDir = path.Join(os.TempDir(), "covdata")
-		Expect(os.Mkdir(goCoverDir, 0o755)).To(Succeed())
+		Expect(os.MkdirAll(goCoverDir, 0o755)).To(Succeed())
 	}
 
 	dumpK3kCoverageData(ctx, goCoverDir)