From ba1648e56015a6c0fa3e7e21706a4a969c2ab158 Mon Sep 17 00:00:00 2001 From: Enrico Candino Date: Mon, 13 Apr 2026 12:50:38 +0200 Subject: [PATCH] Refactoring of tests moving common logic in `tests/framework` (#769) * first commit * refactor: update golangci configuration and reorder imports in namespace.go * refactor: update golangci configuration for linters and formatters * check restart for logs fetch * rverted a bit the structure * requested changes * WriteLogs rename --- .golangci.yml | 33 +-- pkg/controller/cluster/agent/shared.go | 1 + pkg/controller/cluster/agent/virtual.go | 1 + tests/cli/cli_test.go | 37 ++-- tests/cli/common_test.go | 55 ----- tests/cli/tests_suite_test.go | 202 +++--------------- tests/e2e/cluster_addons_test.go | 15 +- tests/e2e/cluster_app_test.go | 3 +- tests/e2e/cluster_certs_test.go | 5 +- tests/e2e/cluster_create_test.go | 6 +- tests/e2e/cluster_network_test.go | 4 +- tests/e2e/cluster_persistence_test.go | 9 +- tests/e2e/cluster_pod_test.go | 3 +- .../cluster_policy_sync_storageclass_test.go | 3 +- tests/e2e/cluster_registry_test.go | 5 +- tests/e2e/cluster_status_test.go | 5 +- tests/e2e/cluster_sync_storageclass_test.go | 3 +- tests/e2e/cluster_sync_test.go | 3 +- tests/e2e/cluster_update_test.go | 43 ++-- tests/e2e/common_test.go | 58 +---- tests/e2e/k8s_restclientgetter_test.go | 56 ----- tests/e2e/tests_suite_test.go | 198 +++-------------- tests/framework/client/config.go | 105 +++++++++ .../client/getter.go} | 13 +- tests/framework/client/scheme.go | 27 +++ tests/framework/container/helm.go | 105 +++++++++ tests/framework/container/k3s.go | 63 ++++++ tests/framework/env/env.go | 34 +++ tests/framework/k3k/namespace.go | 69 ++++++ tests/framework/log/log.go | 71 ++++++ .../integration/cluster/cluster_suite_test.go | 17 +- .../k3k-kubelet/syncer_suite_test.go | 16 +- tests/integration/policy/policy_suite_test.go | 23 +- 33 files changed, 649 insertions(+), 642 deletions(-) delete mode 100644 tests/cli/common_test.go delete mode 100644 tests/e2e/k8s_restclientgetter_test.go create mode 100644 tests/framework/client/config.go rename tests/{cli/k8s_restclientgetter_test.go => framework/client/getter.go} (67%) create mode 100644 tests/framework/client/scheme.go create mode 100644 tests/framework/container/helm.go create mode 100644 tests/framework/container/k3s.go create mode 100644 tests/framework/env/env.go create mode 100644 tests/framework/k3k/namespace.go create mode 100644 tests/framework/log/log.go diff --git a/.golangci.yml b/.golangci.yml index 73c776b4..f187f7ac 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -2,26 +2,33 @@ version: "2" linters: enable: - - misspell - - wsl_v5 + - misspell + - wsl_v5 + exclusions: + rules: + # Allow dot imports for Ginkgo/Gomega in test framework + - path: tests/framework/ + text: "dot imports" + linters: + - staticcheck formatters: enable: - - gci - - gofmt - - gofumpt + - gci + - gofmt + - gofumpt settings: gci: # The default order is `standard > default > custom > blank > dot > alias > localmodule`. custom-order: true sections: - - standard - - default - - alias - - localmodule - - dot - - blank + - standard + - default + - alias + - localmodule + - dot + - blank gofmt: rewrite-rules: - - pattern: 'interface{}' - replacement: 'any' + - pattern: 'interface{}' + replacement: 'any' diff --git a/pkg/controller/cluster/agent/shared.go b/pkg/controller/cluster/agent/shared.go index 9d07c4a8..c8a7b5c4 100644 --- a/pkg/controller/cluster/agent/shared.go +++ b/pkg/controller/cluster/agent/shared.go @@ -253,6 +253,7 @@ func (s *SharedAgent) podSpec(ctx context.Context) v1.PodSpec { } podSpec.RuntimeClassName = runtimeClassName + return podSpec } diff --git a/pkg/controller/cluster/agent/virtual.go b/pkg/controller/cluster/agent/virtual.go index 53267055..22569cc3 100644 --- a/pkg/controller/cluster/agent/virtual.go +++ b/pkg/controller/cluster/agent/virtual.go @@ -137,6 +137,7 @@ func (v *VirtualAgent) deployment(ctx context.Context) error { func (v *VirtualAgent) podSpec(ctx context.Context, image, name string) v1.PodSpec { log := ctrl.LoggerFrom(ctx) + var limit v1.ResourceList args := v.cluster.Spec.AgentArgs diff --git a/tests/cli/cli_test.go b/tests/cli/cli_test.go index dd3d2ee8..c30c4889 100644 --- a/tests/cli/cli_test.go +++ b/tests/cli/cli_test.go @@ -13,6 +13,7 @@ import ( "github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1" "github.com/rancher/k3k/pkg/controller/policy" + fwk3k "github.com/rancher/k3k/tests/framework/k3k" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -54,11 +55,11 @@ var _ = When("using the k3kcli", Label("cli"), func() { ) clusterName := "cluster-" + rand.String(5) - namespace := NewNamespace() + namespace := fwk3k.CreateNamespace(k8s) clusterNamespace := namespace.Name DeferCleanup(func() { - DeleteNamespaces(namespace.Name) + fwk3k.DeleteNamespaces(k8s, namespace.Name) }) _, stderr, err = K3kcli("cluster", "create", "--namespace", clusterNamespace, clusterName) @@ -93,11 +94,11 @@ var _ = When("using the k3kcli", Label("cli"), func() { ) clusterName := "cluster-" + rand.String(5) - namespace := NewNamespace() + namespace := fwk3k.CreateNamespace(k8s) clusterNamespace := namespace.Name DeferCleanup(func() { - DeleteNamespaces(clusterNamespace) + fwk3k.DeleteNamespaces(k8s, clusterNamespace) }) _, stderr, err = K3kcli("cluster", "create", "--version", k3sVersion, clusterName) @@ -142,11 +143,11 @@ var _ = When("using the k3kcli", Label("cli"), func() { err error ) - namespace := NewNamespace() + namespace := fwk3k.CreateNamespace(k8s) namespaceName := namespace.Name DeferCleanup(func() { - DeleteNamespaces(namespaceName) + fwk3k.DeleteNamespaces(k8s, namespaceName) }) By("Creating a policy and binding to a namespace") @@ -213,11 +214,11 @@ var _ = When("using the k3kcli", Label("cli"), func() { clusterName := "cluster-" + rand.String(5) - namespace := NewNamespace() + namespace := fwk3k.CreateNamespace(k8s) clusterNamespace := namespace.Name DeferCleanup(func() { - DeleteNamespaces(clusterNamespace) + fwk3k.DeleteNamespaces(k8s, clusterNamespace) }) // Create the cluster first @@ -247,11 +248,11 @@ var _ = When("using the k3kcli", Label("cli"), func() { clusterName := "cluster-" + rand.String(5) - namespace := NewNamespace() + namespace := fwk3k.CreateNamespace(k8s) clusterNamespace := namespace.Name DeferCleanup(func() { - DeleteNamespaces(clusterNamespace) + fwk3k.DeleteNamespaces(k8s, clusterNamespace) }) // Create the cluster with initial version @@ -280,11 +281,11 @@ var _ = When("using the k3kcli", Label("cli"), func() { clusterName := "cluster-" + rand.String(5) - namespace := NewNamespace() + namespace := fwk3k.CreateNamespace(k8s) clusterNamespace := namespace.Name DeferCleanup(func() { - DeleteNamespaces(clusterNamespace) + fwk3k.DeleteNamespaces(k8s, clusterNamespace) }) // Create the cluster with a version @@ -325,11 +326,11 @@ var _ = When("using the k3kcli", Label("cli"), func() { clusterName := "cluster-" + rand.String(5) - namespace := NewNamespace() + namespace := fwk3k.CreateNamespace(k8s) clusterNamespace := namespace.Name DeferCleanup(func() { - DeleteNamespaces(clusterNamespace) + fwk3k.DeleteNamespaces(k8s, clusterNamespace) }) // Create the cluster first @@ -359,11 +360,11 @@ var _ = When("using the k3kcli", Label("cli"), func() { clusterName := "cluster-" + rand.String(5) - namespace := NewNamespace() + namespace := fwk3k.CreateNamespace(k8s) clusterNamespace := namespace.Name DeferCleanup(func() { - DeleteNamespaces(clusterNamespace) + fwk3k.DeleteNamespaces(k8s, clusterNamespace) }) // Create the cluster first @@ -394,11 +395,11 @@ var _ = When("using the k3kcli", Label("cli"), func() { ) clusterName := "cluster-" + rand.String(5) - namespace := NewNamespace() + namespace := fwk3k.CreateNamespace(k8s) clusterNamespace := namespace.Name DeferCleanup(func() { - DeleteNamespaces(clusterNamespace) + fwk3k.DeleteNamespaces(k8s, clusterNamespace) }) _, stderr, err = K3kcli("cluster", "create", "--namespace", clusterNamespace, clusterName) diff --git a/tests/cli/common_test.go b/tests/cli/common_test.go deleted file mode 100644 index 5c5301fd..00000000 --- a/tests/cli/common_test.go +++ /dev/null @@ -1,55 +0,0 @@ -package cli_test - -import ( - "context" - "fmt" - "os" - "sync" - - "k8s.io/utils/ptr" - "sigs.k8s.io/controller-runtime/pkg/client" - - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -func NewNamespace() *v1.Namespace { - GinkgoHelper() - - namespace := &v1.Namespace{ObjectMeta: metav1.ObjectMeta{GenerateName: "ns-", Labels: map[string]string{"e2e": "true"}}} - namespace, err := k8s.CoreV1().Namespaces().Create(context.Background(), namespace, metav1.CreateOptions{}) - Expect(err).To(Not(HaveOccurred())) - - return namespace -} - -func DeleteNamespaces(names ...string) { - GinkgoHelper() - - if _, found := os.LookupEnv("KEEP_NAMESPACES"); found { - By(fmt.Sprintf("Keeping namespace %v", names)) - return - } - - wg := sync.WaitGroup{} - wg.Add(len(names)) - - for _, name := range names { - go func() { - defer wg.Done() - defer GinkgoRecover() - - By(fmt.Sprintf("Deleting namespace %s", name)) - - err := k8s.CoreV1().Namespaces().Delete(context.Background(), name, metav1.DeleteOptions{ - GracePeriodSeconds: ptr.To[int64](0), - }) - Expect(client.IgnoreNotFound(err)).To(Not(HaveOccurred())) - }() - } - - wg.Wait() -} diff --git a/tests/cli/tests_suite_test.go b/tests/cli/tests_suite_test.go index b01bbbed..9d06dd7a 100644 --- a/tests/cli/tests_suite_test.go +++ b/tests/cli/tests_suite_test.go @@ -2,31 +2,18 @@ package cli_test import ( "context" - "io" - "maps" "os" - "path" - "strings" "testing" - "time" - "github.com/go-logr/zapr" "github.com/testcontainers/testcontainers-go" "github.com/testcontainers/testcontainers-go/modules/k3s" - "go.uber.org/zap" - "helm.sh/helm/v3/pkg/action" - "helm.sh/helm/v3/pkg/chart/loader" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" - "k8s.io/client-go/tools/clientcmd" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/log" - v1 "k8s.io/api/core/v1" - clientgoscheme "k8s.io/client-go/kubernetes/scheme" - - "github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1" + fwclient "github.com/rancher/k3k/tests/framework/client" + fwcontainer "github.com/rancher/k3k/tests/framework/container" + fwlog "github.com/rancher/k3k/tests/framework/log" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -45,12 +32,11 @@ func TestTests(t *testing.T) { } var ( - k3sContainer *k3s.K3sContainer - restcfg *rest.Config - k8s *kubernetes.Clientset - k8sClient client.Client - kubeconfigPath string - helmActionConfig *action.Configuration + k3sContainer *k3s.K3sContainer + restcfg *rest.Config + k8s *kubernetes.Clientset + k8sClient client.Client + kubeconfigPath string ) var _ = BeforeSuite(func() { @@ -65,147 +51,41 @@ var _ = BeforeSuite(func() { } installK3SDocker(ctx, repo+"/k3k", repo+"/k3k-kubelet") - initKubernetesClient() + initKubernetesClient(ctx) installK3kChart(repo+"/k3k", repo+"/k3k-kubelet") } else { - initKubernetesClient() + initKubernetesClient(ctx) } }) -func initKubernetesClient() { - var ( - err error - kubeconfig []byte - ) - - logger, err := zap.NewDevelopment() - Expect(err).NotTo(HaveOccurred()) - - log.SetLogger(zapr.NewLogger(logger)) - - kubeconfigPath := os.Getenv("KUBECONFIG") - Expect(kubeconfigPath).To(Not(BeEmpty())) - - kubeconfig, err = os.ReadFile(kubeconfigPath) - Expect(err).To(Not(HaveOccurred())) - - restcfg, err = clientcmd.RESTConfigFromKubeConfig(kubeconfig) - Expect(err).To(Not(HaveOccurred())) - - k8s, err = kubernetes.NewForConfig(restcfg) - Expect(err).To(Not(HaveOccurred())) - - scheme := buildScheme() - k8sClient, err = client.New(restcfg, client.Options{Scheme: scheme}) - Expect(err).NotTo(HaveOccurred()) -} - -func buildScheme() *runtime.Scheme { - scheme := runtime.NewScheme() - - err := clientgoscheme.AddToScheme(scheme) - Expect(err).NotTo(HaveOccurred()) - err = v1beta1.AddToScheme(scheme) - Expect(err).NotTo(HaveOccurred()) - - return scheme -} - func installK3SDocker(ctx context.Context, controllerImage, kubeletImage string) { - var ( - err error - kubeconfig []byte - ) - k3sHostVersion := os.Getenv("K3S_HOST_VERSION") if k3sHostVersion == "" { k3sHostVersion = k3sVersion } - k3sHostVersion = strings.ReplaceAll(k3sHostVersion, "+", "-") + k3sContainer, kubeconfigPath = fwcontainer.SetupK3s(ctx, k3sHostVersion, controllerImage, kubeletImage) +} - k3sContainer, err = k3s.Run(ctx, "rancher/k3s:"+k3sHostVersion) - Expect(err).To(Not(HaveOccurred())) +func initKubernetesClient(ctx context.Context) { + scheme := fwclient.NewScheme() + config, err := fwclient.InitFromKubeconfig(ctx, scheme, k3sContainer) + Expect(err).NotTo(HaveOccurred()) - containerIP, err := k3sContainer.ContainerIP(ctx) - Expect(err).To(Not(HaveOccurred())) - - GinkgoWriter.Println("K3s containerIP: " + containerIP) - - kubeconfig, err = k3sContainer.GetKubeConfig(context.Background()) - Expect(err).To(Not(HaveOccurred())) - - tmpFile, err := os.CreateTemp("", "kubeconfig-") - Expect(err).To(Not(HaveOccurred())) - - _, err = tmpFile.Write(kubeconfig) - Expect(err).To(Not(HaveOccurred())) - Expect(tmpFile.Close()).To(Succeed()) - kubeconfigPath = tmpFile.Name() - - err = k3sContainer.LoadImages(ctx, controllerImage+":dev", kubeletImage+":dev") - Expect(err).To(Not(HaveOccurred())) - DeferCleanup(os.Remove, kubeconfigPath) - - Expect(os.Setenv("KUBECONFIG", kubeconfigPath)).To(Succeed()) - GinkgoWriter.Printf("KUBECONFIG set to: %s\n", kubeconfigPath) + restcfg = config.RestConfig + k8s = config.Clientset + k8sClient = config.Client } func installK3kChart(controllerImage, kubeletImage string) { - pwd, err := os.Getwd() - Expect(err).To(Not(HaveOccurred())) - - k3kChart, err := loader.Load(path.Join(pwd, "../../charts/k3k")) - Expect(err).To(Not(HaveOccurred())) - - helmActionConfig = new(action.Configuration) - + installer := fwcontainer.NewHelmInstaller(controllerImage, kubeletImage, kubeconfigPath) kubeconfig, err := os.ReadFile(kubeconfigPath) - Expect(err).To(Not(HaveOccurred())) + Expect(err).NotTo(HaveOccurred()) - restClientGetter, err := NewRESTClientGetter(kubeconfig) - Expect(err).To(Not(HaveOccurred())) + restClientGetter, err := fwclient.NewRESTClientGetter(kubeconfig) + Expect(err).NotTo(HaveOccurred()) - err = helmActionConfig.Init(restClientGetter, k3kNamespace, os.Getenv("HELM_DRIVER"), func(format string, v ...any) { - GinkgoWriter.Printf("[Helm] "+format+"\n", v...) - }) - Expect(err).To(Not(HaveOccurred())) - - iCli := action.NewInstall(helmActionConfig) - iCli.ReleaseName = "k3k" - iCli.Namespace = k3kNamespace - iCli.CreateNamespace = true - iCli.Timeout = time.Minute - iCli.Wait = true - - controllerMap, _ := k3kChart.Values["controller"].(map[string]any) - - extraEnvArray, _ := controllerMap["extraEnv"].([]map[string]any) - extraEnvArray = append(extraEnvArray, map[string]any{ - "name": "DEBUG", - "value": "true", - }) - controllerMap["extraEnv"] = extraEnvArray - - imageMap, _ := controllerMap["image"].(map[string]any) - maps.Copy(imageMap, map[string]any{ - "repository": controllerImage, - "tag": "dev", - "pullPolicy": "IfNotPresent", - }) - - agentMap, _ := k3kChart.Values["agent"].(map[string]any) - sharedAgentMap, _ := agentMap["shared"].(map[string]any) - sharedAgentImageMap, _ := sharedAgentMap["image"].(map[string]any) - maps.Copy(sharedAgentImageMap, map[string]any{ - "repository": kubeletImage, - "tag": "dev", - }) - - release, err := iCli.Run(k3kChart, k3kChart.Values) - Expect(err).To(Not(HaveOccurred())) - - GinkgoWriter.Printf("Helm release '%s' installed in '%s' namespace\n", release.Name, release.Namespace) + installer.InstallK3kChart(restClientGetter) } var _ = AfterSuite(func() { @@ -215,40 +95,12 @@ var _ = AfterSuite(func() { // dump k3s logs k3sLogs, err := k3sContainer.Logs(ctx) Expect(err).To(Not(HaveOccurred())) - writeLogs("k3s.log", k3sLogs) + fwlog.WriteLogs("k3s.log", k3sLogs) // dump k3k controller logs - k3kLogs := getK3kLogs(ctx) - writeLogs("k3k.log", k3kLogs) + k3kLogs := fwlog.GetK3kPodLogs(ctx, k8sClient, k8s, k3kNamespace) + fwlog.WriteLogs("k3k.log", k3kLogs) testcontainers.CleanupContainer(GinkgoTB(), k3sContainer) } }) - -func getK3kLogs(ctx context.Context) io.ReadCloser { - var podList v1.PodList - - err := k8sClient.List(ctx, &podList, &client.ListOptions{Namespace: k3kNamespace}) - Expect(err).To(Not(HaveOccurred())) - Expect(podList.Items).NotTo(BeEmpty()) - - k3kPod := podList.Items[0] - req := k8s.CoreV1().Pods(k3kPod.Namespace).GetLogs(k3kPod.Name, &v1.PodLogOptions{}) - podLogs, err := req.Stream(ctx) - Expect(err).To(Not(HaveOccurred())) - - return podLogs -} - -func writeLogs(filename string, logs io.ReadCloser) { - logsStr, err := io.ReadAll(logs) - Expect(err).To(Not(HaveOccurred())) - - tempfile := path.Join(os.TempDir(), filename) - err = os.WriteFile(tempfile, []byte(logsStr), 0o644) - Expect(err).To(Not(HaveOccurred())) - - GinkgoWriter.Println("logs written to: " + filename) - - _ = logs.Close() -} diff --git a/tests/e2e/cluster_addons_test.go b/tests/e2e/cluster_addons_test.go index 977efc27..cd29da09 100644 --- a/tests/e2e/cluster_addons_test.go +++ b/tests/e2e/cluster_addons_test.go @@ -9,6 +9,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1" + fwk3k "github.com/rancher/k3k/tests/framework/k3k" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -27,14 +28,14 @@ var _ = When("a cluster with secretMounts configuration is used to load addons", BeforeEach(func() { ctx := context.Background() - namespace := NewNamespace() + namespace := fwk3k.CreateNamespace(k8s) // Create the addon secret err := createAddonSecret(ctx, namespace.Name) Expect(err).ToNot(HaveOccurred()) DeferCleanup(func() { - DeleteNamespaces(namespace.Name) + fwk3k.DeleteNamespaces(k8s, namespace.Name) }) cluster := NewCluster(namespace.Name) @@ -96,14 +97,14 @@ var _ = When("a cluster with addon configuration is used with addons secret in t BeforeEach(func() { ctx := context.Background() - namespace := NewNamespace() + namespace := fwk3k.CreateNamespace(k8s) // Create the addon secret err := createAddonSecret(ctx, namespace.Name) Expect(err).ToNot(HaveOccurred()) DeferCleanup(func() { - DeleteNamespaces(namespace.Name) + fwk3k.DeleteNamespaces(k8s, namespace.Name) }) cluster := NewCluster(namespace.Name) @@ -154,15 +155,15 @@ var _ = When("a cluster with addon configuration is used with addons secret in t BeforeEach(func() { ctx := context.Background() - namespace := NewNamespace() - secretNamespace := NewNamespace() + namespace := fwk3k.CreateNamespace(k8s) + secretNamespace := fwk3k.CreateNamespace(k8s) // Create the addon secret err := createAddonSecret(ctx, secretNamespace.Name) Expect(err).ToNot(HaveOccurred()) DeferCleanup(func() { - DeleteNamespaces(namespace.Name, secretNamespace.Name) + fwk3k.DeleteNamespaces(k8s, namespace.Name, secretNamespace.Name) }) cluster := NewCluster(namespace.Name) diff --git a/tests/e2e/cluster_app_test.go b/tests/e2e/cluster_app_test.go index 0461f2ee..615a5276 100644 --- a/tests/e2e/cluster_app_test.go +++ b/tests/e2e/cluster_app_test.go @@ -12,6 +12,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/rancher/k3k/k3k-kubelet/translate" + fwk3k "github.com/rancher/k3k/tests/framework/k3k" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -28,7 +29,7 @@ var _ = Context("In a shared cluster", Label(e2eTestLabel), Ordered, func() { translator = translate.NewHostTranslator(virtualCluster.Cluster) DeferCleanup(func() { - DeleteNamespaces(virtualCluster.Cluster.Namespace) + fwk3k.DeleteNamespaces(k8s, virtualCluster.Cluster.Namespace) }) }) diff --git a/tests/e2e/cluster_certs_test.go b/tests/e2e/cluster_certs_test.go index 7d5b6cb3..de47e3c5 100644 --- a/tests/e2e/cluster_certs_test.go +++ b/tests/e2e/cluster_certs_test.go @@ -6,6 +6,7 @@ import ( "strings" "github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1" + fwk3k "github.com/rancher/k3k/tests/framework/k3k" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -17,10 +18,10 @@ var _ = When("a cluster with custom certificates is installed with individual ce BeforeEach(func() { ctx := context.Background() - namespace := NewNamespace() + namespace := fwk3k.CreateNamespace(k8s) DeferCleanup(func() { - DeleteNamespaces(namespace.Name) + fwk3k.DeleteNamespaces(k8s, namespace.Name) }) // create custom cert secret diff --git a/tests/e2e/cluster_create_test.go b/tests/e2e/cluster_create_test.go index 94fcf8a7..7f4ea602 100644 --- a/tests/e2e/cluster_create_test.go +++ b/tests/e2e/cluster_create_test.go @@ -5,6 +5,8 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + fwk3k "github.com/rancher/k3k/tests/framework/k3k" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) @@ -13,10 +15,10 @@ var _ = When("creating a shared mode cluster", Label(e2eTestLabel), Label(slowTe var virtualCluster *VirtualCluster BeforeEach(func() { - namespace := NewNamespace() + namespace := fwk3k.CreateNamespace(k8s) DeferCleanup(func() { - DeleteNamespaces(namespace.Name) + fwk3k.DeleteNamespaces(k8s, namespace.Name) }) cluster := NewCluster(namespace.Name) diff --git a/tests/e2e/cluster_network_test.go b/tests/e2e/cluster_network_test.go index 0841b60f..0f2d0a84 100644 --- a/tests/e2e/cluster_network_test.go +++ b/tests/e2e/cluster_network_test.go @@ -1,6 +1,8 @@ package k3k_test import ( + fwk3k "github.com/rancher/k3k/tests/framework/k3k" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) @@ -18,7 +20,7 @@ var _ = When("two virtual clusters are installed", Label(e2eTestLabel), Label(ne }) AfterEach(func() { - DeleteNamespaces(cluster1.Cluster.Namespace, cluster2.Cluster.Namespace) + fwk3k.DeleteNamespaces(k8s, cluster1.Cluster.Namespace, cluster2.Cluster.Namespace) }) It("can create pods in each of them that are isolated", func() { diff --git a/tests/e2e/cluster_persistence_test.go b/tests/e2e/cluster_persistence_test.go index 148006f8..ebc02638 100644 --- a/tests/e2e/cluster_persistence_test.go +++ b/tests/e2e/cluster_persistence_test.go @@ -14,6 +14,7 @@ import ( "github.com/rancher/k3k/k3k-kubelet/translate" "github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1" + fwk3k "github.com/rancher/k3k/tests/framework/k3k" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -26,7 +27,7 @@ var _ = When("an ephemeral cluster is installed", Label(e2eTestLabel), Label(per virtualCluster = NewVirtualCluster() DeferCleanup(func() { - DeleteNamespaces(virtualCluster.Cluster.Namespace) + fwk3k.DeleteNamespaces(k8s, virtualCluster.Cluster.Namespace) }) }) @@ -119,7 +120,7 @@ var _ = When("a dynamic cluster is installed", Label(e2eTestLabel), Label(persis }) AfterEach(func() { - DeleteNamespaces(virtualCluster.Cluster.Namespace) + fwk3k.DeleteNamespaces(k8s, virtualCluster.Cluster.Namespace) }) It("can create a nginx pod", func() { @@ -152,10 +153,10 @@ var _ = When("a dynamic cluster is installed", Label(e2eTestLabel), Label(persis It("can delete a HA cluster", func() { ctx := context.Background() - namespace := NewNamespace() + namespace := fwk3k.CreateNamespace(k8s) DeferCleanup(func() { - DeleteNamespaces(virtualCluster.Cluster.Namespace) + fwk3k.DeleteNamespaces(k8s, virtualCluster.Cluster.Namespace) }) cluster := NewCluster(namespace.Name) diff --git a/tests/e2e/cluster_pod_test.go b/tests/e2e/cluster_pod_test.go index 44f48499..f8f304d6 100644 --- a/tests/e2e/cluster_pod_test.go +++ b/tests/e2e/cluster_pod_test.go @@ -14,6 +14,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/rancher/k3k/k3k-kubelet/translate" + fwk3k "github.com/rancher/k3k/tests/framework/k3k" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -30,7 +31,7 @@ var _ = Context("In a shared cluster", Label(e2eTestLabel), Ordered, func() { translator = translate.NewHostTranslator(virtualCluster.Cluster) DeferCleanup(func() { - DeleteNamespaces(virtualCluster.Cluster.Namespace) + fwk3k.DeleteNamespaces(k8s, virtualCluster.Cluster.Namespace) }) }) diff --git a/tests/e2e/cluster_policy_sync_storageclass_test.go b/tests/e2e/cluster_policy_sync_storageclass_test.go index 735ba461..da58e150 100644 --- a/tests/e2e/cluster_policy_sync_storageclass_test.go +++ b/tests/e2e/cluster_policy_sync_storageclass_test.go @@ -15,6 +15,7 @@ import ( "github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1" "github.com/rancher/k3k/pkg/controller/cluster" "github.com/rancher/k3k/pkg/controller/policy" + fwk3k "github.com/rancher/k3k/tests/framework/k3k" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -109,7 +110,7 @@ var _ = When("a shared mode cluster is created in a namespace with a policy", Or } DeferCleanup(func() { - DeleteNamespaces(ns.Name) + fwk3k.DeleteNamespaces(k8s, ns.Name) err = k8s.StorageV1().StorageClasses().Delete(ctx, storageClassEnabled.Name, metav1.DeleteOptions{}) Expect(err).To(Not(HaveOccurred())) diff --git a/tests/e2e/cluster_registry_test.go b/tests/e2e/cluster_registry_test.go index 1122da64..b5daafc6 100644 --- a/tests/e2e/cluster_registry_test.go +++ b/tests/e2e/cluster_registry_test.go @@ -13,6 +13,7 @@ import ( "github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1" "github.com/rancher/k3k/pkg/controller/policy" + fwk3k "github.com/rancher/k3k/tests/framework/k3k" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -35,7 +36,7 @@ var _ = When("a cluster with private registry configuration is used", Label("e2e } Expect(k8sClient.Create(ctx, vcp)).To(Succeed()) - namespace := NewNamespace() + namespace := fwk3k.CreateNamespace(k8s) err := k8sClient.Get(ctx, client.ObjectKeyFromObject(namespace), namespace) Expect(err).To(Not(HaveOccurred())) @@ -46,7 +47,7 @@ var _ = When("a cluster with private registry configuration is used", Label("e2e Expect(k8sClient.Update(ctx, namespace)).To(Succeed()) DeferCleanup(func() { - DeleteNamespaces(namespace.Name) + fwk3k.DeleteNamespaces(k8s, namespace.Name) Expect(k8sClient.Delete(ctx, vcp)).To(Succeed()) }) diff --git a/tests/e2e/cluster_status_test.go b/tests/e2e/cluster_status_test.go index df94ff6f..4aa68842 100644 --- a/tests/e2e/cluster_status_test.go +++ b/tests/e2e/cluster_status_test.go @@ -14,6 +14,7 @@ import ( "github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1" "github.com/rancher/k3k/pkg/controller/cluster" "github.com/rancher/k3k/pkg/controller/policy" + fwk3k "github.com/rancher/k3k/tests/framework/k3k" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -36,7 +37,7 @@ var _ = When("a cluster's status is tracked", Label(e2eTestLabel), Label(statusT } Expect(k8sClient.Create(ctx, vcp)).To(Succeed()) - namespace = NewNamespace() + namespace = fwk3k.CreateNamespace(k8s) err := k8sClient.Get(ctx, client.ObjectKeyFromObject(namespace), namespace) Expect(err).To(Not(HaveOccurred())) @@ -51,7 +52,7 @@ var _ = When("a cluster's status is tracked", Label(e2eTestLabel), Label(statusT err := k8sClient.Delete(context.Background(), vcp) Expect(err).To(Not(HaveOccurred())) - DeleteNamespaces(namespace.Name) + fwk3k.DeleteNamespaces(k8s, namespace.Name) }) Context("and the cluster is created with a valid configuration", func() { diff --git a/tests/e2e/cluster_sync_storageclass_test.go b/tests/e2e/cluster_sync_storageclass_test.go index 99df140c..87fee89d 100644 --- a/tests/e2e/cluster_sync_storageclass_test.go +++ b/tests/e2e/cluster_sync_storageclass_test.go @@ -11,6 +11,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/rancher/k3k/pkg/controller/cluster" + fwk3k "github.com/rancher/k3k/tests/framework/k3k" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -53,7 +54,7 @@ var _ = When("a shared mode cluster is created", Ordered, Label(e2eTestLabel), f Expect(err).To(Not(HaveOccurred())) DeferCleanup(func() { - DeleteNamespaces(virtualCluster.Cluster.Namespace) + fwk3k.DeleteNamespaces(k8s, virtualCluster.Cluster.Namespace) err = k8s.StorageV1().StorageClasses().Delete(ctx, storageClassEnabled.Name, metav1.DeleteOptions{}) Expect(err).To(Not(HaveOccurred())) diff --git a/tests/e2e/cluster_sync_test.go b/tests/e2e/cluster_sync_test.go index 498554ff..b7b24642 100644 --- a/tests/e2e/cluster_sync_test.go +++ b/tests/e2e/cluster_sync_test.go @@ -9,6 +9,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/rancher/k3k/k3k-kubelet/translate" + fwk3k "github.com/rancher/k3k/tests/framework/k3k" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -25,7 +26,7 @@ var _ = When("a shared mode cluster is created", Ordered, Label(e2eTestLabel), f virtualCluster = NewVirtualCluster() DeferCleanup(func() { - DeleteNamespaces(virtualCluster.Cluster.Namespace) + fwk3k.DeleteNamespaces(k8s, virtualCluster.Cluster.Namespace) }) }) diff --git a/tests/e2e/cluster_update_test.go b/tests/e2e/cluster_update_test.go index 06f9101d..af66c4c3 100644 --- a/tests/e2e/cluster_update_test.go +++ b/tests/e2e/cluster_update_test.go @@ -13,6 +13,7 @@ import ( ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" "github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1" + fwk3k "github.com/rancher/k3k/tests/framework/k3k" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -24,10 +25,10 @@ var _ = When("a shared mode cluster update its envs", Label(e2eTestLabel), Label ctx := context.Background() BeforeEach(func() { - namespace := NewNamespace() + namespace := fwk3k.CreateNamespace(k8s) DeferCleanup(func() { - DeleteNamespaces(namespace.Name) + fwk3k.DeleteNamespaces(k8s, namespace.Name) }) cluster := NewCluster(namespace.Name) @@ -180,10 +181,10 @@ var _ = When("a shared mode cluster update its server args", Label(e2eTestLabel) ctx := context.Background() BeforeEach(func() { - namespace := NewNamespace() + namespace := fwk3k.CreateNamespace(k8s) DeferCleanup(func() { - DeleteNamespaces(namespace.Name) + fwk3k.DeleteNamespaces(k8s, namespace.Name) }) cluster := NewCluster(namespace.Name) @@ -246,10 +247,10 @@ var _ = When("a virtual mode cluster update its envs", Label(e2eTestLabel), Labe ctx := context.Background() BeforeEach(func() { - namespace := NewNamespace() + namespace := fwk3k.CreateNamespace(k8s) DeferCleanup(func() { - DeleteNamespaces(namespace.Name) + fwk3k.DeleteNamespaces(k8s, namespace.Name) }) cluster := NewCluster(namespace.Name) @@ -399,10 +400,10 @@ var _ = When("a virtual mode cluster update its server args", Label(e2eTestLabel ctx := context.Background() BeforeEach(func() { - namespace := NewNamespace() + namespace := fwk3k.CreateNamespace(k8s) DeferCleanup(func() { - DeleteNamespaces(namespace.Name) + fwk3k.DeleteNamespaces(k8s, namespace.Name) }) cluster := NewCluster(namespace.Name) @@ -470,10 +471,10 @@ var _ = When("a shared mode cluster update its version", Label(e2eTestLabel), La BeforeEach(func() { ctx := context.Background() - namespace := NewNamespace() + namespace := fwk3k.CreateNamespace(k8s) DeferCleanup(func() { - DeleteNamespaces(namespace.Name) + fwk3k.DeleteNamespaces(k8s, namespace.Name) }) cluster := NewCluster(namespace.Name) @@ -504,7 +505,7 @@ var _ = When("a shared mode cluster update its version", Label(e2eTestLabel), La nginxPod, _ = virtualCluster.NewNginxPod("") DeferCleanup(func() { - DeleteNamespaces(virtualCluster.Cluster.Namespace) + fwk3k.DeleteNamespaces(k8s, virtualCluster.Cluster.Namespace) }) }) @@ -559,10 +560,10 @@ var _ = When("a virtual mode cluster update its version", Label(e2eTestLabel), L BeforeEach(func() { ctx := context.Background() - namespace := NewNamespace() + namespace := fwk3k.CreateNamespace(k8s) DeferCleanup(func() { - DeleteNamespaces(namespace.Name) + fwk3k.DeleteNamespaces(k8s, namespace.Name) }) cluster := NewCluster(namespace.Name) @@ -663,10 +664,10 @@ var _ = When("a shared mode cluster scales up servers", Label(e2eTestLabel), Lab BeforeEach(func() { ctx := context.Background() - namespace := NewNamespace() + namespace := fwk3k.CreateNamespace(k8s) DeferCleanup(func() { - DeleteNamespaces(namespace.Name) + fwk3k.DeleteNamespaces(k8s, namespace.Name) }) cluster := NewCluster(namespace.Name) @@ -751,10 +752,10 @@ var _ = When("a shared mode cluster scales down servers", Label(e2eTestLabel), L BeforeEach(func() { ctx := context.Background() - namespace := NewNamespace() + namespace := fwk3k.CreateNamespace(k8s) DeferCleanup(func() { - DeleteNamespaces(namespace.Name) + fwk3k.DeleteNamespaces(k8s, namespace.Name) }) cluster := NewCluster(namespace.Name) @@ -841,10 +842,10 @@ var _ = When("a virtual mode cluster scales up servers", Label(e2eTestLabel), La BeforeEach(func() { ctx := context.Background() - namespace := NewNamespace() + namespace := fwk3k.CreateNamespace(k8s) DeferCleanup(func() { - DeleteNamespaces(namespace.Name) + fwk3k.DeleteNamespaces(k8s, namespace.Name) }) cluster := NewCluster(namespace.Name) @@ -929,10 +930,10 @@ var _ = When("a virtual mode cluster scales down servers", Label(e2eTestLabel), BeforeEach(func() { ctx := context.Background() - namespace := NewNamespace() + namespace := fwk3k.CreateNamespace(k8s) DeferCleanup(func() { - DeleteNamespaces(namespace.Name) + fwk3k.DeleteNamespaces(k8s, namespace.Name) }) cluster := NewCluster(namespace.Name) diff --git a/tests/e2e/common_test.go b/tests/e2e/common_test.go index 06f83a47..c8af5c02 100644 --- a/tests/e2e/common_test.go +++ b/tests/e2e/common_test.go @@ -4,8 +4,6 @@ import ( "bytes" "context" "fmt" - "net/url" - "os" "strings" "sync" "time" @@ -16,8 +14,6 @@ import ( "k8s.io/client-go/tools/remotecommand" "k8s.io/kubectl/pkg/scheme" "k8s.io/kubernetes/pkg/api/v1/pod" - "k8s.io/utils/ptr" - "sigs.k8s.io/controller-runtime/pkg/client" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -27,6 +23,7 @@ import ( "github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1" "github.com/rancher/k3k/pkg/controller/certs" "github.com/rancher/k3k/pkg/controller/kubeconfig" + fwk3k "github.com/rancher/k3k/tests/framework/k3k" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -48,7 +45,7 @@ func NewVirtualCluster() *VirtualCluster { // By default, create an ephemeral cl func NewVirtualClusterWithType(persistenceType v1beta1.PersistenceMode) *VirtualCluster { GinkgoHelper() - namespace := NewNamespace() + namespace := fwk3k.CreateNamespace(k8s) cluster := NewCluster(namespace.Name) cluster.Spec.Persistence.Type = persistenceType @@ -90,44 +87,6 @@ func NewVirtualClusters(n int) []*VirtualCluster { return clusters } -func NewNamespace() *v1.Namespace { - GinkgoHelper() - - namespace := &v1.Namespace{ObjectMeta: metav1.ObjectMeta{GenerateName: "ns-", Labels: map[string]string{"e2e": "true"}}} - namespace, err := k8s.CoreV1().Namespaces().Create(context.Background(), namespace, metav1.CreateOptions{}) - Expect(err).To(Not(HaveOccurred())) - - return namespace -} - -func DeleteNamespaces(names ...string) { - GinkgoHelper() - - if _, found := os.LookupEnv("KEEP_NAMESPACES"); found { - By(fmt.Sprintf("Keeping namespace %v", names)) - return - } - - wg := sync.WaitGroup{} - wg.Add(len(names)) - - for _, name := range names { - go func() { - defer wg.Done() - defer GinkgoRecover() - - By(fmt.Sprintf("Deleting namespace %s", name)) - - err := k8s.CoreV1().Namespaces().Delete(context.Background(), name, metav1.DeleteOptions{ - GracePeriodSeconds: ptr.To[int64](0), - }) - Expect(client.IgnoreNotFound(err)).To(Not(HaveOccurred())) - }() - } - - wg.Wait() -} - func NewCluster(namespace string) *v1beta1.Cluster { return &v1beta1.Cluster{ ObjectMeta: metav1.ObjectMeta{ @@ -464,16 +423,3 @@ func isArgFound(pod *v1.Pod, arg string) bool { return false } - -func getServerIP(ctx context.Context, cfg *rest.Config) (string, error) { - if k3sContainer != nil { - return k3sContainer.ContainerIP(ctx) - } - - u, err := url.Parse(cfg.Host) - if err != nil { - return "", err - } - // If Host includes a port, u.Hostname() extracts just the hostname part - return u.Hostname(), nil -} diff --git a/tests/e2e/k8s_restclientgetter_test.go b/tests/e2e/k8s_restclientgetter_test.go deleted file mode 100644 index fb30e01d..00000000 --- a/tests/e2e/k8s_restclientgetter_test.go +++ /dev/null @@ -1,56 +0,0 @@ -package k3k_test - -import ( - "k8s.io/apimachinery/pkg/api/meta" - "k8s.io/client-go/discovery" - "k8s.io/client-go/rest" - "k8s.io/client-go/restmapper" - "k8s.io/client-go/tools/clientcmd" - - memory "k8s.io/client-go/discovery/cached" -) - -type RESTClientGetter struct { - clientconfig clientcmd.ClientConfig - restConfig *rest.Config - discoveryClient discovery.CachedDiscoveryInterface -} - -func NewRESTClientGetter(kubeconfig []byte) (*RESTClientGetter, error) { - clientconfig, err := clientcmd.NewClientConfigFromBytes([]byte(kubeconfig)) - if err != nil { - return nil, err - } - - restConfig, err := clientconfig.ClientConfig() - if err != nil { - return nil, err - } - - dc, err := discovery.NewDiscoveryClientForConfig(restConfig) - if err != nil { - return nil, err - } - - return &RESTClientGetter{ - clientconfig: clientconfig, - restConfig: restConfig, - discoveryClient: memory.NewMemCacheClient(dc), - }, nil -} - -func (r *RESTClientGetter) ToRESTConfig() (*rest.Config, error) { - return r.restConfig, nil -} - -func (r *RESTClientGetter) ToDiscoveryClient() (discovery.CachedDiscoveryInterface, error) { - return r.discoveryClient, nil -} - -func (r *RESTClientGetter) ToRESTMapper() (meta.RESTMapper, error) { - return restmapper.NewDeferredDiscoveryRESTMapper(r.discoveryClient), nil -} - -func (r *RESTClientGetter) ToRawKubeConfigLoader() clientcmd.ClientConfig { - return r.clientconfig -} diff --git a/tests/e2e/tests_suite_test.go b/tests/e2e/tests_suite_test.go index aaf927b8..96feffb3 100644 --- a/tests/e2e/tests_suite_test.go +++ b/tests/e2e/tests_suite_test.go @@ -5,41 +5,34 @@ import ( "context" "fmt" "io" - "maps" "os" "os/exec" "path" "path/filepath" - "strings" "testing" "time" - "github.com/go-logr/zapr" "github.com/testcontainers/testcontainers-go" "github.com/testcontainers/testcontainers-go/modules/k3s" - "go.uber.org/zap" - "helm.sh/helm/v3/pkg/action" - "helm.sh/helm/v3/pkg/chart/loader" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" - "k8s.io/client-go/tools/clientcmd" "k8s.io/client-go/tools/remotecommand" "k8s.io/kubernetes/pkg/api/v1/pod" "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/log" appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - clientgoscheme "k8s.io/client-go/kubernetes/scheme" - "github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1" + fwclient "github.com/rancher/k3k/tests/framework/client" + fwcontainer "github.com/rancher/k3k/tests/framework/container" + fwlog "github.com/rancher/k3k/tests/framework/log" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -73,13 +66,12 @@ func TestTests(t *testing.T) { } var ( - k3sContainer *k3s.K3sContainer - hostIP string - restcfg *rest.Config - k8s *kubernetes.Clientset - k8sClient client.Client - kubeconfigPath string - helmActionConfig *action.Configuration + k3sContainer *k3s.K3sContainer + hostIP string + restcfg *rest.Config + k8s *kubernetes.Clientset + k8sClient client.Client + kubeconfigPath string ) var _ = BeforeSuite(func() { @@ -105,143 +97,35 @@ var _ = BeforeSuite(func() { patchPVC(ctx, k8s) }) -func initKubernetesClient(ctx context.Context) { - var ( - err error - kubeconfig []byte - ) - - logger, err := zap.NewDevelopment() - Expect(err).NotTo(HaveOccurred()) - - log.SetLogger(zapr.NewLogger(logger)) - - kubeconfigPath := os.Getenv("KUBECONFIG") - Expect(kubeconfigPath).To(Not(BeEmpty())) - - kubeconfig, err = os.ReadFile(kubeconfigPath) - Expect(err).To(Not(HaveOccurred())) - - restcfg, err = clientcmd.RESTConfigFromKubeConfig(kubeconfig) - Expect(err).To(Not(HaveOccurred())) - - hostIP, err = getServerIP(ctx, restcfg) - Expect(err).To(Not(HaveOccurred())) - - k8s, err = kubernetes.NewForConfig(restcfg) - Expect(err).To(Not(HaveOccurred())) - - scheme := buildScheme() - k8sClient, err = client.New(restcfg, client.Options{Scheme: scheme}) - Expect(err).NotTo(HaveOccurred()) -} - -func buildScheme() *runtime.Scheme { - scheme := runtime.NewScheme() - - err := clientgoscheme.AddToScheme(scheme) - Expect(err).NotTo(HaveOccurred()) - err = v1beta1.AddToScheme(scheme) - Expect(err).NotTo(HaveOccurred()) - - return scheme -} - func installK3SDocker(ctx context.Context, controllerImage, kubeletImage string) { - var ( - err error - kubeconfig []byte - ) - k3sHostVersion := os.Getenv("K3S_HOST_VERSION") if k3sHostVersion == "" { k3sHostVersion = k3sVersion } - k3sHostVersion = strings.ReplaceAll(k3sHostVersion, "+", "-") + k3sContainer, kubeconfigPath = fwcontainer.SetupK3s(ctx, k3sHostVersion, controllerImage, kubeletImage) +} - k3sContainer, err = k3s.Run(ctx, "rancher/k3s:"+k3sHostVersion) - Expect(err).To(Not(HaveOccurred())) +func initKubernetesClient(ctx context.Context) { + scheme := fwclient.NewScheme() + config, err := fwclient.InitFromKubeconfig(ctx, scheme, k3sContainer) + Expect(err).NotTo(HaveOccurred()) - containerIP, err := k3sContainer.ContainerIP(ctx) - Expect(err).To(Not(HaveOccurred())) - - GinkgoWriter.Println("K3s containerIP: " + containerIP) - - kubeconfig, err = k3sContainer.GetKubeConfig(context.Background()) - Expect(err).To(Not(HaveOccurred())) - - tmpFile, err := os.CreateTemp("", "kubeconfig-") - Expect(err).To(Not(HaveOccurred())) - - _, err = tmpFile.Write(kubeconfig) - Expect(err).To(Not(HaveOccurred())) - Expect(tmpFile.Close()).To(Succeed()) - kubeconfigPath = tmpFile.Name() - - err = k3sContainer.LoadImages(ctx, controllerImage+":dev", kubeletImage+":dev") - Expect(err).To(Not(HaveOccurred())) - DeferCleanup(os.Remove, kubeconfigPath) - - Expect(os.Setenv("KUBECONFIG", kubeconfigPath)).To(Succeed()) - GinkgoWriter.Printf("KUBECONFIG set to: %s\n", kubeconfigPath) + hostIP = config.HostIP + restcfg = config.RestConfig + k8s = config.Clientset + k8sClient = config.Client } func installK3kChart(controllerImage, kubeletImage string) { - pwd, err := os.Getwd() - Expect(err).To(Not(HaveOccurred())) - - k3kChart, err := loader.Load(path.Join(pwd, "../charts/k3k")) - Expect(err).To(Not(HaveOccurred())) - - helmActionConfig = new(action.Configuration) - + installer := fwcontainer.NewHelmInstaller(controllerImage, kubeletImage, kubeconfigPath) kubeconfig, err := os.ReadFile(kubeconfigPath) - Expect(err).To(Not(HaveOccurred())) + Expect(err).NotTo(HaveOccurred()) - restClientGetter, err := NewRESTClientGetter(kubeconfig) - Expect(err).To(Not(HaveOccurred())) + restClientGetter, err := fwclient.NewRESTClientGetter(kubeconfig) + Expect(err).NotTo(HaveOccurred()) - err = helmActionConfig.Init(restClientGetter, k3kNamespace, os.Getenv("HELM_DRIVER"), func(format string, v ...any) { - GinkgoWriter.Printf("[Helm] "+format+"\n", v...) - }) - Expect(err).To(Not(HaveOccurred())) - - iCli := action.NewInstall(helmActionConfig) - iCli.ReleaseName = "k3k" - iCli.Namespace = k3kNamespace - iCli.CreateNamespace = true - iCli.Timeout = time.Minute - iCli.Wait = true - - controllerMap, _ := k3kChart.Values["controller"].(map[string]any) - - extraEnvArray, _ := controllerMap["extraEnv"].([]map[string]any) - extraEnvArray = append(extraEnvArray, map[string]any{ - "name": "DEBUG", - "value": "true", - }) - controllerMap["extraEnv"] = extraEnvArray - - imageMap, _ := controllerMap["image"].(map[string]any) - maps.Copy(imageMap, map[string]any{ - "repository": controllerImage, - "tag": "dev", - "pullPolicy": "IfNotPresent", - }) - - agentMap, _ := k3kChart.Values["agent"].(map[string]any) - sharedAgentMap, _ := agentMap["shared"].(map[string]any) - sharedAgentImageMap, _ := sharedAgentMap["image"].(map[string]any) - maps.Copy(sharedAgentImageMap, map[string]any{ - "repository": kubeletImage, - "tag": "dev", - }) - - release, err := iCli.Run(k3kChart, k3kChart.Values) - Expect(err).To(Not(HaveOccurred())) - - GinkgoWriter.Printf("Helm release '%s' installed in '%s' namespace\n", release.Name, release.Namespace) + installer.InstallK3kChart(restClientGetter) } func patchPVC(ctx context.Context, clientset *kubernetes.Clientset) { @@ -343,11 +227,11 @@ var _ = AfterSuite(func() { // dump k3s logs k3sLogs, err := k3sContainer.Logs(ctx) Expect(err).To(Not(HaveOccurred())) - writeLogs("k3s.log", k3sLogs) + fwlog.WriteLogs("k3s.log", k3sLogs) // dump k3k controller logs - k3kLogs := getK3kLogs(ctx) - writeLogs("k3k.log", k3kLogs) + k3kLogs := fwlog.GetK3kPodLogs(ctx, k8sClient, k8s, k3kNamespace) + fwlog.WriteLogs("k3k.log", k3kLogs) testcontainers.CleanupContainer(GinkgoTB(), k3sContainer) } @@ -451,34 +335,6 @@ func dumpK3kCoverageData(ctx context.Context, folder string) { Expect(k8sClient.Delete(ctx, tarPod)).To(Succeed()) } -func getK3kLogs(ctx context.Context) io.ReadCloser { - var podList v1.PodList - - err := k8sClient.List(ctx, &podList, &client.ListOptions{Namespace: k3kNamespace}) - Expect(err).To(Not(HaveOccurred())) - Expect(podList.Items).NotTo(BeEmpty()) - - k3kPod := podList.Items[0] - req := k8s.CoreV1().Pods(k3kPod.Namespace).GetLogs(k3kPod.Name, &v1.PodLogOptions{Previous: true}) - podLogs, err := req.Stream(ctx) - Expect(err).To(Not(HaveOccurred())) - - return podLogs -} - -func writeLogs(filename string, logs io.ReadCloser) { - logsStr, err := io.ReadAll(logs) - Expect(err).To(Not(HaveOccurred())) - - tempfile := path.Join(os.TempDir(), filename) - err = os.WriteFile(tempfile, []byte(logsStr), 0o644) - Expect(err).To(Not(HaveOccurred())) - - GinkgoWriter.Println("logs written to: " + filename) - - _ = logs.Close() -} - func readFileWithinPod(ctx context.Context, client *kubernetes.Clientset, config *rest.Config, name, namespace, path string) ([]byte, error) { command := []string{"cat", path} diff --git a/tests/framework/client/config.go b/tests/framework/client/config.go new file mode 100644 index 00000000..38c28154 --- /dev/null +++ b/tests/framework/client/config.go @@ -0,0 +1,105 @@ +package client + +import ( + "context" + "fmt" + "net/url" + "os" + + "github.com/go-logr/zapr" + "github.com/testcontainers/testcontainers-go/modules/k3s" + "go.uber.org/zap" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" +) + +// Config holds the Kubernetes client configuration and clients. +type Config struct { + RestConfig *rest.Config + Clientset *kubernetes.Clientset + Client client.Client + HostIP string +} + +// InitFromKubeconfig initializes Kubernetes clients from the KUBECONFIG environment variable. +// It sets up logging, reads the kubeconfig file, creates REST config and clients. +// The scheme parameter should be created using the scheme package. +func InitFromKubeconfig(ctx context.Context, scheme *runtime.Scheme, k3sContainer *k3s.K3sContainer) (*Config, error) { + // Setup logger + logger, err := zap.NewDevelopment() + if err != nil { + return nil, fmt.Errorf("failed to create logger: %w", err) + } + + log.SetLogger(zapr.NewLogger(logger)) + + // Get kubeconfig path from environment + kubeconfigPath := os.Getenv("KUBECONFIG") + if kubeconfigPath == "" { + return nil, fmt.Errorf("KUBECONFIG environment variable is not set") + } + + // Read kubeconfig file + kubeconfig, err := os.ReadFile(kubeconfigPath) + if err != nil { + return nil, fmt.Errorf("failed to read kubeconfig from %s: %w", kubeconfigPath, err) + } + + return InitFromBytes(ctx, kubeconfig, scheme, k3sContainer) +} + +// InitFromBytes initializes Kubernetes clients from kubeconfig bytes. +// The scheme parameter should be created using the scheme package. +func InitFromBytes(ctx context.Context, kubeconfig []byte, scheme *runtime.Scheme, k3sContainer *k3s.K3sContainer) (*Config, error) { + // Create REST config from kubeconfig + restConfig, err := clientcmd.RESTConfigFromKubeConfig(kubeconfig) + if err != nil { + return nil, fmt.Errorf("failed to create REST config: %w", err) + } + + // Extract host IP from REST config + hostIP, err := getServerIP(ctx, restConfig, k3sContainer) + if err != nil { + return nil, fmt.Errorf("failed to get server IP: %w", err) + } + + // Create Kubernetes clientset + clientset, err := kubernetes.NewForConfig(restConfig) + if err != nil { + return nil, fmt.Errorf("failed to create Kubernetes clientset: %w", err) + } + + // Create controller-runtime client + runtimeClient, err := client.New(restConfig, client.Options{Scheme: scheme}) + if err != nil { + return nil, fmt.Errorf("failed to create controller-runtime client: %w", err) + } + + return &Config{ + RestConfig: restConfig, + Clientset: clientset, + Client: runtimeClient, + HostIP: hostIP, + }, nil +} + +// getServerIP extracts the server IP from the REST config. +// If running with testcontainers, it returns the container IP. +// Otherwise, it parses the hostname from the REST config host. +func getServerIP(ctx context.Context, cfg *rest.Config, k3sContainer *k3s.K3sContainer) (string, error) { + if k3sContainer != nil { + return k3sContainer.ContainerIP(ctx) + } + + u, err := url.Parse(cfg.Host) + if err != nil { + return "", fmt.Errorf("failed to parse REST config host: %w", err) + } + + // If Host includes a port, u.Hostname() extracts just the hostname part + return u.Hostname(), nil +} diff --git a/tests/cli/k8s_restclientgetter_test.go b/tests/framework/client/getter.go similarity index 67% rename from tests/cli/k8s_restclientgetter_test.go rename to tests/framework/client/getter.go index 65064128..630e0643 100644 --- a/tests/cli/k8s_restclientgetter_test.go +++ b/tests/framework/client/getter.go @@ -1,4 +1,4 @@ -package cli_test +package client import ( "k8s.io/apimachinery/pkg/api/meta" @@ -10,14 +10,19 @@ import ( memory "k8s.io/client-go/discovery/cached" ) +// RESTClientGetter is a Kubernetes REST client getter implementation that satisfies +// the genericclioptions.RESTClientGetter interface. This is used primarily for Helm +// operations in tests. type RESTClientGetter struct { clientconfig clientcmd.ClientConfig restConfig *rest.Config discoveryClient discovery.CachedDiscoveryInterface } +// NewRESTClientGetter creates a new RESTClientGetter from kubeconfig bytes. +// This is used for Helm operations in tests. func NewRESTClientGetter(kubeconfig []byte) (*RESTClientGetter, error) { - clientconfig, err := clientcmd.NewClientConfigFromBytes([]byte(kubeconfig)) + clientconfig, err := clientcmd.NewClientConfigFromBytes(kubeconfig) if err != nil { return nil, err } @@ -39,18 +44,22 @@ func NewRESTClientGetter(kubeconfig []byte) (*RESTClientGetter, error) { }, nil } +// ToRESTConfig returns the REST config. func (r *RESTClientGetter) ToRESTConfig() (*rest.Config, error) { return r.restConfig, nil } +// ToDiscoveryClient returns the cached discovery client. func (r *RESTClientGetter) ToDiscoveryClient() (discovery.CachedDiscoveryInterface, error) { return r.discoveryClient, nil } +// ToRESTMapper returns a REST mapper from the discovery client. func (r *RESTClientGetter) ToRESTMapper() (meta.RESTMapper, error) { return restmapper.NewDeferredDiscoveryRESTMapper(r.discoveryClient), nil } +// ToRawKubeConfigLoader returns the raw kubeconfig loader. func (r *RESTClientGetter) ToRawKubeConfigLoader() clientcmd.ClientConfig { return r.clientconfig } diff --git a/tests/framework/client/scheme.go b/tests/framework/client/scheme.go new file mode 100644 index 00000000..a0a5e614 --- /dev/null +++ b/tests/framework/client/scheme.go @@ -0,0 +1,27 @@ +package client + +import ( + "k8s.io/apimachinery/pkg/runtime" + + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + + "github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1" +) + +// NewScheme creates a new Kubernetes runtime scheme with core APIs and k3k CRDs. +// This is suitable for most k3k test scenarios including integration and E2E tests. +func NewScheme() *runtime.Scheme { + scheme := runtime.NewScheme() + + // Add core Kubernetes scheme (includes most common types) + if err := clientgoscheme.AddToScheme(scheme); err != nil { + panic(err) + } + + // Add k3k CRDs + if err := v1beta1.AddToScheme(scheme); err != nil { + panic(err) + } + + return scheme +} diff --git a/tests/framework/container/helm.go b/tests/framework/container/helm.go new file mode 100644 index 00000000..84b68531 --- /dev/null +++ b/tests/framework/container/helm.go @@ -0,0 +1,105 @@ +package container + +import ( + "maps" + "os" + "path" + "time" + + "helm.sh/helm/v3/pkg/action" + "helm.sh/helm/v3/pkg/chart/loader" + + fwclient "github.com/rancher/k3k/tests/framework/client" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// HelmInstaller provides configuration for Helm chart installation. +type HelmInstaller struct { + ChartPath string + Namespace string + ReleaseName string + Timeout time.Duration + Wait bool + ControllerImage string + KubeletImage string + KubeconfigPath string +} + +// InstallK3kChart installs the k3k Helm chart with the specified configuration. +// It uses the provided RESTClientGetter for authentication and returns the Helm action configuration. +func (h *HelmInstaller) InstallK3kChart(restClientGetter *fwclient.RESTClientGetter) *action.Configuration { + GinkgoHelper() + + // Load chart + k3kChart, err := loader.Load(h.ChartPath) + Expect(err).To(Not(HaveOccurred())) + + // Initialize Helm action configuration + helmActionConfig := new(action.Configuration) + + err = helmActionConfig.Init(restClientGetter, h.Namespace, os.Getenv("HELM_DRIVER"), func(format string, v ...any) { + GinkgoWriter.Printf("[Helm] "+format+"\n", v...) + }) + Expect(err).To(Not(HaveOccurred())) + + // Create install action + iCli := action.NewInstall(helmActionConfig) + iCli.ReleaseName = h.ReleaseName + iCli.Namespace = h.Namespace + iCli.CreateNamespace = true + iCli.Timeout = h.Timeout + iCli.Wait = h.Wait + + // Configure controller image + controllerMap, _ := k3kChart.Values["controller"].(map[string]any) + + extraEnvArray, _ := controllerMap["extraEnv"].([]map[string]any) + extraEnvArray = append(extraEnvArray, map[string]any{ + "name": "DEBUG", + "value": "true", + }) + controllerMap["extraEnv"] = extraEnvArray + + imageMap, _ := controllerMap["image"].(map[string]any) + maps.Copy(imageMap, map[string]any{ + "repository": h.ControllerImage, + "tag": "dev", + "pullPolicy": "IfNotPresent", + }) + + // Configure agent image + agentMap, _ := k3kChart.Values["agent"].(map[string]any) + sharedAgentMap, _ := agentMap["shared"].(map[string]any) + sharedAgentImageMap, _ := sharedAgentMap["image"].(map[string]any) + maps.Copy(sharedAgentImageMap, map[string]any{ + "repository": h.KubeletImage, + "tag": "dev", + }) + + // Install chart + release, err := iCli.Run(k3kChart, k3kChart.Values) + Expect(err).To(Not(HaveOccurred())) + + GinkgoWriter.Printf("Helm release '%s' installed in '%s' namespace\n", release.Name, release.Namespace) + + return helmActionConfig +} + +// NewHelmInstaller creates a new HelmInstaller with default values. +func NewHelmInstaller(controllerImage, kubeletImage, kubeconfigPath string) *HelmInstaller { + pwd, err := os.Getwd() + Expect(err).To(Not(HaveOccurred())) + + return &HelmInstaller{ + ChartPath: path.Join(pwd, "../../charts/k3k"), + Namespace: "k3k-system", + ReleaseName: "k3k", + Timeout: time.Minute, + Wait: true, + ControllerImage: controllerImage, + KubeletImage: kubeletImage, + KubeconfigPath: kubeconfigPath, + } +} diff --git a/tests/framework/container/k3s.go b/tests/framework/container/k3s.go new file mode 100644 index 00000000..e87fe7ac --- /dev/null +++ b/tests/framework/container/k3s.go @@ -0,0 +1,63 @@ +package container + +import ( + "context" + "os" + "strings" + + "github.com/testcontainers/testcontainers-go/modules/k3s" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// SetupK3s creates and starts a K3s testcontainer with the specified version and loads the provided images. +// It returns the container instance and a temporary kubeconfig file path. +// The kubeconfig is automatically cleaned up via DeferCleanup. +func SetupK3s(ctx context.Context, k3sVersion, controllerImage, kubeletImage string) (*k3s.K3sContainer, string) { + GinkgoHelper() + + var ( + err error + kubeconfig []byte + ) + + // Normalize version string (replace + with -) + k3sVersion = strings.ReplaceAll(k3sVersion, "+", "-") + + // Start K3s container + k3sContainer, err := k3s.Run(ctx, "rancher/k3s:"+k3sVersion) + Expect(err).To(Not(HaveOccurred())) + + containerIP, err := k3sContainer.ContainerIP(ctx) + Expect(err).To(Not(HaveOccurred())) + + GinkgoWriter.Println("K3s containerIP: " + containerIP) + + // Get kubeconfig from container + kubeconfig, err = k3sContainer.GetKubeConfig(ctx) + Expect(err).To(Not(HaveOccurred())) + + // Write kubeconfig to temp file + tmpFile, err := os.CreateTemp("", "kubeconfig-") + Expect(err).To(Not(HaveOccurred())) + + _, err = tmpFile.Write(kubeconfig) + Expect(err).To(Not(HaveOccurred())) + Expect(tmpFile.Close()).To(Succeed()) + + kubeconfigPath := tmpFile.Name() + + // Load images into the K3s container + err = k3sContainer.LoadImages(ctx, controllerImage+":dev", kubeletImage+":dev") + Expect(err).To(Not(HaveOccurred())) + + // Register cleanup + DeferCleanup(os.Remove, kubeconfigPath) + + // Set KUBECONFIG environment variable + Expect(os.Setenv("KUBECONFIG", kubeconfigPath)).To(Succeed()) + GinkgoWriter.Printf("KUBECONFIG set to: %s\n", kubeconfigPath) + + return k3sContainer, kubeconfigPath +} diff --git a/tests/framework/env/env.go b/tests/framework/env/env.go new file mode 100644 index 00000000..16ff6ea5 --- /dev/null +++ b/tests/framework/env/env.go @@ -0,0 +1,34 @@ +package env + +import ( + "path/filepath" + + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/envtest" +) + +// Config holds the envtest environment configuration. +type Config struct { + CRDPaths []string + Scheme *runtime.Scheme +} + +// NewEnvironment creates a new envtest.Environment with CRDs from the k3k charts directory. +// The scheme parameter should be created using the scheme package. +func NewEnvironment(scheme *runtime.Scheme) *envtest.Environment { + return &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "charts", "k3k", "templates", "crds")}, + ErrorIfCRDPathMissing: true, + Scheme: scheme, + } +} + +// NewEnvironmentWithPaths creates a new envtest.Environment with custom CRD paths. +// This is useful when running tests from different locations in the directory tree. +func NewEnvironmentWithPaths(scheme *runtime.Scheme, crdPaths ...string) *envtest.Environment { + return &envtest.Environment{ + CRDDirectoryPaths: crdPaths, + ErrorIfCRDPathMissing: true, + Scheme: scheme, + } +} diff --git a/tests/framework/k3k/namespace.go b/tests/framework/k3k/namespace.go new file mode 100644 index 00000000..bbe446e7 --- /dev/null +++ b/tests/framework/k3k/namespace.go @@ -0,0 +1,69 @@ +package k3k + +import ( + "context" + "fmt" + "os" + "sync" + + "k8s.io/client-go/kubernetes" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// CreateNamespace creates a new namespace with a generated name and the "e2e: true" label. +// The namespace is created using the provided Kubernetes clientset. +func CreateNamespace(clientset kubernetes.Interface) *v1.Namespace { + GinkgoHelper() + + namespace := &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "ns-", + Labels: map[string]string{ + "e2e": "true", + }, + }, + } + + namespace, err := clientset.CoreV1().Namespaces().Create(context.Background(), namespace, metav1.CreateOptions{}) + Expect(err).To(Not(HaveOccurred())) + + return namespace +} + +// DeleteNamespaces deletes the specified namespaces in parallel. +// If the KEEP_NAMESPACES environment variable is set, namespaces are preserved instead. +// This is useful for debugging test failures. +func DeleteNamespaces(clientset kubernetes.Interface, names ...string) { + GinkgoHelper() + + if _, found := os.LookupEnv("KEEP_NAMESPACES"); found { + By(fmt.Sprintf("Keeping namespaces %v", names)) + return + } + + wg := sync.WaitGroup{} + wg.Add(len(names)) + + for _, name := range names { + go func() { + defer wg.Done() + defer GinkgoRecover() + + By(fmt.Sprintf("Deleting namespace %s", name)) + + err := clientset.CoreV1().Namespaces().Delete(context.Background(), name, metav1.DeleteOptions{ + GracePeriodSeconds: ptr.To[int64](0), + }) + Expect(client.IgnoreNotFound(err)).To(Not(HaveOccurred())) + }() + } + + wg.Wait() +} diff --git a/tests/framework/log/log.go b/tests/framework/log/log.go new file mode 100644 index 00000000..27394fab --- /dev/null +++ b/tests/framework/log/log.go @@ -0,0 +1,71 @@ +package log + +import ( + "context" + "io" + "os" + "path" + + "k8s.io/client-go/kubernetes" + "sigs.k8s.io/controller-runtime/pkg/client" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// GetK3kPodLogs retrieves logs from the first k3k pod in the specified namespace. +// This is useful for debugging test failures. +func GetK3kPodLogs(ctx context.Context, k8sClient client.Client, clientset kubernetes.Interface, namespace string) io.ReadCloser { + GinkgoHelper() + + var podList v1.PodList + + err := k8sClient.List(ctx, &podList, &client.ListOptions{Namespace: namespace}) + Expect(err).To(Not(HaveOccurred())) + Expect(podList.Items).NotTo(BeEmpty()) + + k3kPod := podList.Items[0] + + // Fetch complete pod object to access status information + pod, err := clientset.CoreV1().Pods(k3kPod.Namespace).Get(ctx, k3kPod.Name, metav1.GetOptions{}) + Expect(err).To(Not(HaveOccurred())) + + // Detect if the container has been restarted (e.g., for coverage dumping in E2E tests) + fetchPrevious := false + + if len(pod.Status.ContainerStatuses) > 0 { + containerStatus := pod.Status.ContainerStatuses[0] + + if containerStatus.RestartCount > 0 { + fetchPrevious = true + + GinkgoWriter.Printf("Container has been restarted %d time(s), fetching previous logs\n", containerStatus.RestartCount) + } + } + + req := clientset.CoreV1().Pods(pod.Namespace).GetLogs(pod.Name, &v1.PodLogOptions{Previous: fetchPrevious}) + podLogs, err := req.Stream(ctx) + Expect(err).To(Not(HaveOccurred())) + + return podLogs +} + +// WriteLogs writes the provided logs to a temporary file with the specified filename. +// The file is written to os.TempDir() and the full path is logged to GinkgoWriter. +func WriteLogs(filename string, logs io.ReadCloser) { + GinkgoHelper() + + logsStr, err := io.ReadAll(logs) + Expect(err).To(Not(HaveOccurred())) + + tempfile := path.Join(os.TempDir(), filename) + err = os.WriteFile(tempfile, logsStr, 0o644) + Expect(err).To(Not(HaveOccurred())) + + GinkgoWriter.Println("logs written to: " + tempfile) + + _ = logs.Close() +} diff --git a/tests/integration/cluster/cluster_suite_test.go b/tests/integration/cluster/cluster_suite_test.go index 5a9c53af..e4a3fdbe 100644 --- a/tests/integration/cluster/cluster_suite_test.go +++ b/tests/integration/cluster/cluster_suite_test.go @@ -8,19 +8,17 @@ import ( "github.com/go-logr/zapr" "go.uber.org/zap" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/envtest" - clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" - "github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1" "github.com/rancher/k3k/pkg/controller/cluster" "github.com/rancher/k3k/pkg/controller/cluster/agent" + fwclient "github.com/rancher/k3k/tests/framework/client" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -56,7 +54,7 @@ var _ = BeforeSuite(func() { k8s, err = kubernetes.NewForConfig(cfg) Expect(err).NotTo(HaveOccurred()) - scheme := buildScheme() + scheme := fwclient.NewScheme() k8sClient, err = client.New(cfg, client.Options{Scheme: scheme}) Expect(err).NotTo(HaveOccurred()) @@ -97,14 +95,3 @@ var _ = AfterSuite(func() { err := testEnv.Stop() Expect(err).NotTo(HaveOccurred()) }) - -func buildScheme() *runtime.Scheme { - scheme := runtime.NewScheme() - - err := clientgoscheme.AddToScheme(scheme) - Expect(err).NotTo(HaveOccurred()) - err = v1beta1.AddToScheme(scheme) - Expect(err).NotTo(HaveOccurred()) - - return scheme -} diff --git a/tests/integration/k3k-kubelet/syncer_suite_test.go b/tests/integration/k3k-kubelet/syncer_suite_test.go index 6c54d8a4..a1d9758c 100644 --- a/tests/integration/k3k-kubelet/syncer_suite_test.go +++ b/tests/integration/k3k-kubelet/syncer_suite_test.go @@ -10,17 +10,16 @@ import ( "github.com/go-logr/zapr" "go.uber.org/zap" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/envtest" - clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "github.com/rancher/k3k/k3k-kubelet/translate" "github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1" + fwclient "github.com/rancher/k3k/tests/framework/client" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -95,7 +94,7 @@ func NewTestEnv() *TestEnv { CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "charts", "k3k", "templates", "crds")}, ErrorIfCRDPathMissing: true, BinaryAssetsDirectory: tempDir, - Scheme: buildScheme(), + Scheme: fwclient.NewScheme(), } cfg, err := testEnv.Start() @@ -114,17 +113,6 @@ func NewTestEnv() *TestEnv { } } -func buildScheme() *runtime.Scheme { - scheme := runtime.NewScheme() - - err := clientgoscheme.AddToScheme(scheme) - Expect(err).NotTo(HaveOccurred()) - err = v1beta1.AddToScheme(scheme) - Expect(err).NotTo(HaveOccurred()) - - return scheme -} - var _ = Describe("Kubelet Controller", func() { var ( ctx context.Context diff --git a/tests/integration/policy/policy_suite_test.go b/tests/integration/policy/policy_suite_test.go index 3ad9371c..820618ee 100644 --- a/tests/integration/policy/policy_suite_test.go +++ b/tests/integration/policy/policy_suite_test.go @@ -7,18 +7,14 @@ import ( "github.com/go-logr/zapr" "go.uber.org/zap" - "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/envtest" - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - networkingv1 "k8s.io/api/networking/v1" ctrl "sigs.k8s.io/controller-runtime" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" - "github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1" "github.com/rancher/k3k/pkg/controller/policy" + fwclient "github.com/rancher/k3k/tests/framework/client" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -46,7 +42,7 @@ var _ = BeforeSuite(func() { cfg, err := testEnv.Start() Expect(err).NotTo(HaveOccurred()) - scheme := buildScheme() + scheme := fwclient.NewScheme() k8sClient, err = client.New(cfg, client.Options{Scheme: scheme}) Expect(err).NotTo(HaveOccurred()) @@ -75,18 +71,3 @@ var _ = AfterSuite(func() { err := testEnv.Stop() Expect(err).NotTo(HaveOccurred()) }) - -func buildScheme() *runtime.Scheme { - scheme := runtime.NewScheme() - - err := corev1.AddToScheme(scheme) - Expect(err).NotTo(HaveOccurred()) - err = appsv1.AddToScheme(scheme) - Expect(err).NotTo(HaveOccurred()) - err = networkingv1.AddToScheme(scheme) - Expect(err).NotTo(HaveOccurred()) - err = v1beta1.AddToScheme(scheme) - Expect(err).NotTo(HaveOccurred()) - - return scheme -}