Refactor test suite to remove K3s container dependency (#971)

- Removed K3s container setup and teardown logic from CLI and E2E test suites.
- Simplified Kubernetes client initialization by eliminating the need for K3s container.
- Updated InitFromKubeconfig function to no longer require K3s container as a parameter.
- Deleted unused HelmInstaller and RESTClientGetter implementations.
- Cleaned up logging functions related to K3s and Helm operations.

tests/cli/tests_suite_test.go

@@ -2,18 +2,13 @@ package cli_test
 
 import (
 	"context"
-	"os"
 	"testing"
 
-	"github.com/testcontainers/testcontainers-go"
-	"github.com/testcontainers/testcontainers-go/modules/k3s"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	fwclient "github.com/rancher/k3k/tests/framework/client"
-	fwcontainer "github.com/rancher/k3k/tests/framework/container"
-	fwlog "github.com/rancher/k3k/tests/framework/log"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -32,75 +27,23 @@ func TestTests(t *testing.T) {
 }
 
 var (
-	k3sContainer   *k3s.K3sContainer
-	restcfg        *rest.Config
-	k8s            *kubernetes.Clientset
-	k8sClient      client.Client
-	kubeconfigPath string
+	restcfg   *rest.Config
+	k8s       *kubernetes.Clientset
+	k8sClient client.Client
 )
 
 var _ = BeforeSuite(func() {
 	ctx := context.Background()
 
-	_, dockerInstallEnabled := os.LookupEnv("K3K_DOCKER_INSTALL")
-
-	if dockerInstallEnabled {
-		repo := os.Getenv("REPO")
-		if repo == "" {
-			repo = "rancher"
-		}
-
-		installK3SDocker(ctx, repo+"/k3k", repo+"/k3k-kubelet")
-		initKubernetesClient(ctx)
-		installK3kChart(repo+"/k3k", repo+"/k3k-kubelet")
-	} else {
-		initKubernetesClient(ctx)
-	}
+	initKubernetesClient(ctx)
 })
 
-func installK3SDocker(ctx context.Context, controllerImage, kubeletImage string) {
-	k3sHostVersion := os.Getenv("K3S_HOST_VERSION")
-	if k3sHostVersion == "" {
-		k3sHostVersion = k3sVersion
-	}
-
-	k3sContainer, kubeconfigPath = fwcontainer.SetupK3s(ctx, k3sHostVersion, controllerImage, kubeletImage)
-}
-
 func initKubernetesClient(ctx context.Context) {
 	scheme := fwclient.NewScheme()
-	config, err := fwclient.InitFromKubeconfig(ctx, scheme, k3sContainer)
+	config, err := fwclient.InitFromKubeconfig(ctx, scheme)
 	Expect(err).NotTo(HaveOccurred())
 
 	restcfg = config.RestConfig
 	k8s = config.Clientset
 	k8sClient = config.Client
 }
-
-func installK3kChart(controllerImage, kubeletImage string) {
-	installer := fwcontainer.NewHelmInstaller(controllerImage, kubeletImage, kubeconfigPath)
-	kubeconfig, err := os.ReadFile(kubeconfigPath)
-	Expect(err).NotTo(HaveOccurred())
-
-	restClientGetter, err := fwclient.NewRESTClientGetter(kubeconfig)
-	Expect(err).NotTo(HaveOccurred())
-
-	installer.InstallK3kChart(restClientGetter)
-}
-
-var _ = AfterSuite(func() {
-	ctx := context.Background()
-
-	if k3sContainer != nil {
-		// dump k3s logs
-		k3sLogs, err := k3sContainer.Logs(ctx)
-		Expect(err).To(Not(HaveOccurred()))
-		fwlog.WriteLogs("k3s.log", k3sLogs)
-
-		// dump k3k controller logs
-		k3kLogs := fwlog.GetK3kPodLogs(ctx, k8sClient, k8s, k3kNamespace)
-		fwlog.WriteLogs("k3k.log", k3kLogs)
-
-		testcontainers.CleanupContainer(GinkgoTB(), k3sContainer)
-	}
-})

tests/e2e/tests_suite_test.go

@@ -12,8 +12,6 @@ import (
 	"testing"
 	"time"
 
-	"github.com/testcontainers/testcontainers-go"
-	"github.com/testcontainers/testcontainers-go/modules/k3s"
 	"k8s.io/apimachinery/pkg/api/resource"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
@@ -31,8 +29,6 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	fwclient "github.com/rancher/k3k/tests/framework/client"
-	fwcontainer "github.com/rancher/k3k/tests/framework/container"
-	fwlog "github.com/rancher/k3k/tests/framework/log"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -66,12 +62,10 @@ func TestTests(t *testing.T) {
 }
 
 var (
-	k3sContainer   *k3s.K3sContainer
-	hostIP         string
-	restcfg        *rest.Config
-	k8s            *kubernetes.Clientset
-	k8sClient      client.Client
-	kubeconfigPath string
+	hostIP    string
+	restcfg   *rest.Config
+	k8s       *kubernetes.Clientset
+	k8sClient client.Client
 )
 
 var _ = BeforeSuite(func() {
@@ -79,36 +73,14 @@ var _ = BeforeSuite(func() {
 
 	GinkgoWriter.Println("GOCOVERDIR:", os.Getenv("GOCOVERDIR"))
 
-	_, dockerInstallEnabled := os.LookupEnv("K3K_DOCKER_INSTALL")
-
-	if dockerInstallEnabled {
-		repo := os.Getenv("REPO")
-		if repo == "" {
-			repo = "rancher"
-		}
-
-		installK3SDocker(ctx, repo+"/k3k", repo+"/k3k-kubelet")
-		initKubernetesClient(ctx)
-		installK3kChart(repo+"/k3k", repo+"/k3k-kubelet")
-	} else {
-		initKubernetesClient(ctx)
-	}
+	initKubernetesClient(ctx)
 
 	patchPVC(ctx, k8s)
 })
 
-func installK3SDocker(ctx context.Context, controllerImage, kubeletImage string) {
-	k3sHostVersion := os.Getenv("K3S_HOST_VERSION")
-	if k3sHostVersion == "" {
-		k3sHostVersion = k3sVersion
-	}
-
-	k3sContainer, kubeconfigPath = fwcontainer.SetupK3s(ctx, k3sHostVersion, controllerImage, kubeletImage)
-}
-
 func initKubernetesClient(ctx context.Context) {
 	scheme := fwclient.NewScheme()
-	config, err := fwclient.InitFromKubeconfig(ctx, scheme, k3sContainer)
+	config, err := fwclient.InitFromKubeconfig(ctx, scheme)
 	Expect(err).NotTo(HaveOccurred())
 
 	hostIP = config.HostIP
@@ -117,17 +89,6 @@ func initKubernetesClient(ctx context.Context) {
 	k8sClient = config.Client
 }
 
-func installK3kChart(controllerImage, kubeletImage string) {
-	installer := fwcontainer.NewHelmInstaller(controllerImage, kubeletImage, kubeconfigPath)
-	kubeconfig, err := os.ReadFile(kubeconfigPath)
-	Expect(err).NotTo(HaveOccurred())
-
-	restClientGetter, err := fwclient.NewRESTClientGetter(kubeconfig)
-	Expect(err).NotTo(HaveOccurred())
-
-	installer.InstallK3kChart(restClientGetter)
-}
-
 func patchPVC(ctx context.Context, clientset *kubernetes.Clientset) {
 	deployments, err := clientset.AppsV1().Deployments(k3kNamespace).List(ctx, metav1.ListOptions{})
 	Expect(err).To(Not(HaveOccurred()))
@@ -222,19 +183,6 @@ var _ = AfterSuite(func() {
 	}
 
 	dumpK3kCoverageData(ctx, goCoverDir)
-
-	if k3sContainer != nil {
-		// dump k3s logs
-		k3sLogs, err := k3sContainer.Logs(ctx)
-		Expect(err).To(Not(HaveOccurred()))
-		fwlog.WriteLogs("k3s.log", k3sLogs)
-
-		// dump k3k controller logs
-		k3kLogs := fwlog.GetK3kPodLogs(ctx, k8sClient, k8s, k3kNamespace)
-		fwlog.WriteLogs("k3k.log", k3kLogs)
-
-		testcontainers.CleanupContainer(GinkgoTB(), k3sContainer)
-	}
 })
 
 // dumpK3kCoverageData will kill the K3k controller container to force it to dump the coverage data.

tests/framework/client/config.go

@@ -7,7 +7,6 @@ import (
 	"os"
 
 	"github.com/go-logr/zapr"
-	"github.com/testcontainers/testcontainers-go/modules/k3s"
 	"go.uber.org/zap"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/kubernetes"
@@ -28,7 +27,7 @@ type Config struct {
 // InitFromKubeconfig initializes Kubernetes clients from the KUBECONFIG environment variable.
 // It sets up logging, reads the kubeconfig file, creates REST config and clients.
 // The scheme parameter should be created using the scheme package.
-func InitFromKubeconfig(ctx context.Context, scheme *runtime.Scheme, k3sContainer *k3s.K3sContainer) (*Config, error) {
+func InitFromKubeconfig(ctx context.Context, scheme *runtime.Scheme) (*Config, error) {
 	// Setup logger
 	logger, err := zap.NewDevelopment()
 	if err != nil {
@@ -49,12 +48,12 @@ func InitFromKubeconfig(ctx context.Context, scheme *runtime.Scheme, k3sContaine
 		return nil, fmt.Errorf("failed to read kubeconfig from %s: %w", kubeconfigPath, err)
 	}
 
-	return InitFromBytes(ctx, kubeconfig, scheme, k3sContainer)
+	return InitFromBytes(ctx, kubeconfig, scheme)
 }
 
 // InitFromBytes initializes Kubernetes clients from kubeconfig bytes.
 // The scheme parameter should be created using the scheme package.
-func InitFromBytes(ctx context.Context, kubeconfig []byte, scheme *runtime.Scheme, k3sContainer *k3s.K3sContainer) (*Config, error) {
+func InitFromBytes(ctx context.Context, kubeconfig []byte, scheme *runtime.Scheme) (*Config, error) {
 	// Create REST config from kubeconfig
 	restConfig, err := clientcmd.RESTConfigFromKubeConfig(kubeconfig)
 	if err != nil {
@@ -62,7 +61,7 @@ func InitFromBytes(ctx context.Context, kubeconfig []byte, scheme *runtime.Schem
 	}
 
 	// Extract host IP from REST config
-	hostIP, err := getServerIP(ctx, restConfig, k3sContainer)
+	hostIP, err := getServerIP(restConfig)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get server IP: %w", err)
 	}
@@ -87,14 +86,8 @@ func InitFromBytes(ctx context.Context, kubeconfig []byte, scheme *runtime.Schem
 	}, nil
 }
 
-// getServerIP extracts the server IP from the REST config.
-// If running with testcontainers, it returns the container IP.
-// Otherwise, it parses the hostname from the REST config host.
-func getServerIP(ctx context.Context, cfg *rest.Config, k3sContainer *k3s.K3sContainer) (string, error) {
-	if k3sContainer != nil {
-		return k3sContainer.ContainerIP(ctx)
-	}
-
+// getServerIP extracts the server IP by parsing the hostname from the REST config host.
+func getServerIP(cfg *rest.Config) (string, error) {
 	u, err := url.Parse(cfg.Host)
 	if err != nil {
 		return "", fmt.Errorf("failed to parse REST config host: %w", err)

tests/framework/client/getter.go

@@ -1,65 +0,0 @@
-package client
-
-import (
-	"k8s.io/apimachinery/pkg/api/meta"
-	"k8s.io/client-go/discovery"
-	"k8s.io/client-go/rest"
-	"k8s.io/client-go/restmapper"
-	"k8s.io/client-go/tools/clientcmd"
-
-	memory "k8s.io/client-go/discovery/cached"
-)
-
-// RESTClientGetter is a Kubernetes REST client getter implementation that satisfies
-// the genericclioptions.RESTClientGetter interface. This is used primarily for Helm
-// operations in tests.
-type RESTClientGetter struct {
-	clientconfig    clientcmd.ClientConfig
-	restConfig      *rest.Config
-	discoveryClient discovery.CachedDiscoveryInterface
-}
-
-// NewRESTClientGetter creates a new RESTClientGetter from kubeconfig bytes.
-// This is used for Helm operations in tests.
-func NewRESTClientGetter(kubeconfig []byte) (*RESTClientGetter, error) {
-	clientconfig, err := clientcmd.NewClientConfigFromBytes(kubeconfig)
-	if err != nil {
-		return nil, err
-	}
-
-	restConfig, err := clientconfig.ClientConfig()
-	if err != nil {
-		return nil, err
-	}
-
-	dc, err := discovery.NewDiscoveryClientForConfig(restConfig)
-	if err != nil {
-		return nil, err
-	}
-
-	return &RESTClientGetter{
-		clientconfig:    clientconfig,
-		restConfig:      restConfig,
-		discoveryClient: memory.NewMemCacheClient(dc),
-	}, nil
-}
-
-// ToRESTConfig returns the REST config.
-func (r *RESTClientGetter) ToRESTConfig() (*rest.Config, error) {
-	return r.restConfig, nil
-}
-
-// ToDiscoveryClient returns the cached discovery client.
-func (r *RESTClientGetter) ToDiscoveryClient() (discovery.CachedDiscoveryInterface, error) {
-	return r.discoveryClient, nil
-}
-
-// ToRESTMapper returns a REST mapper from the discovery client.
-func (r *RESTClientGetter) ToRESTMapper() (meta.RESTMapper, error) {
-	return restmapper.NewDeferredDiscoveryRESTMapper(r.discoveryClient), nil
-}
-
-// ToRawKubeConfigLoader returns the raw kubeconfig loader.
-func (r *RESTClientGetter) ToRawKubeConfigLoader() clientcmd.ClientConfig {
-	return r.clientconfig
-}

tests/framework/container/helm.go

@@ -1,105 +0,0 @@
-package container
-
-import (
-	"maps"
-	"os"
-	"path"
-	"time"
-
-	"helm.sh/helm/v3/pkg/action"
-	"helm.sh/helm/v3/pkg/chart/loader"
-
-	fwclient "github.com/rancher/k3k/tests/framework/client"
-
-	. "github.com/onsi/ginkgo/v2"
-	. "github.com/onsi/gomega"
-)
-
-// HelmInstaller provides configuration for Helm chart installation.
-type HelmInstaller struct {
-	ChartPath       string
-	Namespace       string
-	ReleaseName     string
-	Timeout         time.Duration
-	Wait            bool
-	ControllerImage string
-	KubeletImage    string
-	KubeconfigPath  string
-}
-
-// InstallK3kChart installs the k3k Helm chart with the specified configuration.
-// It uses the provided RESTClientGetter for authentication and returns the Helm action configuration.
-func (h *HelmInstaller) InstallK3kChart(restClientGetter *fwclient.RESTClientGetter) *action.Configuration {
-	GinkgoHelper()
-
-	// Load chart
-	k3kChart, err := loader.Load(h.ChartPath)
-	Expect(err).To(Not(HaveOccurred()))
-
-	// Initialize Helm action configuration
-	helmActionConfig := new(action.Configuration)
-
-	err = helmActionConfig.Init(restClientGetter, h.Namespace, os.Getenv("HELM_DRIVER"), func(format string, v ...any) {
-		GinkgoWriter.Printf("[Helm] "+format+"\n", v...)
-	})
-	Expect(err).To(Not(HaveOccurred()))
-
-	// Create install action
-	iCli := action.NewInstall(helmActionConfig)
-	iCli.ReleaseName = h.ReleaseName
-	iCli.Namespace = h.Namespace
-	iCli.CreateNamespace = true
-	iCli.Timeout = h.Timeout
-	iCli.Wait = h.Wait
-
-	// Configure controller image
-	controllerMap, _ := k3kChart.Values["controller"].(map[string]any)
-
-	extraEnvArray, _ := controllerMap["extraEnv"].([]map[string]any)
-	extraEnvArray = append(extraEnvArray, map[string]any{
-		"name":  "DEBUG",
-		"value": "true",
-	})
-	controllerMap["extraEnv"] = extraEnvArray
-
-	imageMap, _ := controllerMap["image"].(map[string]any)
-	maps.Copy(imageMap, map[string]any{
-		"repository": h.ControllerImage,
-		"tag":        "dev",
-		"pullPolicy": "IfNotPresent",
-	})
-
-	// Configure agent image
-	agentMap, _ := k3kChart.Values["agent"].(map[string]any)
-	sharedAgentMap, _ := agentMap["shared"].(map[string]any)
-	sharedAgentImageMap, _ := sharedAgentMap["image"].(map[string]any)
-	maps.Copy(sharedAgentImageMap, map[string]any{
-		"repository": h.KubeletImage,
-		"tag":        "dev",
-	})
-
-	// Install chart
-	release, err := iCli.Run(k3kChart, k3kChart.Values)
-	Expect(err).To(Not(HaveOccurred()))
-
-	GinkgoWriter.Printf("Helm release '%s' installed in '%s' namespace\n", release.Name, release.Namespace)
-
-	return helmActionConfig
-}
-
-// NewHelmInstaller creates a new HelmInstaller with default values.
-func NewHelmInstaller(controllerImage, kubeletImage, kubeconfigPath string) *HelmInstaller {
-	pwd, err := os.Getwd()
-	Expect(err).To(Not(HaveOccurred()))
-
-	return &HelmInstaller{
-		ChartPath:       path.Join(pwd, "../../charts/k3k"),
-		Namespace:       "k3k-system",
-		ReleaseName:     "k3k",
-		Timeout:         time.Minute,
-		Wait:            true,
-		ControllerImage: controllerImage,
-		KubeletImage:    kubeletImage,
-		KubeconfigPath:  kubeconfigPath,
-	}
-}

tests/framework/container/k3s.go

@@ -1,63 +0,0 @@
-package container
-
-import (
-	"context"
-	"os"
-	"strings"
-
-	"github.com/testcontainers/testcontainers-go/modules/k3s"
-
-	. "github.com/onsi/ginkgo/v2"
-	. "github.com/onsi/gomega"
-)
-
-// SetupK3s creates and starts a K3s testcontainer with the specified version and loads the provided images.
-// It returns the container instance and a temporary kubeconfig file path.
-// The kubeconfig is automatically cleaned up via DeferCleanup.
-func SetupK3s(ctx context.Context, k3sVersion, controllerImage, kubeletImage string) (*k3s.K3sContainer, string) {
-	GinkgoHelper()
-
-	var (
-		err        error
-		kubeconfig []byte
-	)
-
-	// Normalize version string (replace + with -)
-	k3sVersion = strings.ReplaceAll(k3sVersion, "+", "-")
-
-	// Start K3s container
-	k3sContainer, err := k3s.Run(ctx, "rancher/k3s:"+k3sVersion)
-	Expect(err).To(Not(HaveOccurred()))
-
-	containerIP, err := k3sContainer.ContainerIP(ctx)
-	Expect(err).To(Not(HaveOccurred()))
-
-	GinkgoWriter.Println("K3s containerIP: " + containerIP)
-
-	// Get kubeconfig from container
-	kubeconfig, err = k3sContainer.GetKubeConfig(ctx)
-	Expect(err).To(Not(HaveOccurred()))
-
-	// Write kubeconfig to temp file
-	tmpFile, err := os.CreateTemp("", "kubeconfig-")
-	Expect(err).To(Not(HaveOccurred()))
-
-	_, err = tmpFile.Write(kubeconfig)
-	Expect(err).To(Not(HaveOccurred()))
-	Expect(tmpFile.Close()).To(Succeed())
-
-	kubeconfigPath := tmpFile.Name()
-
-	// Load images into the K3s container
-	err = k3sContainer.LoadImages(ctx, controllerImage+":dev", kubeletImage+":dev")
-	Expect(err).To(Not(HaveOccurred()))
-
-	// Register cleanup
-	DeferCleanup(os.Remove, kubeconfigPath)
-
-	// Set KUBECONFIG environment variable
-	Expect(os.Setenv("KUBECONFIG", kubeconfigPath)).To(Succeed())
-	GinkgoWriter.Printf("KUBECONFIG set to: %s\n", kubeconfigPath)
-
-	return k3sContainer, kubeconfigPath
-}

tests/framework/log/log.go

@@ -1,71 +0,0 @@
-package log
-
-import (
-	"context"
-	"io"
-	"os"
-	"path"
-
-	"k8s.io/client-go/kubernetes"
-	"sigs.k8s.io/controller-runtime/pkg/client"
-
-	corev1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-
-	. "github.com/onsi/ginkgo/v2"
-	. "github.com/onsi/gomega"
-)
-
-// GetK3kPodLogs retrieves logs from the first k3k pod in the specified namespace.
-// This is useful for debugging test failures.
-func GetK3kPodLogs(ctx context.Context, k8sClient client.Client, clientset kubernetes.Interface, namespace string) io.ReadCloser {
-	GinkgoHelper()
-
-	var podList corev1.PodList
-
-	err := k8sClient.List(ctx, &podList, &client.ListOptions{Namespace: namespace})
-	Expect(err).To(Not(HaveOccurred()))
-	Expect(podList.Items).NotTo(BeEmpty())
-
-	k3kPod := podList.Items[0]
-
-	// Fetch complete pod object to access status information
-	pod, err := clientset.CoreV1().Pods(k3kPod.Namespace).Get(ctx, k3kPod.Name, metav1.GetOptions{})
-	Expect(err).To(Not(HaveOccurred()))
-
-	// Detect if the container has been restarted (e.g., for coverage dumping in E2E tests)
-	fetchPrevious := false
-
-	if len(pod.Status.ContainerStatuses) > 0 {
-		containerStatus := pod.Status.ContainerStatuses[0]
-
-		if containerStatus.RestartCount > 0 {
-			fetchPrevious = true
-
-			GinkgoWriter.Printf("Container has been restarted %d time(s), fetching previous logs\n", containerStatus.RestartCount)
-		}
-	}
-
-	req := clientset.CoreV1().Pods(pod.Namespace).GetLogs(pod.Name, &corev1.PodLogOptions{Previous: fetchPrevious})
-	podLogs, err := req.Stream(ctx)
-	Expect(err).To(Not(HaveOccurred()))
-
-	return podLogs
-}
-
-// WriteLogs writes the provided logs to a temporary file with the specified filename.
-// The file is written to os.TempDir() and the full path is logged to GinkgoWriter.
-func WriteLogs(filename string, logs io.ReadCloser) {
-	GinkgoHelper()
-
-	logsStr, err := io.ReadAll(logs)
-	Expect(err).To(Not(HaveOccurred()))
-
-	tempfile := path.Join(os.TempDir(), filename)
-	err = os.WriteFile(tempfile, logsStr, 0o644)
-	Expect(err).To(Not(HaveOccurred()))
-
-	GinkgoWriter.Println("logs written to: " + tempfile)
-
-	_ = logs.Close()
-}