[v1.0] Refactor tests to their own directories (#725)

* Refactor tests to their own directories (#723)

* Move cli tests

* Move e2e tests to their own directory

* Move integration tests

* Fix path within the cli tests

* Move k3k-kubelet tests

* Improve the various make test- options

* Remove dead code from cli tests

* Update development.md with the new make commands

* Remove tests that didn't exist in v1.0

Makefile

@@ -60,24 +60,32 @@ test:	## Run all the tests
 	$(GINKGO) $(GINKGO_FLAGS) --label-filter=$(label-filter)
 
 .PHONY: test-unit
-test-unit:	## Run the unit tests (skips the e2e)
+test-unit:	## Run the unit tests (skips the e2e and integration tests)
 	$(GINKGO) $(GINKGO_FLAGS) --skip-file=tests/*
 
-.PHONY: test-controller
-test-controller:	## Run the controller tests (pkg/controller)
-	$(GINKGO) $(GINKGO_FLAGS) pkg/controller
+.PHONY: test-kubelet
+test-kubelet:	## Run the k3k-kubelet controller tests (tests/integration/k3k-kubelet)
+	$(GINKGO) $(GINKGO_FLAGS) tests/integration/k3k-kubelet
 
-.PHONY: test-kubelet-controller
-test-kubelet-controller:	## Run the controller tests (pkg/controller)
-	$(GINKGO) $(GINKGO_FLAGS) k3k-kubelet/controller
+.PHONY: test-policy
+test-policy:	## Run the policy controller tests (tests/integration/policy)
+	$(GINKGO) $(GINKGO_FLAGS) tests/integration/policy
+
+.PHONY: test-cluster
+test-cluster:	## Run the cluster controller tests (tests/integration/cluster)
+	$(GINKGO) $(GINKGO_FLAGS) tests/integration/cluster
+
+.PHONY: test-integration
+test-integration:	## Run the controller tests that use envtest (tests/integration)
+	$(GINKGO) $(GINKGO_FLAGS) tests/integration
 
 .PHONY: test-e2e
 test-e2e:	## Run the e2e tests
-	$(GINKGO) $(GINKGO_FLAGS) --label-filter="$(E2E_LABEL_FILTER)" tests
+	$(GINKGO) $(GINKGO_FLAGS) --label-filter="$(E2E_LABEL_FILTER)" tests/e2e
 
 .PHONY: test-cli
 test-cli:	## Run the cli tests
-	$(GINKGO) $(GINKGO_FLAGS) --label-filter=cli --flake-attempts=3 tests
+	$(GINKGO) $(GINKGO_FLAGS) --flake-attempts=3 tests/cli
 
 .PHONY: generate
 generate:	## Generate the CRDs specs

docs/development.md

@@ -51,9 +51,11 @@ To see all the available Make commands you can run `make help`, i.e:
   package                        Package the k3k and k3k-kubelet Docker images
   push                           Push the K3k images to the registry
   test                           Run all the tests
-  test-unit                      Run the unit tests (skips the e2e)
-  test-controller                Run the controller tests (pkg/controller)
-  test-kubelet-controller        Run the controller tests (pkg/controller)
+  test-unit                      Run the unit tests (skips the e2e and integration tests)
+  test-kubelet                   Run the k3k-kubelet controller tests (tests/integration/k3k-kubelet)
+  test-policy                    Run the policy controller tests (tests/integration/policy)
+  test-cluster                   Run the cluster controller tests (tests/integration/cluster)
+  test-integration               Run the controller tests (pkg/controller)
   test-e2e                       Run the e2e tests
   test-cli                       Run the cli tests
   generate                       Generate the CRDs specs

tests/cli/cli_test.go

@@ -1,4 +1,4 @@
-package k3k_test
+package cli_test
 
 import (
 	"bytes"

tests/cli/common_test.go

@@ -0,0 +1,55 @@
+package cli_test
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"sync"
+
+	"k8s.io/utils/ptr"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+func NewNamespace() *v1.Namespace {
+	GinkgoHelper()
+
+	namespace := &v1.Namespace{ObjectMeta: metav1.ObjectMeta{GenerateName: "ns-", Labels: map[string]string{"e2e": "true"}}}
+	namespace, err := k8s.CoreV1().Namespaces().Create(context.Background(), namespace, metav1.CreateOptions{})
+	Expect(err).To(Not(HaveOccurred()))
+
+	return namespace
+}
+
+func DeleteNamespaces(names ...string) {
+	GinkgoHelper()
+
+	if _, found := os.LookupEnv("KEEP_NAMESPACES"); found {
+		By(fmt.Sprintf("Keeping namespace %v", names))
+		return
+	}
+
+	wg := sync.WaitGroup{}
+	wg.Add(len(names))
+
+	for _, name := range names {
+		go func() {
+			defer wg.Done()
+			defer GinkgoRecover()
+
+			By(fmt.Sprintf("Deleting namespace %s", name))
+
+			err := k8s.CoreV1().Namespaces().Delete(context.Background(), name, metav1.DeleteOptions{
+				GracePeriodSeconds: ptr.To[int64](0),
+			})
+			Expect(client.IgnoreNotFound(err)).To(Not(HaveOccurred()))
+		}()
+	}
+
+	wg.Wait()
+}

tests/cli/k8s_restclientgetter_test.go

@@ -0,0 +1,56 @@
+package cli_test
+
+import (
+	"k8s.io/apimachinery/pkg/api/meta"
+	"k8s.io/client-go/discovery"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/restmapper"
+	"k8s.io/client-go/tools/clientcmd"
+
+	memory "k8s.io/client-go/discovery/cached"
+)
+
+type RESTClientGetter struct {
+	clientconfig    clientcmd.ClientConfig
+	restConfig      *rest.Config
+	discoveryClient discovery.CachedDiscoveryInterface
+}
+
+func NewRESTClientGetter(kubeconfig []byte) (*RESTClientGetter, error) {
+	clientconfig, err := clientcmd.NewClientConfigFromBytes([]byte(kubeconfig))
+	if err != nil {
+		return nil, err
+	}
+
+	restConfig, err := clientconfig.ClientConfig()
+	if err != nil {
+		return nil, err
+	}
+
+	dc, err := discovery.NewDiscoveryClientForConfig(restConfig)
+	if err != nil {
+		return nil, err
+	}
+
+	return &RESTClientGetter{
+		clientconfig:    clientconfig,
+		restConfig:      restConfig,
+		discoveryClient: memory.NewMemCacheClient(dc),
+	}, nil
+}
+
+func (r *RESTClientGetter) ToRESTConfig() (*rest.Config, error) {
+	return r.restConfig, nil
+}
+
+func (r *RESTClientGetter) ToDiscoveryClient() (discovery.CachedDiscoveryInterface, error) {
+	return r.discoveryClient, nil
+}
+
+func (r *RESTClientGetter) ToRESTMapper() (meta.RESTMapper, error) {
+	return restmapper.NewDeferredDiscoveryRESTMapper(r.discoveryClient), nil
+}
+
+func (r *RESTClientGetter) ToRawKubeConfigLoader() clientcmd.ClientConfig {
+	return r.clientconfig
+}

tests/cli/tests_suite_test.go

@@ -0,0 +1,234 @@
+package cli_test
+
+import (
+	"context"
+	"io"
+	"maps"
+	"os"
+	"path"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/go-logr/zapr"
+	"github.com/testcontainers/testcontainers-go"
+	"github.com/testcontainers/testcontainers-go/modules/k3s"
+	"go.uber.org/zap"
+	"helm.sh/helm/v3/pkg/action"
+	"helm.sh/helm/v3/pkg/chart/loader"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/log"
+
+	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+
+	"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+const (
+	k3kNamespace = "k3k-system"
+
+	k3sVersion    = "v1.35.2-k3s1"
+	k3sOldVersion = "v1.35.0-k3s1"
+)
+
+func TestTests(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Tests Suite")
+}
+
+var (
+	k3sContainer     *k3s.K3sContainer
+	restcfg          *rest.Config
+	k8s              *kubernetes.Clientset
+	k8sClient        client.Client
+	kubeconfigPath   string
+	helmActionConfig *action.Configuration
+)
+
+var _ = BeforeSuite(func() {
+	ctx := context.Background()
+
+	_, dockerInstallEnabled := os.LookupEnv("K3K_DOCKER_INSTALL")
+
+	if dockerInstallEnabled {
+		repo := os.Getenv("REPO")
+		if repo == "" {
+			repo = "rancher"
+		}
+
+		installK3SDocker(ctx, repo+"/k3k", repo+"/k3k-kubelet")
+		initKubernetesClient()
+		installK3kChart(repo+"/k3k", repo+"/k3k-kubelet")
+	} else {
+		initKubernetesClient()
+	}
+})
+
+func initKubernetesClient() {
+	var (
+		err        error
+		kubeconfig []byte
+	)
+
+	logger, err := zap.NewDevelopment()
+	Expect(err).NotTo(HaveOccurred())
+
+	log.SetLogger(zapr.NewLogger(logger))
+
+	kubeconfigPath := os.Getenv("KUBECONFIG")
+	Expect(kubeconfigPath).To(Not(BeEmpty()))
+
+	kubeconfig, err = os.ReadFile(kubeconfigPath)
+	Expect(err).To(Not(HaveOccurred()))
+
+	restcfg, err = clientcmd.RESTConfigFromKubeConfig(kubeconfig)
+	Expect(err).To(Not(HaveOccurred()))
+
+	k8s, err = kubernetes.NewForConfig(restcfg)
+	Expect(err).To(Not(HaveOccurred()))
+
+	scheme := buildScheme()
+	k8sClient, err = client.New(restcfg, client.Options{Scheme: scheme})
+	Expect(err).NotTo(HaveOccurred())
+}
+
+func buildScheme() *runtime.Scheme {
+	scheme := runtime.NewScheme()
+
+	err := clientgoscheme.AddToScheme(scheme)
+	Expect(err).NotTo(HaveOccurred())
+	err = v1beta1.AddToScheme(scheme)
+	Expect(err).NotTo(HaveOccurred())
+
+	return scheme
+}
+
+func installK3SDocker(ctx context.Context, controllerImage, kubeletImage string) {
+	var (
+		err        error
+		kubeconfig []byte
+	)
+
+	k3sHostVersion := os.Getenv("K3S_HOST_VERSION")
+	if k3sHostVersion == "" {
+		k3sHostVersion = k3sVersion
+	}
+
+	k3sHostVersion = strings.ReplaceAll(k3sHostVersion, "+", "-")
+
+	k3sContainer, err = k3s.Run(ctx, "rancher/k3s:"+k3sHostVersion)
+	Expect(err).To(Not(HaveOccurred()))
+
+	containerIP, err := k3sContainer.ContainerIP(ctx)
+	Expect(err).To(Not(HaveOccurred()))
+
+	GinkgoWriter.Println("K3s containerIP: " + containerIP)
+
+	kubeconfig, err = k3sContainer.GetKubeConfig(context.Background())
+	Expect(err).To(Not(HaveOccurred()))
+
+	tmpFile, err := os.CreateTemp("", "kubeconfig-")
+	Expect(err).To(Not(HaveOccurred()))
+
+	_, err = tmpFile.Write(kubeconfig)
+	Expect(err).To(Not(HaveOccurred()))
+	Expect(tmpFile.Close()).To(Succeed())
+	kubeconfigPath = tmpFile.Name()
+
+	err = k3sContainer.LoadImages(ctx, controllerImage+":dev", kubeletImage+":dev")
+	Expect(err).To(Not(HaveOccurred()))
+	DeferCleanup(os.Remove, kubeconfigPath)
+
+	Expect(os.Setenv("KUBECONFIG", kubeconfigPath)).To(Succeed())
+	GinkgoWriter.Printf("KUBECONFIG set to: %s\n", kubeconfigPath)
+}
+
+func installK3kChart(controllerImage, kubeletImage string) {
+	pwd, err := os.Getwd()
+	Expect(err).To(Not(HaveOccurred()))
+
+	k3kChart, err := loader.Load(path.Join(pwd, "../../charts/k3k"))
+	Expect(err).To(Not(HaveOccurred()))
+
+	helmActionConfig = new(action.Configuration)
+
+	kubeconfig, err := os.ReadFile(kubeconfigPath)
+	Expect(err).To(Not(HaveOccurred()))
+
+	restClientGetter, err := NewRESTClientGetter(kubeconfig)
+	Expect(err).To(Not(HaveOccurred()))
+
+	err = helmActionConfig.Init(restClientGetter, k3kNamespace, os.Getenv("HELM_DRIVER"), func(format string, v ...any) {
+		GinkgoWriter.Printf("[Helm] "+format+"\n", v...)
+	})
+	Expect(err).To(Not(HaveOccurred()))
+
+	iCli := action.NewInstall(helmActionConfig)
+	iCli.ReleaseName = "k3k"
+	iCli.Namespace = k3kNamespace
+	iCli.CreateNamespace = true
+	iCli.Timeout = time.Minute
+	iCli.Wait = true
+
+	controllerMap, _ := k3kChart.Values["controller"].(map[string]any)
+
+	extraEnvArray, _ := controllerMap["extraEnv"].([]map[string]any)
+	extraEnvArray = append(extraEnvArray, map[string]any{
+		"name":  "DEBUG",
+		"value": "true",
+	})
+	controllerMap["extraEnv"] = extraEnvArray
+
+	imageMap, _ := controllerMap["image"].(map[string]any)
+	maps.Copy(imageMap, map[string]any{
+		"repository": controllerImage,
+		"tag":        "dev",
+		"pullPolicy": "IfNotPresent",
+	})
+
+	agentMap, _ := k3kChart.Values["agent"].(map[string]any)
+	sharedAgentMap, _ := agentMap["shared"].(map[string]any)
+	sharedAgentImageMap, _ := sharedAgentMap["image"].(map[string]any)
+	maps.Copy(sharedAgentImageMap, map[string]any{
+		"repository": kubeletImage,
+		"tag":        "dev",
+	})
+
+	release, err := iCli.Run(k3kChart, k3kChart.Values)
+	Expect(err).To(Not(HaveOccurred()))
+
+	GinkgoWriter.Printf("Helm release '%s' installed in '%s' namespace\n", release.Name, release.Namespace)
+}
+
+var _ = AfterSuite(func() {
+	ctx := context.Background()
+
+	if k3sContainer != nil {
+		// dump k3s logs
+		k3sLogs, err := k3sContainer.Logs(ctx)
+		Expect(err).To(Not(HaveOccurred()))
+		writeLogs("k3s.log", k3sLogs)
+
+		testcontainers.CleanupContainer(GinkgoTB(), k3sContainer)
+	}
+})
+
+func writeLogs(filename string, logs io.ReadCloser) {
+	logsStr, err := io.ReadAll(logs)
+	Expect(err).To(Not(HaveOccurred()))
+
+	tempfile := path.Join(os.TempDir(), filename)
+	err = os.WriteFile(tempfile, []byte(logsStr), 0o644)
+	Expect(err).To(Not(HaveOccurred()))
+
+	GinkgoWriter.Println("logs written to: " + filename)
+
+	_ = logs.Close()
+}