Refactoring of tests moving common logic in tests/framework (#769)

* first commit

* refactor: update golangci configuration and reorder imports in namespace.go

* refactor: update golangci configuration for linters and formatters

* check restart for logs fetch

* rverted a bit the structure

* requested changes

* WriteLogs rename
This commit is contained in:
Enrico Candino
2026-04-13 12:50:38 +02:00
committed by GitHub
parent cd4e5340e3
commit ba1648e560
33 changed files with 649 additions and 642 deletions
+20 -13
View File
@@ -2,26 +2,33 @@ version: "2"
linters:
enable:
- misspell
- wsl_v5
- misspell
- wsl_v5
exclusions:
rules:
# Allow dot imports for Ginkgo/Gomega in test framework
- path: tests/framework/
text: "dot imports"
linters:
- staticcheck
formatters:
enable:
- gci
- gofmt
- gofumpt
- gci
- gofmt
- gofumpt
settings:
gci:
# The default order is `standard > default > custom > blank > dot > alias > localmodule`.
custom-order: true
sections:
- standard
- default
- alias
- localmodule
- dot
- blank
- standard
- default
- alias
- localmodule
- dot
- blank
gofmt:
rewrite-rules:
- pattern: 'interface{}'
replacement: 'any'
- pattern: 'interface{}'
replacement: 'any'
+1
View File
@@ -253,6 +253,7 @@ func (s *SharedAgent) podSpec(ctx context.Context) v1.PodSpec {
}
podSpec.RuntimeClassName = runtimeClassName
return podSpec
}
+1
View File
@@ -137,6 +137,7 @@ func (v *VirtualAgent) deployment(ctx context.Context) error {
func (v *VirtualAgent) podSpec(ctx context.Context, image, name string) v1.PodSpec {
log := ctrl.LoggerFrom(ctx)
var limit v1.ResourceList
args := v.cluster.Spec.AgentArgs
+19 -18
View File
@@ -13,6 +13,7 @@ import (
"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
"github.com/rancher/k3k/pkg/controller/policy"
fwk3k "github.com/rancher/k3k/tests/framework/k3k"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
@@ -54,11 +55,11 @@ var _ = When("using the k3kcli", Label("cli"), func() {
)
clusterName := "cluster-" + rand.String(5)
namespace := NewNamespace()
namespace := fwk3k.CreateNamespace(k8s)
clusterNamespace := namespace.Name
DeferCleanup(func() {
DeleteNamespaces(namespace.Name)
fwk3k.DeleteNamespaces(k8s, namespace.Name)
})
_, stderr, err = K3kcli("cluster", "create", "--namespace", clusterNamespace, clusterName)
@@ -93,11 +94,11 @@ var _ = When("using the k3kcli", Label("cli"), func() {
)
clusterName := "cluster-" + rand.String(5)
namespace := NewNamespace()
namespace := fwk3k.CreateNamespace(k8s)
clusterNamespace := namespace.Name
DeferCleanup(func() {
DeleteNamespaces(clusterNamespace)
fwk3k.DeleteNamespaces(k8s, clusterNamespace)
})
_, stderr, err = K3kcli("cluster", "create", "--version", k3sVersion, clusterName)
@@ -142,11 +143,11 @@ var _ = When("using the k3kcli", Label("cli"), func() {
err error
)
namespace := NewNamespace()
namespace := fwk3k.CreateNamespace(k8s)
namespaceName := namespace.Name
DeferCleanup(func() {
DeleteNamespaces(namespaceName)
fwk3k.DeleteNamespaces(k8s, namespaceName)
})
By("Creating a policy and binding to a namespace")
@@ -213,11 +214,11 @@ var _ = When("using the k3kcli", Label("cli"), func() {
clusterName := "cluster-" + rand.String(5)
namespace := NewNamespace()
namespace := fwk3k.CreateNamespace(k8s)
clusterNamespace := namespace.Name
DeferCleanup(func() {
DeleteNamespaces(clusterNamespace)
fwk3k.DeleteNamespaces(k8s, clusterNamespace)
})
// Create the cluster first
@@ -247,11 +248,11 @@ var _ = When("using the k3kcli", Label("cli"), func() {
clusterName := "cluster-" + rand.String(5)
namespace := NewNamespace()
namespace := fwk3k.CreateNamespace(k8s)
clusterNamespace := namespace.Name
DeferCleanup(func() {
DeleteNamespaces(clusterNamespace)
fwk3k.DeleteNamespaces(k8s, clusterNamespace)
})
// Create the cluster with initial version
@@ -280,11 +281,11 @@ var _ = When("using the k3kcli", Label("cli"), func() {
clusterName := "cluster-" + rand.String(5)
namespace := NewNamespace()
namespace := fwk3k.CreateNamespace(k8s)
clusterNamespace := namespace.Name
DeferCleanup(func() {
DeleteNamespaces(clusterNamespace)
fwk3k.DeleteNamespaces(k8s, clusterNamespace)
})
// Create the cluster with a version
@@ -325,11 +326,11 @@ var _ = When("using the k3kcli", Label("cli"), func() {
clusterName := "cluster-" + rand.String(5)
namespace := NewNamespace()
namespace := fwk3k.CreateNamespace(k8s)
clusterNamespace := namespace.Name
DeferCleanup(func() {
DeleteNamespaces(clusterNamespace)
fwk3k.DeleteNamespaces(k8s, clusterNamespace)
})
// Create the cluster first
@@ -359,11 +360,11 @@ var _ = When("using the k3kcli", Label("cli"), func() {
clusterName := "cluster-" + rand.String(5)
namespace := NewNamespace()
namespace := fwk3k.CreateNamespace(k8s)
clusterNamespace := namespace.Name
DeferCleanup(func() {
DeleteNamespaces(clusterNamespace)
fwk3k.DeleteNamespaces(k8s, clusterNamespace)
})
// Create the cluster first
@@ -394,11 +395,11 @@ var _ = When("using the k3kcli", Label("cli"), func() {
)
clusterName := "cluster-" + rand.String(5)
namespace := NewNamespace()
namespace := fwk3k.CreateNamespace(k8s)
clusterNamespace := namespace.Name
DeferCleanup(func() {
DeleteNamespaces(clusterNamespace)
fwk3k.DeleteNamespaces(k8s, clusterNamespace)
})
_, stderr, err = K3kcli("cluster", "create", "--namespace", clusterNamespace, clusterName)
-55
View File
@@ -1,55 +0,0 @@
package cli_test
import (
"context"
"fmt"
"os"
"sync"
"k8s.io/utils/ptr"
"sigs.k8s.io/controller-runtime/pkg/client"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
func NewNamespace() *v1.Namespace {
GinkgoHelper()
namespace := &v1.Namespace{ObjectMeta: metav1.ObjectMeta{GenerateName: "ns-", Labels: map[string]string{"e2e": "true"}}}
namespace, err := k8s.CoreV1().Namespaces().Create(context.Background(), namespace, metav1.CreateOptions{})
Expect(err).To(Not(HaveOccurred()))
return namespace
}
func DeleteNamespaces(names ...string) {
GinkgoHelper()
if _, found := os.LookupEnv("KEEP_NAMESPACES"); found {
By(fmt.Sprintf("Keeping namespace %v", names))
return
}
wg := sync.WaitGroup{}
wg.Add(len(names))
for _, name := range names {
go func() {
defer wg.Done()
defer GinkgoRecover()
By(fmt.Sprintf("Deleting namespace %s", name))
err := k8s.CoreV1().Namespaces().Delete(context.Background(), name, metav1.DeleteOptions{
GracePeriodSeconds: ptr.To[int64](0),
})
Expect(client.IgnoreNotFound(err)).To(Not(HaveOccurred()))
}()
}
wg.Wait()
}
+27 -175
View File
@@ -2,31 +2,18 @@ package cli_test
import (
"context"
"io"
"maps"
"os"
"path"
"strings"
"testing"
"time"
"github.com/go-logr/zapr"
"github.com/testcontainers/testcontainers-go"
"github.com/testcontainers/testcontainers-go/modules/k3s"
"go.uber.org/zap"
"helm.sh/helm/v3/pkg/action"
"helm.sh/helm/v3/pkg/chart/loader"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/log"
v1 "k8s.io/api/core/v1"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
fwclient "github.com/rancher/k3k/tests/framework/client"
fwcontainer "github.com/rancher/k3k/tests/framework/container"
fwlog "github.com/rancher/k3k/tests/framework/log"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
@@ -45,12 +32,11 @@ func TestTests(t *testing.T) {
}
var (
k3sContainer *k3s.K3sContainer
restcfg *rest.Config
k8s *kubernetes.Clientset
k8sClient client.Client
kubeconfigPath string
helmActionConfig *action.Configuration
k3sContainer *k3s.K3sContainer
restcfg *rest.Config
k8s *kubernetes.Clientset
k8sClient client.Client
kubeconfigPath string
)
var _ = BeforeSuite(func() {
@@ -65,147 +51,41 @@ var _ = BeforeSuite(func() {
}
installK3SDocker(ctx, repo+"/k3k", repo+"/k3k-kubelet")
initKubernetesClient()
initKubernetesClient(ctx)
installK3kChart(repo+"/k3k", repo+"/k3k-kubelet")
} else {
initKubernetesClient()
initKubernetesClient(ctx)
}
})
func initKubernetesClient() {
var (
err error
kubeconfig []byte
)
logger, err := zap.NewDevelopment()
Expect(err).NotTo(HaveOccurred())
log.SetLogger(zapr.NewLogger(logger))
kubeconfigPath := os.Getenv("KUBECONFIG")
Expect(kubeconfigPath).To(Not(BeEmpty()))
kubeconfig, err = os.ReadFile(kubeconfigPath)
Expect(err).To(Not(HaveOccurred()))
restcfg, err = clientcmd.RESTConfigFromKubeConfig(kubeconfig)
Expect(err).To(Not(HaveOccurred()))
k8s, err = kubernetes.NewForConfig(restcfg)
Expect(err).To(Not(HaveOccurred()))
scheme := buildScheme()
k8sClient, err = client.New(restcfg, client.Options{Scheme: scheme})
Expect(err).NotTo(HaveOccurred())
}
func buildScheme() *runtime.Scheme {
scheme := runtime.NewScheme()
err := clientgoscheme.AddToScheme(scheme)
Expect(err).NotTo(HaveOccurred())
err = v1beta1.AddToScheme(scheme)
Expect(err).NotTo(HaveOccurred())
return scheme
}
func installK3SDocker(ctx context.Context, controllerImage, kubeletImage string) {
var (
err error
kubeconfig []byte
)
k3sHostVersion := os.Getenv("K3S_HOST_VERSION")
if k3sHostVersion == "" {
k3sHostVersion = k3sVersion
}
k3sHostVersion = strings.ReplaceAll(k3sHostVersion, "+", "-")
k3sContainer, kubeconfigPath = fwcontainer.SetupK3s(ctx, k3sHostVersion, controllerImage, kubeletImage)
}
k3sContainer, err = k3s.Run(ctx, "rancher/k3s:"+k3sHostVersion)
Expect(err).To(Not(HaveOccurred()))
func initKubernetesClient(ctx context.Context) {
scheme := fwclient.NewScheme()
config, err := fwclient.InitFromKubeconfig(ctx, scheme, k3sContainer)
Expect(err).NotTo(HaveOccurred())
containerIP, err := k3sContainer.ContainerIP(ctx)
Expect(err).To(Not(HaveOccurred()))
GinkgoWriter.Println("K3s containerIP: " + containerIP)
kubeconfig, err = k3sContainer.GetKubeConfig(context.Background())
Expect(err).To(Not(HaveOccurred()))
tmpFile, err := os.CreateTemp("", "kubeconfig-")
Expect(err).To(Not(HaveOccurred()))
_, err = tmpFile.Write(kubeconfig)
Expect(err).To(Not(HaveOccurred()))
Expect(tmpFile.Close()).To(Succeed())
kubeconfigPath = tmpFile.Name()
err = k3sContainer.LoadImages(ctx, controllerImage+":dev", kubeletImage+":dev")
Expect(err).To(Not(HaveOccurred()))
DeferCleanup(os.Remove, kubeconfigPath)
Expect(os.Setenv("KUBECONFIG", kubeconfigPath)).To(Succeed())
GinkgoWriter.Printf("KUBECONFIG set to: %s\n", kubeconfigPath)
restcfg = config.RestConfig
k8s = config.Clientset
k8sClient = config.Client
}
func installK3kChart(controllerImage, kubeletImage string) {
pwd, err := os.Getwd()
Expect(err).To(Not(HaveOccurred()))
k3kChart, err := loader.Load(path.Join(pwd, "../../charts/k3k"))
Expect(err).To(Not(HaveOccurred()))
helmActionConfig = new(action.Configuration)
installer := fwcontainer.NewHelmInstaller(controllerImage, kubeletImage, kubeconfigPath)
kubeconfig, err := os.ReadFile(kubeconfigPath)
Expect(err).To(Not(HaveOccurred()))
Expect(err).NotTo(HaveOccurred())
restClientGetter, err := NewRESTClientGetter(kubeconfig)
Expect(err).To(Not(HaveOccurred()))
restClientGetter, err := fwclient.NewRESTClientGetter(kubeconfig)
Expect(err).NotTo(HaveOccurred())
err = helmActionConfig.Init(restClientGetter, k3kNamespace, os.Getenv("HELM_DRIVER"), func(format string, v ...any) {
GinkgoWriter.Printf("[Helm] "+format+"\n", v...)
})
Expect(err).To(Not(HaveOccurred()))
iCli := action.NewInstall(helmActionConfig)
iCli.ReleaseName = "k3k"
iCli.Namespace = k3kNamespace
iCli.CreateNamespace = true
iCli.Timeout = time.Minute
iCli.Wait = true
controllerMap, _ := k3kChart.Values["controller"].(map[string]any)
extraEnvArray, _ := controllerMap["extraEnv"].([]map[string]any)
extraEnvArray = append(extraEnvArray, map[string]any{
"name": "DEBUG",
"value": "true",
})
controllerMap["extraEnv"] = extraEnvArray
imageMap, _ := controllerMap["image"].(map[string]any)
maps.Copy(imageMap, map[string]any{
"repository": controllerImage,
"tag": "dev",
"pullPolicy": "IfNotPresent",
})
agentMap, _ := k3kChart.Values["agent"].(map[string]any)
sharedAgentMap, _ := agentMap["shared"].(map[string]any)
sharedAgentImageMap, _ := sharedAgentMap["image"].(map[string]any)
maps.Copy(sharedAgentImageMap, map[string]any{
"repository": kubeletImage,
"tag": "dev",
})
release, err := iCli.Run(k3kChart, k3kChart.Values)
Expect(err).To(Not(HaveOccurred()))
GinkgoWriter.Printf("Helm release '%s' installed in '%s' namespace\n", release.Name, release.Namespace)
installer.InstallK3kChart(restClientGetter)
}
var _ = AfterSuite(func() {
@@ -215,40 +95,12 @@ var _ = AfterSuite(func() {
// dump k3s logs
k3sLogs, err := k3sContainer.Logs(ctx)
Expect(err).To(Not(HaveOccurred()))
writeLogs("k3s.log", k3sLogs)
fwlog.WriteLogs("k3s.log", k3sLogs)
// dump k3k controller logs
k3kLogs := getK3kLogs(ctx)
writeLogs("k3k.log", k3kLogs)
k3kLogs := fwlog.GetK3kPodLogs(ctx, k8sClient, k8s, k3kNamespace)
fwlog.WriteLogs("k3k.log", k3kLogs)
testcontainers.CleanupContainer(GinkgoTB(), k3sContainer)
}
})
func getK3kLogs(ctx context.Context) io.ReadCloser {
var podList v1.PodList
err := k8sClient.List(ctx, &podList, &client.ListOptions{Namespace: k3kNamespace})
Expect(err).To(Not(HaveOccurred()))
Expect(podList.Items).NotTo(BeEmpty())
k3kPod := podList.Items[0]
req := k8s.CoreV1().Pods(k3kPod.Namespace).GetLogs(k3kPod.Name, &v1.PodLogOptions{})
podLogs, err := req.Stream(ctx)
Expect(err).To(Not(HaveOccurred()))
return podLogs
}
func writeLogs(filename string, logs io.ReadCloser) {
logsStr, err := io.ReadAll(logs)
Expect(err).To(Not(HaveOccurred()))
tempfile := path.Join(os.TempDir(), filename)
err = os.WriteFile(tempfile, []byte(logsStr), 0o644)
Expect(err).To(Not(HaveOccurred()))
GinkgoWriter.Println("logs written to: " + filename)
_ = logs.Close()
}
+8 -7
View File
@@ -9,6 +9,7 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
fwk3k "github.com/rancher/k3k/tests/framework/k3k"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
@@ -27,14 +28,14 @@ var _ = When("a cluster with secretMounts configuration is used to load addons",
BeforeEach(func() {
ctx := context.Background()
namespace := NewNamespace()
namespace := fwk3k.CreateNamespace(k8s)
// Create the addon secret
err := createAddonSecret(ctx, namespace.Name)
Expect(err).ToNot(HaveOccurred())
DeferCleanup(func() {
DeleteNamespaces(namespace.Name)
fwk3k.DeleteNamespaces(k8s, namespace.Name)
})
cluster := NewCluster(namespace.Name)
@@ -96,14 +97,14 @@ var _ = When("a cluster with addon configuration is used with addons secret in t
BeforeEach(func() {
ctx := context.Background()
namespace := NewNamespace()
namespace := fwk3k.CreateNamespace(k8s)
// Create the addon secret
err := createAddonSecret(ctx, namespace.Name)
Expect(err).ToNot(HaveOccurred())
DeferCleanup(func() {
DeleteNamespaces(namespace.Name)
fwk3k.DeleteNamespaces(k8s, namespace.Name)
})
cluster := NewCluster(namespace.Name)
@@ -154,15 +155,15 @@ var _ = When("a cluster with addon configuration is used with addons secret in t
BeforeEach(func() {
ctx := context.Background()
namespace := NewNamespace()
secretNamespace := NewNamespace()
namespace := fwk3k.CreateNamespace(k8s)
secretNamespace := fwk3k.CreateNamespace(k8s)
// Create the addon secret
err := createAddonSecret(ctx, secretNamespace.Name)
Expect(err).ToNot(HaveOccurred())
DeferCleanup(func() {
DeleteNamespaces(namespace.Name, secretNamespace.Name)
fwk3k.DeleteNamespaces(k8s, namespace.Name, secretNamespace.Name)
})
cluster := NewCluster(namespace.Name)
+2 -1
View File
@@ -12,6 +12,7 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/rancher/k3k/k3k-kubelet/translate"
fwk3k "github.com/rancher/k3k/tests/framework/k3k"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
@@ -28,7 +29,7 @@ var _ = Context("In a shared cluster", Label(e2eTestLabel), Ordered, func() {
translator = translate.NewHostTranslator(virtualCluster.Cluster)
DeferCleanup(func() {
DeleteNamespaces(virtualCluster.Cluster.Namespace)
fwk3k.DeleteNamespaces(k8s, virtualCluster.Cluster.Namespace)
})
})
+3 -2
View File
@@ -6,6 +6,7 @@ import (
"strings"
"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
fwk3k "github.com/rancher/k3k/tests/framework/k3k"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
@@ -17,10 +18,10 @@ var _ = When("a cluster with custom certificates is installed with individual ce
BeforeEach(func() {
ctx := context.Background()
namespace := NewNamespace()
namespace := fwk3k.CreateNamespace(k8s)
DeferCleanup(func() {
DeleteNamespaces(namespace.Name)
fwk3k.DeleteNamespaces(k8s, namespace.Name)
})
// create custom cert secret
+4 -2
View File
@@ -5,6 +5,8 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
fwk3k "github.com/rancher/k3k/tests/framework/k3k"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
@@ -13,10 +15,10 @@ var _ = When("creating a shared mode cluster", Label(e2eTestLabel), Label(slowTe
var virtualCluster *VirtualCluster
BeforeEach(func() {
namespace := NewNamespace()
namespace := fwk3k.CreateNamespace(k8s)
DeferCleanup(func() {
DeleteNamespaces(namespace.Name)
fwk3k.DeleteNamespaces(k8s, namespace.Name)
})
cluster := NewCluster(namespace.Name)
+3 -1
View File
@@ -1,6 +1,8 @@
package k3k_test
import (
fwk3k "github.com/rancher/k3k/tests/framework/k3k"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
@@ -18,7 +20,7 @@ var _ = When("two virtual clusters are installed", Label(e2eTestLabel), Label(ne
})
AfterEach(func() {
DeleteNamespaces(cluster1.Cluster.Namespace, cluster2.Cluster.Namespace)
fwk3k.DeleteNamespaces(k8s, cluster1.Cluster.Namespace, cluster2.Cluster.Namespace)
})
It("can create pods in each of them that are isolated", func() {
+5 -4
View File
@@ -14,6 +14,7 @@ import (
"github.com/rancher/k3k/k3k-kubelet/translate"
"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
fwk3k "github.com/rancher/k3k/tests/framework/k3k"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
@@ -26,7 +27,7 @@ var _ = When("an ephemeral cluster is installed", Label(e2eTestLabel), Label(per
virtualCluster = NewVirtualCluster()
DeferCleanup(func() {
DeleteNamespaces(virtualCluster.Cluster.Namespace)
fwk3k.DeleteNamespaces(k8s, virtualCluster.Cluster.Namespace)
})
})
@@ -119,7 +120,7 @@ var _ = When("a dynamic cluster is installed", Label(e2eTestLabel), Label(persis
})
AfterEach(func() {
DeleteNamespaces(virtualCluster.Cluster.Namespace)
fwk3k.DeleteNamespaces(k8s, virtualCluster.Cluster.Namespace)
})
It("can create a nginx pod", func() {
@@ -152,10 +153,10 @@ var _ = When("a dynamic cluster is installed", Label(e2eTestLabel), Label(persis
It("can delete a HA cluster", func() {
ctx := context.Background()
namespace := NewNamespace()
namespace := fwk3k.CreateNamespace(k8s)
DeferCleanup(func() {
DeleteNamespaces(virtualCluster.Cluster.Namespace)
fwk3k.DeleteNamespaces(k8s, virtualCluster.Cluster.Namespace)
})
cluster := NewCluster(namespace.Name)
+2 -1
View File
@@ -14,6 +14,7 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/rancher/k3k/k3k-kubelet/translate"
fwk3k "github.com/rancher/k3k/tests/framework/k3k"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
@@ -30,7 +31,7 @@ var _ = Context("In a shared cluster", Label(e2eTestLabel), Ordered, func() {
translator = translate.NewHostTranslator(virtualCluster.Cluster)
DeferCleanup(func() {
DeleteNamespaces(virtualCluster.Cluster.Namespace)
fwk3k.DeleteNamespaces(k8s, virtualCluster.Cluster.Namespace)
})
})
@@ -15,6 +15,7 @@ import (
"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
"github.com/rancher/k3k/pkg/controller/cluster"
"github.com/rancher/k3k/pkg/controller/policy"
fwk3k "github.com/rancher/k3k/tests/framework/k3k"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
@@ -109,7 +110,7 @@ var _ = When("a shared mode cluster is created in a namespace with a policy", Or
}
DeferCleanup(func() {
DeleteNamespaces(ns.Name)
fwk3k.DeleteNamespaces(k8s, ns.Name)
err = k8s.StorageV1().StorageClasses().Delete(ctx, storageClassEnabled.Name, metav1.DeleteOptions{})
Expect(err).To(Not(HaveOccurred()))
+3 -2
View File
@@ -13,6 +13,7 @@ import (
"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
"github.com/rancher/k3k/pkg/controller/policy"
fwk3k "github.com/rancher/k3k/tests/framework/k3k"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
@@ -35,7 +36,7 @@ var _ = When("a cluster with private registry configuration is used", Label("e2e
}
Expect(k8sClient.Create(ctx, vcp)).To(Succeed())
namespace := NewNamespace()
namespace := fwk3k.CreateNamespace(k8s)
err := k8sClient.Get(ctx, client.ObjectKeyFromObject(namespace), namespace)
Expect(err).To(Not(HaveOccurred()))
@@ -46,7 +47,7 @@ var _ = When("a cluster with private registry configuration is used", Label("e2e
Expect(k8sClient.Update(ctx, namespace)).To(Succeed())
DeferCleanup(func() {
DeleteNamespaces(namespace.Name)
fwk3k.DeleteNamespaces(k8s, namespace.Name)
Expect(k8sClient.Delete(ctx, vcp)).To(Succeed())
})
+3 -2
View File
@@ -14,6 +14,7 @@ import (
"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
"github.com/rancher/k3k/pkg/controller/cluster"
"github.com/rancher/k3k/pkg/controller/policy"
fwk3k "github.com/rancher/k3k/tests/framework/k3k"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
@@ -36,7 +37,7 @@ var _ = When("a cluster's status is tracked", Label(e2eTestLabel), Label(statusT
}
Expect(k8sClient.Create(ctx, vcp)).To(Succeed())
namespace = NewNamespace()
namespace = fwk3k.CreateNamespace(k8s)
err := k8sClient.Get(ctx, client.ObjectKeyFromObject(namespace), namespace)
Expect(err).To(Not(HaveOccurred()))
@@ -51,7 +52,7 @@ var _ = When("a cluster's status is tracked", Label(e2eTestLabel), Label(statusT
err := k8sClient.Delete(context.Background(), vcp)
Expect(err).To(Not(HaveOccurred()))
DeleteNamespaces(namespace.Name)
fwk3k.DeleteNamespaces(k8s, namespace.Name)
})
Context("and the cluster is created with a valid configuration", func() {
+2 -1
View File
@@ -11,6 +11,7 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/rancher/k3k/pkg/controller/cluster"
fwk3k "github.com/rancher/k3k/tests/framework/k3k"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
@@ -53,7 +54,7 @@ var _ = When("a shared mode cluster is created", Ordered, Label(e2eTestLabel), f
Expect(err).To(Not(HaveOccurred()))
DeferCleanup(func() {
DeleteNamespaces(virtualCluster.Cluster.Namespace)
fwk3k.DeleteNamespaces(k8s, virtualCluster.Cluster.Namespace)
err = k8s.StorageV1().StorageClasses().Delete(ctx, storageClassEnabled.Name, metav1.DeleteOptions{})
Expect(err).To(Not(HaveOccurred()))
+2 -1
View File
@@ -9,6 +9,7 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/rancher/k3k/k3k-kubelet/translate"
fwk3k "github.com/rancher/k3k/tests/framework/k3k"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
@@ -25,7 +26,7 @@ var _ = When("a shared mode cluster is created", Ordered, Label(e2eTestLabel), f
virtualCluster = NewVirtualCluster()
DeferCleanup(func() {
DeleteNamespaces(virtualCluster.Cluster.Namespace)
fwk3k.DeleteNamespaces(k8s, virtualCluster.Cluster.Namespace)
})
})
+22 -21
View File
@@ -13,6 +13,7 @@ import (
ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client"
"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
fwk3k "github.com/rancher/k3k/tests/framework/k3k"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
@@ -24,10 +25,10 @@ var _ = When("a shared mode cluster update its envs", Label(e2eTestLabel), Label
ctx := context.Background()
BeforeEach(func() {
namespace := NewNamespace()
namespace := fwk3k.CreateNamespace(k8s)
DeferCleanup(func() {
DeleteNamespaces(namespace.Name)
fwk3k.DeleteNamespaces(k8s, namespace.Name)
})
cluster := NewCluster(namespace.Name)
@@ -180,10 +181,10 @@ var _ = When("a shared mode cluster update its server args", Label(e2eTestLabel)
ctx := context.Background()
BeforeEach(func() {
namespace := NewNamespace()
namespace := fwk3k.CreateNamespace(k8s)
DeferCleanup(func() {
DeleteNamespaces(namespace.Name)
fwk3k.DeleteNamespaces(k8s, namespace.Name)
})
cluster := NewCluster(namespace.Name)
@@ -246,10 +247,10 @@ var _ = When("a virtual mode cluster update its envs", Label(e2eTestLabel), Labe
ctx := context.Background()
BeforeEach(func() {
namespace := NewNamespace()
namespace := fwk3k.CreateNamespace(k8s)
DeferCleanup(func() {
DeleteNamespaces(namespace.Name)
fwk3k.DeleteNamespaces(k8s, namespace.Name)
})
cluster := NewCluster(namespace.Name)
@@ -399,10 +400,10 @@ var _ = When("a virtual mode cluster update its server args", Label(e2eTestLabel
ctx := context.Background()
BeforeEach(func() {
namespace := NewNamespace()
namespace := fwk3k.CreateNamespace(k8s)
DeferCleanup(func() {
DeleteNamespaces(namespace.Name)
fwk3k.DeleteNamespaces(k8s, namespace.Name)
})
cluster := NewCluster(namespace.Name)
@@ -470,10 +471,10 @@ var _ = When("a shared mode cluster update its version", Label(e2eTestLabel), La
BeforeEach(func() {
ctx := context.Background()
namespace := NewNamespace()
namespace := fwk3k.CreateNamespace(k8s)
DeferCleanup(func() {
DeleteNamespaces(namespace.Name)
fwk3k.DeleteNamespaces(k8s, namespace.Name)
})
cluster := NewCluster(namespace.Name)
@@ -504,7 +505,7 @@ var _ = When("a shared mode cluster update its version", Label(e2eTestLabel), La
nginxPod, _ = virtualCluster.NewNginxPod("")
DeferCleanup(func() {
DeleteNamespaces(virtualCluster.Cluster.Namespace)
fwk3k.DeleteNamespaces(k8s, virtualCluster.Cluster.Namespace)
})
})
@@ -559,10 +560,10 @@ var _ = When("a virtual mode cluster update its version", Label(e2eTestLabel), L
BeforeEach(func() {
ctx := context.Background()
namespace := NewNamespace()
namespace := fwk3k.CreateNamespace(k8s)
DeferCleanup(func() {
DeleteNamespaces(namespace.Name)
fwk3k.DeleteNamespaces(k8s, namespace.Name)
})
cluster := NewCluster(namespace.Name)
@@ -663,10 +664,10 @@ var _ = When("a shared mode cluster scales up servers", Label(e2eTestLabel), Lab
BeforeEach(func() {
ctx := context.Background()
namespace := NewNamespace()
namespace := fwk3k.CreateNamespace(k8s)
DeferCleanup(func() {
DeleteNamespaces(namespace.Name)
fwk3k.DeleteNamespaces(k8s, namespace.Name)
})
cluster := NewCluster(namespace.Name)
@@ -751,10 +752,10 @@ var _ = When("a shared mode cluster scales down servers", Label(e2eTestLabel), L
BeforeEach(func() {
ctx := context.Background()
namespace := NewNamespace()
namespace := fwk3k.CreateNamespace(k8s)
DeferCleanup(func() {
DeleteNamespaces(namespace.Name)
fwk3k.DeleteNamespaces(k8s, namespace.Name)
})
cluster := NewCluster(namespace.Name)
@@ -841,10 +842,10 @@ var _ = When("a virtual mode cluster scales up servers", Label(e2eTestLabel), La
BeforeEach(func() {
ctx := context.Background()
namespace := NewNamespace()
namespace := fwk3k.CreateNamespace(k8s)
DeferCleanup(func() {
DeleteNamespaces(namespace.Name)
fwk3k.DeleteNamespaces(k8s, namespace.Name)
})
cluster := NewCluster(namespace.Name)
@@ -929,10 +930,10 @@ var _ = When("a virtual mode cluster scales down servers", Label(e2eTestLabel),
BeforeEach(func() {
ctx := context.Background()
namespace := NewNamespace()
namespace := fwk3k.CreateNamespace(k8s)
DeferCleanup(func() {
DeleteNamespaces(namespace.Name)
fwk3k.DeleteNamespaces(k8s, namespace.Name)
})
cluster := NewCluster(namespace.Name)
+2 -56
View File
@@ -4,8 +4,6 @@ import (
"bytes"
"context"
"fmt"
"net/url"
"os"
"strings"
"sync"
"time"
@@ -16,8 +14,6 @@ import (
"k8s.io/client-go/tools/remotecommand"
"k8s.io/kubectl/pkg/scheme"
"k8s.io/kubernetes/pkg/api/v1/pod"
"k8s.io/utils/ptr"
"sigs.k8s.io/controller-runtime/pkg/client"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -27,6 +23,7 @@ import (
"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
"github.com/rancher/k3k/pkg/controller/certs"
"github.com/rancher/k3k/pkg/controller/kubeconfig"
fwk3k "github.com/rancher/k3k/tests/framework/k3k"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
@@ -48,7 +45,7 @@ func NewVirtualCluster() *VirtualCluster { // By default, create an ephemeral cl
func NewVirtualClusterWithType(persistenceType v1beta1.PersistenceMode) *VirtualCluster {
GinkgoHelper()
namespace := NewNamespace()
namespace := fwk3k.CreateNamespace(k8s)
cluster := NewCluster(namespace.Name)
cluster.Spec.Persistence.Type = persistenceType
@@ -90,44 +87,6 @@ func NewVirtualClusters(n int) []*VirtualCluster {
return clusters
}
func NewNamespace() *v1.Namespace {
GinkgoHelper()
namespace := &v1.Namespace{ObjectMeta: metav1.ObjectMeta{GenerateName: "ns-", Labels: map[string]string{"e2e": "true"}}}
namespace, err := k8s.CoreV1().Namespaces().Create(context.Background(), namespace, metav1.CreateOptions{})
Expect(err).To(Not(HaveOccurred()))
return namespace
}
func DeleteNamespaces(names ...string) {
GinkgoHelper()
if _, found := os.LookupEnv("KEEP_NAMESPACES"); found {
By(fmt.Sprintf("Keeping namespace %v", names))
return
}
wg := sync.WaitGroup{}
wg.Add(len(names))
for _, name := range names {
go func() {
defer wg.Done()
defer GinkgoRecover()
By(fmt.Sprintf("Deleting namespace %s", name))
err := k8s.CoreV1().Namespaces().Delete(context.Background(), name, metav1.DeleteOptions{
GracePeriodSeconds: ptr.To[int64](0),
})
Expect(client.IgnoreNotFound(err)).To(Not(HaveOccurred()))
}()
}
wg.Wait()
}
func NewCluster(namespace string) *v1beta1.Cluster {
return &v1beta1.Cluster{
ObjectMeta: metav1.ObjectMeta{
@@ -464,16 +423,3 @@ func isArgFound(pod *v1.Pod, arg string) bool {
return false
}
func getServerIP(ctx context.Context, cfg *rest.Config) (string, error) {
if k3sContainer != nil {
return k3sContainer.ContainerIP(ctx)
}
u, err := url.Parse(cfg.Host)
if err != nil {
return "", err
}
// If Host includes a port, u.Hostname() extracts just the hostname part
return u.Hostname(), nil
}
-56
View File
@@ -1,56 +0,0 @@
package k3k_test
import (
"k8s.io/apimachinery/pkg/api/meta"
"k8s.io/client-go/discovery"
"k8s.io/client-go/rest"
"k8s.io/client-go/restmapper"
"k8s.io/client-go/tools/clientcmd"
memory "k8s.io/client-go/discovery/cached"
)
type RESTClientGetter struct {
clientconfig clientcmd.ClientConfig
restConfig *rest.Config
discoveryClient discovery.CachedDiscoveryInterface
}
func NewRESTClientGetter(kubeconfig []byte) (*RESTClientGetter, error) {
clientconfig, err := clientcmd.NewClientConfigFromBytes([]byte(kubeconfig))
if err != nil {
return nil, err
}
restConfig, err := clientconfig.ClientConfig()
if err != nil {
return nil, err
}
dc, err := discovery.NewDiscoveryClientForConfig(restConfig)
if err != nil {
return nil, err
}
return &RESTClientGetter{
clientconfig: clientconfig,
restConfig: restConfig,
discoveryClient: memory.NewMemCacheClient(dc),
}, nil
}
func (r *RESTClientGetter) ToRESTConfig() (*rest.Config, error) {
return r.restConfig, nil
}
func (r *RESTClientGetter) ToDiscoveryClient() (discovery.CachedDiscoveryInterface, error) {
return r.discoveryClient, nil
}
func (r *RESTClientGetter) ToRESTMapper() (meta.RESTMapper, error) {
return restmapper.NewDeferredDiscoveryRESTMapper(r.discoveryClient), nil
}
func (r *RESTClientGetter) ToRawKubeConfigLoader() clientcmd.ClientConfig {
return r.clientconfig
}
+27 -171
View File
@@ -5,41 +5,34 @@ import (
"context"
"fmt"
"io"
"maps"
"os"
"os/exec"
"path"
"path/filepath"
"strings"
"testing"
"time"
"github.com/go-logr/zapr"
"github.com/testcontainers/testcontainers-go"
"github.com/testcontainers/testcontainers-go/modules/k3s"
"go.uber.org/zap"
"helm.sh/helm/v3/pkg/action"
"helm.sh/helm/v3/pkg/chart/loader"
"k8s.io/apimachinery/pkg/api/resource"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
"k8s.io/client-go/tools/remotecommand"
"k8s.io/kubernetes/pkg/api/v1/pod"
"k8s.io/utils/ptr"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/log"
appsv1 "k8s.io/api/apps/v1"
v1 "k8s.io/api/core/v1"
networkingv1 "k8s.io/api/networking/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
fwclient "github.com/rancher/k3k/tests/framework/client"
fwcontainer "github.com/rancher/k3k/tests/framework/container"
fwlog "github.com/rancher/k3k/tests/framework/log"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
@@ -73,13 +66,12 @@ func TestTests(t *testing.T) {
}
var (
k3sContainer *k3s.K3sContainer
hostIP string
restcfg *rest.Config
k8s *kubernetes.Clientset
k8sClient client.Client
kubeconfigPath string
helmActionConfig *action.Configuration
k3sContainer *k3s.K3sContainer
hostIP string
restcfg *rest.Config
k8s *kubernetes.Clientset
k8sClient client.Client
kubeconfigPath string
)
var _ = BeforeSuite(func() {
@@ -105,143 +97,35 @@ var _ = BeforeSuite(func() {
patchPVC(ctx, k8s)
})
func initKubernetesClient(ctx context.Context) {
var (
err error
kubeconfig []byte
)
logger, err := zap.NewDevelopment()
Expect(err).NotTo(HaveOccurred())
log.SetLogger(zapr.NewLogger(logger))
kubeconfigPath := os.Getenv("KUBECONFIG")
Expect(kubeconfigPath).To(Not(BeEmpty()))
kubeconfig, err = os.ReadFile(kubeconfigPath)
Expect(err).To(Not(HaveOccurred()))
restcfg, err = clientcmd.RESTConfigFromKubeConfig(kubeconfig)
Expect(err).To(Not(HaveOccurred()))
hostIP, err = getServerIP(ctx, restcfg)
Expect(err).To(Not(HaveOccurred()))
k8s, err = kubernetes.NewForConfig(restcfg)
Expect(err).To(Not(HaveOccurred()))
scheme := buildScheme()
k8sClient, err = client.New(restcfg, client.Options{Scheme: scheme})
Expect(err).NotTo(HaveOccurred())
}
func buildScheme() *runtime.Scheme {
scheme := runtime.NewScheme()
err := clientgoscheme.AddToScheme(scheme)
Expect(err).NotTo(HaveOccurred())
err = v1beta1.AddToScheme(scheme)
Expect(err).NotTo(HaveOccurred())
return scheme
}
func installK3SDocker(ctx context.Context, controllerImage, kubeletImage string) {
var (
err error
kubeconfig []byte
)
k3sHostVersion := os.Getenv("K3S_HOST_VERSION")
if k3sHostVersion == "" {
k3sHostVersion = k3sVersion
}
k3sHostVersion = strings.ReplaceAll(k3sHostVersion, "+", "-")
k3sContainer, kubeconfigPath = fwcontainer.SetupK3s(ctx, k3sHostVersion, controllerImage, kubeletImage)
}
k3sContainer, err = k3s.Run(ctx, "rancher/k3s:"+k3sHostVersion)
Expect(err).To(Not(HaveOccurred()))
func initKubernetesClient(ctx context.Context) {
scheme := fwclient.NewScheme()
config, err := fwclient.InitFromKubeconfig(ctx, scheme, k3sContainer)
Expect(err).NotTo(HaveOccurred())
containerIP, err := k3sContainer.ContainerIP(ctx)
Expect(err).To(Not(HaveOccurred()))
GinkgoWriter.Println("K3s containerIP: " + containerIP)
kubeconfig, err = k3sContainer.GetKubeConfig(context.Background())
Expect(err).To(Not(HaveOccurred()))
tmpFile, err := os.CreateTemp("", "kubeconfig-")
Expect(err).To(Not(HaveOccurred()))
_, err = tmpFile.Write(kubeconfig)
Expect(err).To(Not(HaveOccurred()))
Expect(tmpFile.Close()).To(Succeed())
kubeconfigPath = tmpFile.Name()
err = k3sContainer.LoadImages(ctx, controllerImage+":dev", kubeletImage+":dev")
Expect(err).To(Not(HaveOccurred()))
DeferCleanup(os.Remove, kubeconfigPath)
Expect(os.Setenv("KUBECONFIG", kubeconfigPath)).To(Succeed())
GinkgoWriter.Printf("KUBECONFIG set to: %s\n", kubeconfigPath)
hostIP = config.HostIP
restcfg = config.RestConfig
k8s = config.Clientset
k8sClient = config.Client
}
func installK3kChart(controllerImage, kubeletImage string) {
pwd, err := os.Getwd()
Expect(err).To(Not(HaveOccurred()))
k3kChart, err := loader.Load(path.Join(pwd, "../charts/k3k"))
Expect(err).To(Not(HaveOccurred()))
helmActionConfig = new(action.Configuration)
installer := fwcontainer.NewHelmInstaller(controllerImage, kubeletImage, kubeconfigPath)
kubeconfig, err := os.ReadFile(kubeconfigPath)
Expect(err).To(Not(HaveOccurred()))
Expect(err).NotTo(HaveOccurred())
restClientGetter, err := NewRESTClientGetter(kubeconfig)
Expect(err).To(Not(HaveOccurred()))
restClientGetter, err := fwclient.NewRESTClientGetter(kubeconfig)
Expect(err).NotTo(HaveOccurred())
err = helmActionConfig.Init(restClientGetter, k3kNamespace, os.Getenv("HELM_DRIVER"), func(format string, v ...any) {
GinkgoWriter.Printf("[Helm] "+format+"\n", v...)
})
Expect(err).To(Not(HaveOccurred()))
iCli := action.NewInstall(helmActionConfig)
iCli.ReleaseName = "k3k"
iCli.Namespace = k3kNamespace
iCli.CreateNamespace = true
iCli.Timeout = time.Minute
iCli.Wait = true
controllerMap, _ := k3kChart.Values["controller"].(map[string]any)
extraEnvArray, _ := controllerMap["extraEnv"].([]map[string]any)
extraEnvArray = append(extraEnvArray, map[string]any{
"name": "DEBUG",
"value": "true",
})
controllerMap["extraEnv"] = extraEnvArray
imageMap, _ := controllerMap["image"].(map[string]any)
maps.Copy(imageMap, map[string]any{
"repository": controllerImage,
"tag": "dev",
"pullPolicy": "IfNotPresent",
})
agentMap, _ := k3kChart.Values["agent"].(map[string]any)
sharedAgentMap, _ := agentMap["shared"].(map[string]any)
sharedAgentImageMap, _ := sharedAgentMap["image"].(map[string]any)
maps.Copy(sharedAgentImageMap, map[string]any{
"repository": kubeletImage,
"tag": "dev",
})
release, err := iCli.Run(k3kChart, k3kChart.Values)
Expect(err).To(Not(HaveOccurred()))
GinkgoWriter.Printf("Helm release '%s' installed in '%s' namespace\n", release.Name, release.Namespace)
installer.InstallK3kChart(restClientGetter)
}
func patchPVC(ctx context.Context, clientset *kubernetes.Clientset) {
@@ -343,11 +227,11 @@ var _ = AfterSuite(func() {
// dump k3s logs
k3sLogs, err := k3sContainer.Logs(ctx)
Expect(err).To(Not(HaveOccurred()))
writeLogs("k3s.log", k3sLogs)
fwlog.WriteLogs("k3s.log", k3sLogs)
// dump k3k controller logs
k3kLogs := getK3kLogs(ctx)
writeLogs("k3k.log", k3kLogs)
k3kLogs := fwlog.GetK3kPodLogs(ctx, k8sClient, k8s, k3kNamespace)
fwlog.WriteLogs("k3k.log", k3kLogs)
testcontainers.CleanupContainer(GinkgoTB(), k3sContainer)
}
@@ -451,34 +335,6 @@ func dumpK3kCoverageData(ctx context.Context, folder string) {
Expect(k8sClient.Delete(ctx, tarPod)).To(Succeed())
}
func getK3kLogs(ctx context.Context) io.ReadCloser {
var podList v1.PodList
err := k8sClient.List(ctx, &podList, &client.ListOptions{Namespace: k3kNamespace})
Expect(err).To(Not(HaveOccurred()))
Expect(podList.Items).NotTo(BeEmpty())
k3kPod := podList.Items[0]
req := k8s.CoreV1().Pods(k3kPod.Namespace).GetLogs(k3kPod.Name, &v1.PodLogOptions{Previous: true})
podLogs, err := req.Stream(ctx)
Expect(err).To(Not(HaveOccurred()))
return podLogs
}
func writeLogs(filename string, logs io.ReadCloser) {
logsStr, err := io.ReadAll(logs)
Expect(err).To(Not(HaveOccurred()))
tempfile := path.Join(os.TempDir(), filename)
err = os.WriteFile(tempfile, []byte(logsStr), 0o644)
Expect(err).To(Not(HaveOccurred()))
GinkgoWriter.Println("logs written to: " + filename)
_ = logs.Close()
}
func readFileWithinPod(ctx context.Context, client *kubernetes.Clientset, config *rest.Config, name, namespace, path string) ([]byte, error) {
command := []string{"cat", path}
+105
View File
@@ -0,0 +1,105 @@
package client
import (
"context"
"fmt"
"net/url"
"os"
"github.com/go-logr/zapr"
"github.com/testcontainers/testcontainers-go/modules/k3s"
"go.uber.org/zap"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/log"
)
// Config holds the Kubernetes client configuration and clients.
type Config struct {
RestConfig *rest.Config
Clientset *kubernetes.Clientset
Client client.Client
HostIP string
}
// InitFromKubeconfig initializes Kubernetes clients from the KUBECONFIG environment variable.
// It sets up logging, reads the kubeconfig file, creates REST config and clients.
// The scheme parameter should be created using the scheme package.
func InitFromKubeconfig(ctx context.Context, scheme *runtime.Scheme, k3sContainer *k3s.K3sContainer) (*Config, error) {
// Setup logger
logger, err := zap.NewDevelopment()
if err != nil {
return nil, fmt.Errorf("failed to create logger: %w", err)
}
log.SetLogger(zapr.NewLogger(logger))
// Get kubeconfig path from environment
kubeconfigPath := os.Getenv("KUBECONFIG")
if kubeconfigPath == "" {
return nil, fmt.Errorf("KUBECONFIG environment variable is not set")
}
// Read kubeconfig file
kubeconfig, err := os.ReadFile(kubeconfigPath)
if err != nil {
return nil, fmt.Errorf("failed to read kubeconfig from %s: %w", kubeconfigPath, err)
}
return InitFromBytes(ctx, kubeconfig, scheme, k3sContainer)
}
// InitFromBytes initializes Kubernetes clients from kubeconfig bytes.
// The scheme parameter should be created using the scheme package.
func InitFromBytes(ctx context.Context, kubeconfig []byte, scheme *runtime.Scheme, k3sContainer *k3s.K3sContainer) (*Config, error) {
// Create REST config from kubeconfig
restConfig, err := clientcmd.RESTConfigFromKubeConfig(kubeconfig)
if err != nil {
return nil, fmt.Errorf("failed to create REST config: %w", err)
}
// Extract host IP from REST config
hostIP, err := getServerIP(ctx, restConfig, k3sContainer)
if err != nil {
return nil, fmt.Errorf("failed to get server IP: %w", err)
}
// Create Kubernetes clientset
clientset, err := kubernetes.NewForConfig(restConfig)
if err != nil {
return nil, fmt.Errorf("failed to create Kubernetes clientset: %w", err)
}
// Create controller-runtime client
runtimeClient, err := client.New(restConfig, client.Options{Scheme: scheme})
if err != nil {
return nil, fmt.Errorf("failed to create controller-runtime client: %w", err)
}
return &Config{
RestConfig: restConfig,
Clientset: clientset,
Client: runtimeClient,
HostIP: hostIP,
}, nil
}
// getServerIP extracts the server IP from the REST config.
// If running with testcontainers, it returns the container IP.
// Otherwise, it parses the hostname from the REST config host.
func getServerIP(ctx context.Context, cfg *rest.Config, k3sContainer *k3s.K3sContainer) (string, error) {
if k3sContainer != nil {
return k3sContainer.ContainerIP(ctx)
}
u, err := url.Parse(cfg.Host)
if err != nil {
return "", fmt.Errorf("failed to parse REST config host: %w", err)
}
// If Host includes a port, u.Hostname() extracts just the hostname part
return u.Hostname(), nil
}
@@ -1,4 +1,4 @@
package cli_test
package client
import (
"k8s.io/apimachinery/pkg/api/meta"
@@ -10,14 +10,19 @@ import (
memory "k8s.io/client-go/discovery/cached"
)
// RESTClientGetter is a Kubernetes REST client getter implementation that satisfies
// the genericclioptions.RESTClientGetter interface. This is used primarily for Helm
// operations in tests.
type RESTClientGetter struct {
clientconfig clientcmd.ClientConfig
restConfig *rest.Config
discoveryClient discovery.CachedDiscoveryInterface
}
// NewRESTClientGetter creates a new RESTClientGetter from kubeconfig bytes.
// This is used for Helm operations in tests.
func NewRESTClientGetter(kubeconfig []byte) (*RESTClientGetter, error) {
clientconfig, err := clientcmd.NewClientConfigFromBytes([]byte(kubeconfig))
clientconfig, err := clientcmd.NewClientConfigFromBytes(kubeconfig)
if err != nil {
return nil, err
}
@@ -39,18 +44,22 @@ func NewRESTClientGetter(kubeconfig []byte) (*RESTClientGetter, error) {
}, nil
}
// ToRESTConfig returns the REST config.
func (r *RESTClientGetter) ToRESTConfig() (*rest.Config, error) {
return r.restConfig, nil
}
// ToDiscoveryClient returns the cached discovery client.
func (r *RESTClientGetter) ToDiscoveryClient() (discovery.CachedDiscoveryInterface, error) {
return r.discoveryClient, nil
}
// ToRESTMapper returns a REST mapper from the discovery client.
func (r *RESTClientGetter) ToRESTMapper() (meta.RESTMapper, error) {
return restmapper.NewDeferredDiscoveryRESTMapper(r.discoveryClient), nil
}
// ToRawKubeConfigLoader returns the raw kubeconfig loader.
func (r *RESTClientGetter) ToRawKubeConfigLoader() clientcmd.ClientConfig {
return r.clientconfig
}
+27
View File
@@ -0,0 +1,27 @@
package client
import (
"k8s.io/apimachinery/pkg/runtime"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
)
// NewScheme creates a new Kubernetes runtime scheme with core APIs and k3k CRDs.
// This is suitable for most k3k test scenarios including integration and E2E tests.
func NewScheme() *runtime.Scheme {
scheme := runtime.NewScheme()
// Add core Kubernetes scheme (includes most common types)
if err := clientgoscheme.AddToScheme(scheme); err != nil {
panic(err)
}
// Add k3k CRDs
if err := v1beta1.AddToScheme(scheme); err != nil {
panic(err)
}
return scheme
}
+105
View File
@@ -0,0 +1,105 @@
package container
import (
"maps"
"os"
"path"
"time"
"helm.sh/helm/v3/pkg/action"
"helm.sh/helm/v3/pkg/chart/loader"
fwclient "github.com/rancher/k3k/tests/framework/client"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
// HelmInstaller provides configuration for Helm chart installation.
type HelmInstaller struct {
ChartPath string
Namespace string
ReleaseName string
Timeout time.Duration
Wait bool
ControllerImage string
KubeletImage string
KubeconfigPath string
}
// InstallK3kChart installs the k3k Helm chart with the specified configuration.
// It uses the provided RESTClientGetter for authentication and returns the Helm action configuration.
func (h *HelmInstaller) InstallK3kChart(restClientGetter *fwclient.RESTClientGetter) *action.Configuration {
GinkgoHelper()
// Load chart
k3kChart, err := loader.Load(h.ChartPath)
Expect(err).To(Not(HaveOccurred()))
// Initialize Helm action configuration
helmActionConfig := new(action.Configuration)
err = helmActionConfig.Init(restClientGetter, h.Namespace, os.Getenv("HELM_DRIVER"), func(format string, v ...any) {
GinkgoWriter.Printf("[Helm] "+format+"\n", v...)
})
Expect(err).To(Not(HaveOccurred()))
// Create install action
iCli := action.NewInstall(helmActionConfig)
iCli.ReleaseName = h.ReleaseName
iCli.Namespace = h.Namespace
iCli.CreateNamespace = true
iCli.Timeout = h.Timeout
iCli.Wait = h.Wait
// Configure controller image
controllerMap, _ := k3kChart.Values["controller"].(map[string]any)
extraEnvArray, _ := controllerMap["extraEnv"].([]map[string]any)
extraEnvArray = append(extraEnvArray, map[string]any{
"name": "DEBUG",
"value": "true",
})
controllerMap["extraEnv"] = extraEnvArray
imageMap, _ := controllerMap["image"].(map[string]any)
maps.Copy(imageMap, map[string]any{
"repository": h.ControllerImage,
"tag": "dev",
"pullPolicy": "IfNotPresent",
})
// Configure agent image
agentMap, _ := k3kChart.Values["agent"].(map[string]any)
sharedAgentMap, _ := agentMap["shared"].(map[string]any)
sharedAgentImageMap, _ := sharedAgentMap["image"].(map[string]any)
maps.Copy(sharedAgentImageMap, map[string]any{
"repository": h.KubeletImage,
"tag": "dev",
})
// Install chart
release, err := iCli.Run(k3kChart, k3kChart.Values)
Expect(err).To(Not(HaveOccurred()))
GinkgoWriter.Printf("Helm release '%s' installed in '%s' namespace\n", release.Name, release.Namespace)
return helmActionConfig
}
// NewHelmInstaller creates a new HelmInstaller with default values.
func NewHelmInstaller(controllerImage, kubeletImage, kubeconfigPath string) *HelmInstaller {
pwd, err := os.Getwd()
Expect(err).To(Not(HaveOccurred()))
return &HelmInstaller{
ChartPath: path.Join(pwd, "../../charts/k3k"),
Namespace: "k3k-system",
ReleaseName: "k3k",
Timeout: time.Minute,
Wait: true,
ControllerImage: controllerImage,
KubeletImage: kubeletImage,
KubeconfigPath: kubeconfigPath,
}
}
+63
View File
@@ -0,0 +1,63 @@
package container
import (
"context"
"os"
"strings"
"github.com/testcontainers/testcontainers-go/modules/k3s"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
// SetupK3s creates and starts a K3s testcontainer with the specified version and loads the provided images.
// It returns the container instance and a temporary kubeconfig file path.
// The kubeconfig is automatically cleaned up via DeferCleanup.
func SetupK3s(ctx context.Context, k3sVersion, controllerImage, kubeletImage string) (*k3s.K3sContainer, string) {
GinkgoHelper()
var (
err error
kubeconfig []byte
)
// Normalize version string (replace + with -)
k3sVersion = strings.ReplaceAll(k3sVersion, "+", "-")
// Start K3s container
k3sContainer, err := k3s.Run(ctx, "rancher/k3s:"+k3sVersion)
Expect(err).To(Not(HaveOccurred()))
containerIP, err := k3sContainer.ContainerIP(ctx)
Expect(err).To(Not(HaveOccurred()))
GinkgoWriter.Println("K3s containerIP: " + containerIP)
// Get kubeconfig from container
kubeconfig, err = k3sContainer.GetKubeConfig(ctx)
Expect(err).To(Not(HaveOccurred()))
// Write kubeconfig to temp file
tmpFile, err := os.CreateTemp("", "kubeconfig-")
Expect(err).To(Not(HaveOccurred()))
_, err = tmpFile.Write(kubeconfig)
Expect(err).To(Not(HaveOccurred()))
Expect(tmpFile.Close()).To(Succeed())
kubeconfigPath := tmpFile.Name()
// Load images into the K3s container
err = k3sContainer.LoadImages(ctx, controllerImage+":dev", kubeletImage+":dev")
Expect(err).To(Not(HaveOccurred()))
// Register cleanup
DeferCleanup(os.Remove, kubeconfigPath)
// Set KUBECONFIG environment variable
Expect(os.Setenv("KUBECONFIG", kubeconfigPath)).To(Succeed())
GinkgoWriter.Printf("KUBECONFIG set to: %s\n", kubeconfigPath)
return k3sContainer, kubeconfigPath
}
+34
View File
@@ -0,0 +1,34 @@
package env
import (
"path/filepath"
"k8s.io/apimachinery/pkg/runtime"
"sigs.k8s.io/controller-runtime/pkg/envtest"
)
// Config holds the envtest environment configuration.
type Config struct {
CRDPaths []string
Scheme *runtime.Scheme
}
// NewEnvironment creates a new envtest.Environment with CRDs from the k3k charts directory.
// The scheme parameter should be created using the scheme package.
func NewEnvironment(scheme *runtime.Scheme) *envtest.Environment {
return &envtest.Environment{
CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "charts", "k3k", "templates", "crds")},
ErrorIfCRDPathMissing: true,
Scheme: scheme,
}
}
// NewEnvironmentWithPaths creates a new envtest.Environment with custom CRD paths.
// This is useful when running tests from different locations in the directory tree.
func NewEnvironmentWithPaths(scheme *runtime.Scheme, crdPaths ...string) *envtest.Environment {
return &envtest.Environment{
CRDDirectoryPaths: crdPaths,
ErrorIfCRDPathMissing: true,
Scheme: scheme,
}
}
+69
View File
@@ -0,0 +1,69 @@
package k3k
import (
"context"
"fmt"
"os"
"sync"
"k8s.io/client-go/kubernetes"
"k8s.io/utils/ptr"
"sigs.k8s.io/controller-runtime/pkg/client"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
// CreateNamespace creates a new namespace with a generated name and the "e2e: true" label.
// The namespace is created using the provided Kubernetes clientset.
func CreateNamespace(clientset kubernetes.Interface) *v1.Namespace {
GinkgoHelper()
namespace := &v1.Namespace{
ObjectMeta: metav1.ObjectMeta{
GenerateName: "ns-",
Labels: map[string]string{
"e2e": "true",
},
},
}
namespace, err := clientset.CoreV1().Namespaces().Create(context.Background(), namespace, metav1.CreateOptions{})
Expect(err).To(Not(HaveOccurred()))
return namespace
}
// DeleteNamespaces deletes the specified namespaces in parallel.
// If the KEEP_NAMESPACES environment variable is set, namespaces are preserved instead.
// This is useful for debugging test failures.
func DeleteNamespaces(clientset kubernetes.Interface, names ...string) {
GinkgoHelper()
if _, found := os.LookupEnv("KEEP_NAMESPACES"); found {
By(fmt.Sprintf("Keeping namespaces %v", names))
return
}
wg := sync.WaitGroup{}
wg.Add(len(names))
for _, name := range names {
go func() {
defer wg.Done()
defer GinkgoRecover()
By(fmt.Sprintf("Deleting namespace %s", name))
err := clientset.CoreV1().Namespaces().Delete(context.Background(), name, metav1.DeleteOptions{
GracePeriodSeconds: ptr.To[int64](0),
})
Expect(client.IgnoreNotFound(err)).To(Not(HaveOccurred()))
}()
}
wg.Wait()
}
+71
View File
@@ -0,0 +1,71 @@
package log
import (
"context"
"io"
"os"
"path"
"k8s.io/client-go/kubernetes"
"sigs.k8s.io/controller-runtime/pkg/client"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
// GetK3kPodLogs retrieves logs from the first k3k pod in the specified namespace.
// This is useful for debugging test failures.
func GetK3kPodLogs(ctx context.Context, k8sClient client.Client, clientset kubernetes.Interface, namespace string) io.ReadCloser {
GinkgoHelper()
var podList v1.PodList
err := k8sClient.List(ctx, &podList, &client.ListOptions{Namespace: namespace})
Expect(err).To(Not(HaveOccurred()))
Expect(podList.Items).NotTo(BeEmpty())
k3kPod := podList.Items[0]
// Fetch complete pod object to access status information
pod, err := clientset.CoreV1().Pods(k3kPod.Namespace).Get(ctx, k3kPod.Name, metav1.GetOptions{})
Expect(err).To(Not(HaveOccurred()))
// Detect if the container has been restarted (e.g., for coverage dumping in E2E tests)
fetchPrevious := false
if len(pod.Status.ContainerStatuses) > 0 {
containerStatus := pod.Status.ContainerStatuses[0]
if containerStatus.RestartCount > 0 {
fetchPrevious = true
GinkgoWriter.Printf("Container has been restarted %d time(s), fetching previous logs\n", containerStatus.RestartCount)
}
}
req := clientset.CoreV1().Pods(pod.Namespace).GetLogs(pod.Name, &v1.PodLogOptions{Previous: fetchPrevious})
podLogs, err := req.Stream(ctx)
Expect(err).To(Not(HaveOccurred()))
return podLogs
}
// WriteLogs writes the provided logs to a temporary file with the specified filename.
// The file is written to os.TempDir() and the full path is logged to GinkgoWriter.
func WriteLogs(filename string, logs io.ReadCloser) {
GinkgoHelper()
logsStr, err := io.ReadAll(logs)
Expect(err).To(Not(HaveOccurred()))
tempfile := path.Join(os.TempDir(), filename)
err = os.WriteFile(tempfile, logsStr, 0o644)
Expect(err).To(Not(HaveOccurred()))
GinkgoWriter.Println("logs written to: " + tempfile)
_ = logs.Close()
}
@@ -8,19 +8,17 @@ import (
"github.com/go-logr/zapr"
"go.uber.org/zap"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/record"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/envtest"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
ctrl "sigs.k8s.io/controller-runtime"
metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
"github.com/rancher/k3k/pkg/controller/cluster"
"github.com/rancher/k3k/pkg/controller/cluster/agent"
fwclient "github.com/rancher/k3k/tests/framework/client"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
@@ -56,7 +54,7 @@ var _ = BeforeSuite(func() {
k8s, err = kubernetes.NewForConfig(cfg)
Expect(err).NotTo(HaveOccurred())
scheme := buildScheme()
scheme := fwclient.NewScheme()
k8sClient, err = client.New(cfg, client.Options{Scheme: scheme})
Expect(err).NotTo(HaveOccurred())
@@ -97,14 +95,3 @@ var _ = AfterSuite(func() {
err := testEnv.Stop()
Expect(err).NotTo(HaveOccurred())
})
func buildScheme() *runtime.Scheme {
scheme := runtime.NewScheme()
err := clientgoscheme.AddToScheme(scheme)
Expect(err).NotTo(HaveOccurred())
err = v1beta1.AddToScheme(scheme)
Expect(err).NotTo(HaveOccurred())
return scheme
}
@@ -10,17 +10,16 @@ import (
"github.com/go-logr/zapr"
"go.uber.org/zap"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/kubernetes"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/envtest"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
ctrl "sigs.k8s.io/controller-runtime"
metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
"github.com/rancher/k3k/k3k-kubelet/translate"
"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
fwclient "github.com/rancher/k3k/tests/framework/client"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
@@ -95,7 +94,7 @@ func NewTestEnv() *TestEnv {
CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "charts", "k3k", "templates", "crds")},
ErrorIfCRDPathMissing: true,
BinaryAssetsDirectory: tempDir,
Scheme: buildScheme(),
Scheme: fwclient.NewScheme(),
}
cfg, err := testEnv.Start()
@@ -114,17 +113,6 @@ func NewTestEnv() *TestEnv {
}
}
func buildScheme() *runtime.Scheme {
scheme := runtime.NewScheme()
err := clientgoscheme.AddToScheme(scheme)
Expect(err).NotTo(HaveOccurred())
err = v1beta1.AddToScheme(scheme)
Expect(err).NotTo(HaveOccurred())
return scheme
}
var _ = Describe("Kubelet Controller", func() {
var (
ctx context.Context
+2 -21
View File
@@ -7,18 +7,14 @@ import (
"github.com/go-logr/zapr"
"go.uber.org/zap"
"k8s.io/apimachinery/pkg/runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/envtest"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
networkingv1 "k8s.io/api/networking/v1"
ctrl "sigs.k8s.io/controller-runtime"
metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
"github.com/rancher/k3k/pkg/controller/policy"
fwclient "github.com/rancher/k3k/tests/framework/client"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
@@ -46,7 +42,7 @@ var _ = BeforeSuite(func() {
cfg, err := testEnv.Start()
Expect(err).NotTo(HaveOccurred())
scheme := buildScheme()
scheme := fwclient.NewScheme()
k8sClient, err = client.New(cfg, client.Options{Scheme: scheme})
Expect(err).NotTo(HaveOccurred())
@@ -75,18 +71,3 @@ var _ = AfterSuite(func() {
err := testEnv.Stop()
Expect(err).NotTo(HaveOccurred())
})
func buildScheme() *runtime.Scheme {
scheme := runtime.NewScheme()
err := corev1.AddToScheme(scheme)
Expect(err).NotTo(HaveOccurred())
err = appsv1.AddToScheme(scheme)
Expect(err).NotTo(HaveOccurred())
err = networkingv1.AddToScheme(scheme)
Expect(err).NotTo(HaveOccurred())
err = v1beta1.AddToScheme(scheme)
Expect(err).NotTo(HaveOccurred())
return scheme
}