mirror of
https://github.com/rancher/k3k.git
synced 2026-08-19 04:16:16 +00:00
Refactoring of tests moving common logic in tests/framework (#769)
* first commit * refactor: update golangci configuration and reorder imports in namespace.go * refactor: update golangci configuration for linters and formatters * check restart for logs fetch * rverted a bit the structure * requested changes * WriteLogs rename
This commit is contained in:
+20
-13
@@ -2,26 +2,33 @@ version: "2"
|
||||
|
||||
linters:
|
||||
enable:
|
||||
- misspell
|
||||
- wsl_v5
|
||||
- misspell
|
||||
- wsl_v5
|
||||
exclusions:
|
||||
rules:
|
||||
# Allow dot imports for Ginkgo/Gomega in test framework
|
||||
- path: tests/framework/
|
||||
text: "dot imports"
|
||||
linters:
|
||||
- staticcheck
|
||||
|
||||
formatters:
|
||||
enable:
|
||||
- gci
|
||||
- gofmt
|
||||
- gofumpt
|
||||
- gci
|
||||
- gofmt
|
||||
- gofumpt
|
||||
settings:
|
||||
gci:
|
||||
# The default order is `standard > default > custom > blank > dot > alias > localmodule`.
|
||||
custom-order: true
|
||||
sections:
|
||||
- standard
|
||||
- default
|
||||
- alias
|
||||
- localmodule
|
||||
- dot
|
||||
- blank
|
||||
- standard
|
||||
- default
|
||||
- alias
|
||||
- localmodule
|
||||
- dot
|
||||
- blank
|
||||
gofmt:
|
||||
rewrite-rules:
|
||||
- pattern: 'interface{}'
|
||||
replacement: 'any'
|
||||
- pattern: 'interface{}'
|
||||
replacement: 'any'
|
||||
|
||||
@@ -253,6 +253,7 @@ func (s *SharedAgent) podSpec(ctx context.Context) v1.PodSpec {
|
||||
}
|
||||
|
||||
podSpec.RuntimeClassName = runtimeClassName
|
||||
|
||||
return podSpec
|
||||
}
|
||||
|
||||
|
||||
@@ -137,6 +137,7 @@ func (v *VirtualAgent) deployment(ctx context.Context) error {
|
||||
|
||||
func (v *VirtualAgent) podSpec(ctx context.Context, image, name string) v1.PodSpec {
|
||||
log := ctrl.LoggerFrom(ctx)
|
||||
|
||||
var limit v1.ResourceList
|
||||
|
||||
args := v.cluster.Spec.AgentArgs
|
||||
|
||||
+19
-18
@@ -13,6 +13,7 @@ import (
|
||||
|
||||
"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
|
||||
"github.com/rancher/k3k/pkg/controller/policy"
|
||||
fwk3k "github.com/rancher/k3k/tests/framework/k3k"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
@@ -54,11 +55,11 @@ var _ = When("using the k3kcli", Label("cli"), func() {
|
||||
)
|
||||
|
||||
clusterName := "cluster-" + rand.String(5)
|
||||
namespace := NewNamespace()
|
||||
namespace := fwk3k.CreateNamespace(k8s)
|
||||
clusterNamespace := namespace.Name
|
||||
|
||||
DeferCleanup(func() {
|
||||
DeleteNamespaces(namespace.Name)
|
||||
fwk3k.DeleteNamespaces(k8s, namespace.Name)
|
||||
})
|
||||
|
||||
_, stderr, err = K3kcli("cluster", "create", "--namespace", clusterNamespace, clusterName)
|
||||
@@ -93,11 +94,11 @@ var _ = When("using the k3kcli", Label("cli"), func() {
|
||||
)
|
||||
|
||||
clusterName := "cluster-" + rand.String(5)
|
||||
namespace := NewNamespace()
|
||||
namespace := fwk3k.CreateNamespace(k8s)
|
||||
clusterNamespace := namespace.Name
|
||||
|
||||
DeferCleanup(func() {
|
||||
DeleteNamespaces(clusterNamespace)
|
||||
fwk3k.DeleteNamespaces(k8s, clusterNamespace)
|
||||
})
|
||||
|
||||
_, stderr, err = K3kcli("cluster", "create", "--version", k3sVersion, clusterName)
|
||||
@@ -142,11 +143,11 @@ var _ = When("using the k3kcli", Label("cli"), func() {
|
||||
err error
|
||||
)
|
||||
|
||||
namespace := NewNamespace()
|
||||
namespace := fwk3k.CreateNamespace(k8s)
|
||||
namespaceName := namespace.Name
|
||||
|
||||
DeferCleanup(func() {
|
||||
DeleteNamespaces(namespaceName)
|
||||
fwk3k.DeleteNamespaces(k8s, namespaceName)
|
||||
})
|
||||
|
||||
By("Creating a policy and binding to a namespace")
|
||||
@@ -213,11 +214,11 @@ var _ = When("using the k3kcli", Label("cli"), func() {
|
||||
|
||||
clusterName := "cluster-" + rand.String(5)
|
||||
|
||||
namespace := NewNamespace()
|
||||
namespace := fwk3k.CreateNamespace(k8s)
|
||||
clusterNamespace := namespace.Name
|
||||
|
||||
DeferCleanup(func() {
|
||||
DeleteNamespaces(clusterNamespace)
|
||||
fwk3k.DeleteNamespaces(k8s, clusterNamespace)
|
||||
})
|
||||
|
||||
// Create the cluster first
|
||||
@@ -247,11 +248,11 @@ var _ = When("using the k3kcli", Label("cli"), func() {
|
||||
|
||||
clusterName := "cluster-" + rand.String(5)
|
||||
|
||||
namespace := NewNamespace()
|
||||
namespace := fwk3k.CreateNamespace(k8s)
|
||||
clusterNamespace := namespace.Name
|
||||
|
||||
DeferCleanup(func() {
|
||||
DeleteNamespaces(clusterNamespace)
|
||||
fwk3k.DeleteNamespaces(k8s, clusterNamespace)
|
||||
})
|
||||
|
||||
// Create the cluster with initial version
|
||||
@@ -280,11 +281,11 @@ var _ = When("using the k3kcli", Label("cli"), func() {
|
||||
|
||||
clusterName := "cluster-" + rand.String(5)
|
||||
|
||||
namespace := NewNamespace()
|
||||
namespace := fwk3k.CreateNamespace(k8s)
|
||||
clusterNamespace := namespace.Name
|
||||
|
||||
DeferCleanup(func() {
|
||||
DeleteNamespaces(clusterNamespace)
|
||||
fwk3k.DeleteNamespaces(k8s, clusterNamespace)
|
||||
})
|
||||
|
||||
// Create the cluster with a version
|
||||
@@ -325,11 +326,11 @@ var _ = When("using the k3kcli", Label("cli"), func() {
|
||||
|
||||
clusterName := "cluster-" + rand.String(5)
|
||||
|
||||
namespace := NewNamespace()
|
||||
namespace := fwk3k.CreateNamespace(k8s)
|
||||
clusterNamespace := namespace.Name
|
||||
|
||||
DeferCleanup(func() {
|
||||
DeleteNamespaces(clusterNamespace)
|
||||
fwk3k.DeleteNamespaces(k8s, clusterNamespace)
|
||||
})
|
||||
|
||||
// Create the cluster first
|
||||
@@ -359,11 +360,11 @@ var _ = When("using the k3kcli", Label("cli"), func() {
|
||||
|
||||
clusterName := "cluster-" + rand.String(5)
|
||||
|
||||
namespace := NewNamespace()
|
||||
namespace := fwk3k.CreateNamespace(k8s)
|
||||
clusterNamespace := namespace.Name
|
||||
|
||||
DeferCleanup(func() {
|
||||
DeleteNamespaces(clusterNamespace)
|
||||
fwk3k.DeleteNamespaces(k8s, clusterNamespace)
|
||||
})
|
||||
|
||||
// Create the cluster first
|
||||
@@ -394,11 +395,11 @@ var _ = When("using the k3kcli", Label("cli"), func() {
|
||||
)
|
||||
|
||||
clusterName := "cluster-" + rand.String(5)
|
||||
namespace := NewNamespace()
|
||||
namespace := fwk3k.CreateNamespace(k8s)
|
||||
clusterNamespace := namespace.Name
|
||||
|
||||
DeferCleanup(func() {
|
||||
DeleteNamespaces(clusterNamespace)
|
||||
fwk3k.DeleteNamespaces(k8s, clusterNamespace)
|
||||
})
|
||||
|
||||
_, stderr, err = K3kcli("cluster", "create", "--namespace", clusterNamespace, clusterName)
|
||||
|
||||
@@ -1,55 +0,0 @@
|
||||
package cli_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"sync"
|
||||
|
||||
"k8s.io/utils/ptr"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
|
||||
v1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
func NewNamespace() *v1.Namespace {
|
||||
GinkgoHelper()
|
||||
|
||||
namespace := &v1.Namespace{ObjectMeta: metav1.ObjectMeta{GenerateName: "ns-", Labels: map[string]string{"e2e": "true"}}}
|
||||
namespace, err := k8s.CoreV1().Namespaces().Create(context.Background(), namespace, metav1.CreateOptions{})
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
|
||||
return namespace
|
||||
}
|
||||
|
||||
func DeleteNamespaces(names ...string) {
|
||||
GinkgoHelper()
|
||||
|
||||
if _, found := os.LookupEnv("KEEP_NAMESPACES"); found {
|
||||
By(fmt.Sprintf("Keeping namespace %v", names))
|
||||
return
|
||||
}
|
||||
|
||||
wg := sync.WaitGroup{}
|
||||
wg.Add(len(names))
|
||||
|
||||
for _, name := range names {
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
defer GinkgoRecover()
|
||||
|
||||
By(fmt.Sprintf("Deleting namespace %s", name))
|
||||
|
||||
err := k8s.CoreV1().Namespaces().Delete(context.Background(), name, metav1.DeleteOptions{
|
||||
GracePeriodSeconds: ptr.To[int64](0),
|
||||
})
|
||||
Expect(client.IgnoreNotFound(err)).To(Not(HaveOccurred()))
|
||||
}()
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
}
|
||||
+27
-175
@@ -2,31 +2,18 @@ package cli_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"maps"
|
||||
"os"
|
||||
"path"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/go-logr/zapr"
|
||||
"github.com/testcontainers/testcontainers-go"
|
||||
"github.com/testcontainers/testcontainers-go/modules/k3s"
|
||||
"go.uber.org/zap"
|
||||
"helm.sh/helm/v3/pkg/action"
|
||||
"helm.sh/helm/v3/pkg/chart/loader"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
"k8s.io/client-go/rest"
|
||||
"k8s.io/client-go/tools/clientcmd"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
"sigs.k8s.io/controller-runtime/pkg/log"
|
||||
|
||||
v1 "k8s.io/api/core/v1"
|
||||
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
|
||||
|
||||
"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
|
||||
fwclient "github.com/rancher/k3k/tests/framework/client"
|
||||
fwcontainer "github.com/rancher/k3k/tests/framework/container"
|
||||
fwlog "github.com/rancher/k3k/tests/framework/log"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
@@ -45,12 +32,11 @@ func TestTests(t *testing.T) {
|
||||
}
|
||||
|
||||
var (
|
||||
k3sContainer *k3s.K3sContainer
|
||||
restcfg *rest.Config
|
||||
k8s *kubernetes.Clientset
|
||||
k8sClient client.Client
|
||||
kubeconfigPath string
|
||||
helmActionConfig *action.Configuration
|
||||
k3sContainer *k3s.K3sContainer
|
||||
restcfg *rest.Config
|
||||
k8s *kubernetes.Clientset
|
||||
k8sClient client.Client
|
||||
kubeconfigPath string
|
||||
)
|
||||
|
||||
var _ = BeforeSuite(func() {
|
||||
@@ -65,147 +51,41 @@ var _ = BeforeSuite(func() {
|
||||
}
|
||||
|
||||
installK3SDocker(ctx, repo+"/k3k", repo+"/k3k-kubelet")
|
||||
initKubernetesClient()
|
||||
initKubernetesClient(ctx)
|
||||
installK3kChart(repo+"/k3k", repo+"/k3k-kubelet")
|
||||
} else {
|
||||
initKubernetesClient()
|
||||
initKubernetesClient(ctx)
|
||||
}
|
||||
})
|
||||
|
||||
func initKubernetesClient() {
|
||||
var (
|
||||
err error
|
||||
kubeconfig []byte
|
||||
)
|
||||
|
||||
logger, err := zap.NewDevelopment()
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
log.SetLogger(zapr.NewLogger(logger))
|
||||
|
||||
kubeconfigPath := os.Getenv("KUBECONFIG")
|
||||
Expect(kubeconfigPath).To(Not(BeEmpty()))
|
||||
|
||||
kubeconfig, err = os.ReadFile(kubeconfigPath)
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
|
||||
restcfg, err = clientcmd.RESTConfigFromKubeConfig(kubeconfig)
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
|
||||
k8s, err = kubernetes.NewForConfig(restcfg)
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
|
||||
scheme := buildScheme()
|
||||
k8sClient, err = client.New(restcfg, client.Options{Scheme: scheme})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
}
|
||||
|
||||
func buildScheme() *runtime.Scheme {
|
||||
scheme := runtime.NewScheme()
|
||||
|
||||
err := clientgoscheme.AddToScheme(scheme)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
err = v1beta1.AddToScheme(scheme)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
return scheme
|
||||
}
|
||||
|
||||
func installK3SDocker(ctx context.Context, controllerImage, kubeletImage string) {
|
||||
var (
|
||||
err error
|
||||
kubeconfig []byte
|
||||
)
|
||||
|
||||
k3sHostVersion := os.Getenv("K3S_HOST_VERSION")
|
||||
if k3sHostVersion == "" {
|
||||
k3sHostVersion = k3sVersion
|
||||
}
|
||||
|
||||
k3sHostVersion = strings.ReplaceAll(k3sHostVersion, "+", "-")
|
||||
k3sContainer, kubeconfigPath = fwcontainer.SetupK3s(ctx, k3sHostVersion, controllerImage, kubeletImage)
|
||||
}
|
||||
|
||||
k3sContainer, err = k3s.Run(ctx, "rancher/k3s:"+k3sHostVersion)
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
func initKubernetesClient(ctx context.Context) {
|
||||
scheme := fwclient.NewScheme()
|
||||
config, err := fwclient.InitFromKubeconfig(ctx, scheme, k3sContainer)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
containerIP, err := k3sContainer.ContainerIP(ctx)
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
|
||||
GinkgoWriter.Println("K3s containerIP: " + containerIP)
|
||||
|
||||
kubeconfig, err = k3sContainer.GetKubeConfig(context.Background())
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
|
||||
tmpFile, err := os.CreateTemp("", "kubeconfig-")
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
|
||||
_, err = tmpFile.Write(kubeconfig)
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
Expect(tmpFile.Close()).To(Succeed())
|
||||
kubeconfigPath = tmpFile.Name()
|
||||
|
||||
err = k3sContainer.LoadImages(ctx, controllerImage+":dev", kubeletImage+":dev")
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
DeferCleanup(os.Remove, kubeconfigPath)
|
||||
|
||||
Expect(os.Setenv("KUBECONFIG", kubeconfigPath)).To(Succeed())
|
||||
GinkgoWriter.Printf("KUBECONFIG set to: %s\n", kubeconfigPath)
|
||||
restcfg = config.RestConfig
|
||||
k8s = config.Clientset
|
||||
k8sClient = config.Client
|
||||
}
|
||||
|
||||
func installK3kChart(controllerImage, kubeletImage string) {
|
||||
pwd, err := os.Getwd()
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
|
||||
k3kChart, err := loader.Load(path.Join(pwd, "../../charts/k3k"))
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
|
||||
helmActionConfig = new(action.Configuration)
|
||||
|
||||
installer := fwcontainer.NewHelmInstaller(controllerImage, kubeletImage, kubeconfigPath)
|
||||
kubeconfig, err := os.ReadFile(kubeconfigPath)
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
restClientGetter, err := NewRESTClientGetter(kubeconfig)
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
restClientGetter, err := fwclient.NewRESTClientGetter(kubeconfig)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
err = helmActionConfig.Init(restClientGetter, k3kNamespace, os.Getenv("HELM_DRIVER"), func(format string, v ...any) {
|
||||
GinkgoWriter.Printf("[Helm] "+format+"\n", v...)
|
||||
})
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
|
||||
iCli := action.NewInstall(helmActionConfig)
|
||||
iCli.ReleaseName = "k3k"
|
||||
iCli.Namespace = k3kNamespace
|
||||
iCli.CreateNamespace = true
|
||||
iCli.Timeout = time.Minute
|
||||
iCli.Wait = true
|
||||
|
||||
controllerMap, _ := k3kChart.Values["controller"].(map[string]any)
|
||||
|
||||
extraEnvArray, _ := controllerMap["extraEnv"].([]map[string]any)
|
||||
extraEnvArray = append(extraEnvArray, map[string]any{
|
||||
"name": "DEBUG",
|
||||
"value": "true",
|
||||
})
|
||||
controllerMap["extraEnv"] = extraEnvArray
|
||||
|
||||
imageMap, _ := controllerMap["image"].(map[string]any)
|
||||
maps.Copy(imageMap, map[string]any{
|
||||
"repository": controllerImage,
|
||||
"tag": "dev",
|
||||
"pullPolicy": "IfNotPresent",
|
||||
})
|
||||
|
||||
agentMap, _ := k3kChart.Values["agent"].(map[string]any)
|
||||
sharedAgentMap, _ := agentMap["shared"].(map[string]any)
|
||||
sharedAgentImageMap, _ := sharedAgentMap["image"].(map[string]any)
|
||||
maps.Copy(sharedAgentImageMap, map[string]any{
|
||||
"repository": kubeletImage,
|
||||
"tag": "dev",
|
||||
})
|
||||
|
||||
release, err := iCli.Run(k3kChart, k3kChart.Values)
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
|
||||
GinkgoWriter.Printf("Helm release '%s' installed in '%s' namespace\n", release.Name, release.Namespace)
|
||||
installer.InstallK3kChart(restClientGetter)
|
||||
}
|
||||
|
||||
var _ = AfterSuite(func() {
|
||||
@@ -215,40 +95,12 @@ var _ = AfterSuite(func() {
|
||||
// dump k3s logs
|
||||
k3sLogs, err := k3sContainer.Logs(ctx)
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
writeLogs("k3s.log", k3sLogs)
|
||||
fwlog.WriteLogs("k3s.log", k3sLogs)
|
||||
|
||||
// dump k3k controller logs
|
||||
k3kLogs := getK3kLogs(ctx)
|
||||
writeLogs("k3k.log", k3kLogs)
|
||||
k3kLogs := fwlog.GetK3kPodLogs(ctx, k8sClient, k8s, k3kNamespace)
|
||||
fwlog.WriteLogs("k3k.log", k3kLogs)
|
||||
|
||||
testcontainers.CleanupContainer(GinkgoTB(), k3sContainer)
|
||||
}
|
||||
})
|
||||
|
||||
func getK3kLogs(ctx context.Context) io.ReadCloser {
|
||||
var podList v1.PodList
|
||||
|
||||
err := k8sClient.List(ctx, &podList, &client.ListOptions{Namespace: k3kNamespace})
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
Expect(podList.Items).NotTo(BeEmpty())
|
||||
|
||||
k3kPod := podList.Items[0]
|
||||
req := k8s.CoreV1().Pods(k3kPod.Namespace).GetLogs(k3kPod.Name, &v1.PodLogOptions{})
|
||||
podLogs, err := req.Stream(ctx)
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
|
||||
return podLogs
|
||||
}
|
||||
|
||||
func writeLogs(filename string, logs io.ReadCloser) {
|
||||
logsStr, err := io.ReadAll(logs)
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
|
||||
tempfile := path.Join(os.TempDir(), filename)
|
||||
err = os.WriteFile(tempfile, []byte(logsStr), 0o644)
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
|
||||
GinkgoWriter.Println("logs written to: " + filename)
|
||||
|
||||
_ = logs.Close()
|
||||
}
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
|
||||
"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
|
||||
fwk3k "github.com/rancher/k3k/tests/framework/k3k"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
@@ -27,14 +28,14 @@ var _ = When("a cluster with secretMounts configuration is used to load addons",
|
||||
BeforeEach(func() {
|
||||
ctx := context.Background()
|
||||
|
||||
namespace := NewNamespace()
|
||||
namespace := fwk3k.CreateNamespace(k8s)
|
||||
|
||||
// Create the addon secret
|
||||
err := createAddonSecret(ctx, namespace.Name)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
DeferCleanup(func() {
|
||||
DeleteNamespaces(namespace.Name)
|
||||
fwk3k.DeleteNamespaces(k8s, namespace.Name)
|
||||
})
|
||||
|
||||
cluster := NewCluster(namespace.Name)
|
||||
@@ -96,14 +97,14 @@ var _ = When("a cluster with addon configuration is used with addons secret in t
|
||||
BeforeEach(func() {
|
||||
ctx := context.Background()
|
||||
|
||||
namespace := NewNamespace()
|
||||
namespace := fwk3k.CreateNamespace(k8s)
|
||||
|
||||
// Create the addon secret
|
||||
err := createAddonSecret(ctx, namespace.Name)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
DeferCleanup(func() {
|
||||
DeleteNamespaces(namespace.Name)
|
||||
fwk3k.DeleteNamespaces(k8s, namespace.Name)
|
||||
})
|
||||
|
||||
cluster := NewCluster(namespace.Name)
|
||||
@@ -154,15 +155,15 @@ var _ = When("a cluster with addon configuration is used with addons secret in t
|
||||
BeforeEach(func() {
|
||||
ctx := context.Background()
|
||||
|
||||
namespace := NewNamespace()
|
||||
secretNamespace := NewNamespace()
|
||||
namespace := fwk3k.CreateNamespace(k8s)
|
||||
secretNamespace := fwk3k.CreateNamespace(k8s)
|
||||
|
||||
// Create the addon secret
|
||||
err := createAddonSecret(ctx, secretNamespace.Name)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
DeferCleanup(func() {
|
||||
DeleteNamespaces(namespace.Name, secretNamespace.Name)
|
||||
fwk3k.DeleteNamespaces(k8s, namespace.Name, secretNamespace.Name)
|
||||
})
|
||||
|
||||
cluster := NewCluster(namespace.Name)
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
|
||||
"github.com/rancher/k3k/k3k-kubelet/translate"
|
||||
fwk3k "github.com/rancher/k3k/tests/framework/k3k"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
@@ -28,7 +29,7 @@ var _ = Context("In a shared cluster", Label(e2eTestLabel), Ordered, func() {
|
||||
translator = translate.NewHostTranslator(virtualCluster.Cluster)
|
||||
|
||||
DeferCleanup(func() {
|
||||
DeleteNamespaces(virtualCluster.Cluster.Namespace)
|
||||
fwk3k.DeleteNamespaces(k8s, virtualCluster.Cluster.Namespace)
|
||||
})
|
||||
})
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
|
||||
fwk3k "github.com/rancher/k3k/tests/framework/k3k"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
@@ -17,10 +18,10 @@ var _ = When("a cluster with custom certificates is installed with individual ce
|
||||
BeforeEach(func() {
|
||||
ctx := context.Background()
|
||||
|
||||
namespace := NewNamespace()
|
||||
namespace := fwk3k.CreateNamespace(k8s)
|
||||
|
||||
DeferCleanup(func() {
|
||||
DeleteNamespaces(namespace.Name)
|
||||
fwk3k.DeleteNamespaces(k8s, namespace.Name)
|
||||
})
|
||||
|
||||
// create custom cert secret
|
||||
|
||||
@@ -5,6 +5,8 @@ import (
|
||||
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
|
||||
fwk3k "github.com/rancher/k3k/tests/framework/k3k"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
@@ -13,10 +15,10 @@ var _ = When("creating a shared mode cluster", Label(e2eTestLabel), Label(slowTe
|
||||
var virtualCluster *VirtualCluster
|
||||
|
||||
BeforeEach(func() {
|
||||
namespace := NewNamespace()
|
||||
namespace := fwk3k.CreateNamespace(k8s)
|
||||
|
||||
DeferCleanup(func() {
|
||||
DeleteNamespaces(namespace.Name)
|
||||
fwk3k.DeleteNamespaces(k8s, namespace.Name)
|
||||
})
|
||||
|
||||
cluster := NewCluster(namespace.Name)
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
package k3k_test
|
||||
|
||||
import (
|
||||
fwk3k "github.com/rancher/k3k/tests/framework/k3k"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
@@ -18,7 +20,7 @@ var _ = When("two virtual clusters are installed", Label(e2eTestLabel), Label(ne
|
||||
})
|
||||
|
||||
AfterEach(func() {
|
||||
DeleteNamespaces(cluster1.Cluster.Namespace, cluster2.Cluster.Namespace)
|
||||
fwk3k.DeleteNamespaces(k8s, cluster1.Cluster.Namespace, cluster2.Cluster.Namespace)
|
||||
})
|
||||
|
||||
It("can create pods in each of them that are isolated", func() {
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
|
||||
"github.com/rancher/k3k/k3k-kubelet/translate"
|
||||
"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
|
||||
fwk3k "github.com/rancher/k3k/tests/framework/k3k"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
@@ -26,7 +27,7 @@ var _ = When("an ephemeral cluster is installed", Label(e2eTestLabel), Label(per
|
||||
virtualCluster = NewVirtualCluster()
|
||||
|
||||
DeferCleanup(func() {
|
||||
DeleteNamespaces(virtualCluster.Cluster.Namespace)
|
||||
fwk3k.DeleteNamespaces(k8s, virtualCluster.Cluster.Namespace)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -119,7 +120,7 @@ var _ = When("a dynamic cluster is installed", Label(e2eTestLabel), Label(persis
|
||||
})
|
||||
|
||||
AfterEach(func() {
|
||||
DeleteNamespaces(virtualCluster.Cluster.Namespace)
|
||||
fwk3k.DeleteNamespaces(k8s, virtualCluster.Cluster.Namespace)
|
||||
})
|
||||
|
||||
It("can create a nginx pod", func() {
|
||||
@@ -152,10 +153,10 @@ var _ = When("a dynamic cluster is installed", Label(e2eTestLabel), Label(persis
|
||||
It("can delete a HA cluster", func() {
|
||||
ctx := context.Background()
|
||||
|
||||
namespace := NewNamespace()
|
||||
namespace := fwk3k.CreateNamespace(k8s)
|
||||
|
||||
DeferCleanup(func() {
|
||||
DeleteNamespaces(virtualCluster.Cluster.Namespace)
|
||||
fwk3k.DeleteNamespaces(k8s, virtualCluster.Cluster.Namespace)
|
||||
})
|
||||
|
||||
cluster := NewCluster(namespace.Name)
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
|
||||
"github.com/rancher/k3k/k3k-kubelet/translate"
|
||||
fwk3k "github.com/rancher/k3k/tests/framework/k3k"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
@@ -30,7 +31,7 @@ var _ = Context("In a shared cluster", Label(e2eTestLabel), Ordered, func() {
|
||||
translator = translate.NewHostTranslator(virtualCluster.Cluster)
|
||||
|
||||
DeferCleanup(func() {
|
||||
DeleteNamespaces(virtualCluster.Cluster.Namespace)
|
||||
fwk3k.DeleteNamespaces(k8s, virtualCluster.Cluster.Namespace)
|
||||
})
|
||||
})
|
||||
|
||||
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
|
||||
"github.com/rancher/k3k/pkg/controller/cluster"
|
||||
"github.com/rancher/k3k/pkg/controller/policy"
|
||||
fwk3k "github.com/rancher/k3k/tests/framework/k3k"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
@@ -109,7 +110,7 @@ var _ = When("a shared mode cluster is created in a namespace with a policy", Or
|
||||
}
|
||||
|
||||
DeferCleanup(func() {
|
||||
DeleteNamespaces(ns.Name)
|
||||
fwk3k.DeleteNamespaces(k8s, ns.Name)
|
||||
|
||||
err = k8s.StorageV1().StorageClasses().Delete(ctx, storageClassEnabled.Name, metav1.DeleteOptions{})
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
|
||||
"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
|
||||
"github.com/rancher/k3k/pkg/controller/policy"
|
||||
fwk3k "github.com/rancher/k3k/tests/framework/k3k"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
@@ -35,7 +36,7 @@ var _ = When("a cluster with private registry configuration is used", Label("e2e
|
||||
}
|
||||
Expect(k8sClient.Create(ctx, vcp)).To(Succeed())
|
||||
|
||||
namespace := NewNamespace()
|
||||
namespace := fwk3k.CreateNamespace(k8s)
|
||||
|
||||
err := k8sClient.Get(ctx, client.ObjectKeyFromObject(namespace), namespace)
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
@@ -46,7 +47,7 @@ var _ = When("a cluster with private registry configuration is used", Label("e2e
|
||||
Expect(k8sClient.Update(ctx, namespace)).To(Succeed())
|
||||
|
||||
DeferCleanup(func() {
|
||||
DeleteNamespaces(namespace.Name)
|
||||
fwk3k.DeleteNamespaces(k8s, namespace.Name)
|
||||
Expect(k8sClient.Delete(ctx, vcp)).To(Succeed())
|
||||
})
|
||||
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
|
||||
"github.com/rancher/k3k/pkg/controller/cluster"
|
||||
"github.com/rancher/k3k/pkg/controller/policy"
|
||||
fwk3k "github.com/rancher/k3k/tests/framework/k3k"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
@@ -36,7 +37,7 @@ var _ = When("a cluster's status is tracked", Label(e2eTestLabel), Label(statusT
|
||||
}
|
||||
Expect(k8sClient.Create(ctx, vcp)).To(Succeed())
|
||||
|
||||
namespace = NewNamespace()
|
||||
namespace = fwk3k.CreateNamespace(k8s)
|
||||
|
||||
err := k8sClient.Get(ctx, client.ObjectKeyFromObject(namespace), namespace)
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
@@ -51,7 +52,7 @@ var _ = When("a cluster's status is tracked", Label(e2eTestLabel), Label(statusT
|
||||
err := k8sClient.Delete(context.Background(), vcp)
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
|
||||
DeleteNamespaces(namespace.Name)
|
||||
fwk3k.DeleteNamespaces(k8s, namespace.Name)
|
||||
})
|
||||
|
||||
Context("and the cluster is created with a valid configuration", func() {
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
|
||||
"github.com/rancher/k3k/pkg/controller/cluster"
|
||||
fwk3k "github.com/rancher/k3k/tests/framework/k3k"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
@@ -53,7 +54,7 @@ var _ = When("a shared mode cluster is created", Ordered, Label(e2eTestLabel), f
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
|
||||
DeferCleanup(func() {
|
||||
DeleteNamespaces(virtualCluster.Cluster.Namespace)
|
||||
fwk3k.DeleteNamespaces(k8s, virtualCluster.Cluster.Namespace)
|
||||
|
||||
err = k8s.StorageV1().StorageClasses().Delete(ctx, storageClassEnabled.Name, metav1.DeleteOptions{})
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
|
||||
"github.com/rancher/k3k/k3k-kubelet/translate"
|
||||
fwk3k "github.com/rancher/k3k/tests/framework/k3k"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
@@ -25,7 +26,7 @@ var _ = When("a shared mode cluster is created", Ordered, Label(e2eTestLabel), f
|
||||
virtualCluster = NewVirtualCluster()
|
||||
|
||||
DeferCleanup(func() {
|
||||
DeleteNamespaces(virtualCluster.Cluster.Namespace)
|
||||
fwk3k.DeleteNamespaces(k8s, virtualCluster.Cluster.Namespace)
|
||||
})
|
||||
})
|
||||
|
||||
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client"
|
||||
|
||||
"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
|
||||
fwk3k "github.com/rancher/k3k/tests/framework/k3k"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
@@ -24,10 +25,10 @@ var _ = When("a shared mode cluster update its envs", Label(e2eTestLabel), Label
|
||||
ctx := context.Background()
|
||||
|
||||
BeforeEach(func() {
|
||||
namespace := NewNamespace()
|
||||
namespace := fwk3k.CreateNamespace(k8s)
|
||||
|
||||
DeferCleanup(func() {
|
||||
DeleteNamespaces(namespace.Name)
|
||||
fwk3k.DeleteNamespaces(k8s, namespace.Name)
|
||||
})
|
||||
|
||||
cluster := NewCluster(namespace.Name)
|
||||
@@ -180,10 +181,10 @@ var _ = When("a shared mode cluster update its server args", Label(e2eTestLabel)
|
||||
ctx := context.Background()
|
||||
|
||||
BeforeEach(func() {
|
||||
namespace := NewNamespace()
|
||||
namespace := fwk3k.CreateNamespace(k8s)
|
||||
|
||||
DeferCleanup(func() {
|
||||
DeleteNamespaces(namespace.Name)
|
||||
fwk3k.DeleteNamespaces(k8s, namespace.Name)
|
||||
})
|
||||
|
||||
cluster := NewCluster(namespace.Name)
|
||||
@@ -246,10 +247,10 @@ var _ = When("a virtual mode cluster update its envs", Label(e2eTestLabel), Labe
|
||||
ctx := context.Background()
|
||||
|
||||
BeforeEach(func() {
|
||||
namespace := NewNamespace()
|
||||
namespace := fwk3k.CreateNamespace(k8s)
|
||||
|
||||
DeferCleanup(func() {
|
||||
DeleteNamespaces(namespace.Name)
|
||||
fwk3k.DeleteNamespaces(k8s, namespace.Name)
|
||||
})
|
||||
|
||||
cluster := NewCluster(namespace.Name)
|
||||
@@ -399,10 +400,10 @@ var _ = When("a virtual mode cluster update its server args", Label(e2eTestLabel
|
||||
ctx := context.Background()
|
||||
|
||||
BeforeEach(func() {
|
||||
namespace := NewNamespace()
|
||||
namespace := fwk3k.CreateNamespace(k8s)
|
||||
|
||||
DeferCleanup(func() {
|
||||
DeleteNamespaces(namespace.Name)
|
||||
fwk3k.DeleteNamespaces(k8s, namespace.Name)
|
||||
})
|
||||
|
||||
cluster := NewCluster(namespace.Name)
|
||||
@@ -470,10 +471,10 @@ var _ = When("a shared mode cluster update its version", Label(e2eTestLabel), La
|
||||
|
||||
BeforeEach(func() {
|
||||
ctx := context.Background()
|
||||
namespace := NewNamespace()
|
||||
namespace := fwk3k.CreateNamespace(k8s)
|
||||
|
||||
DeferCleanup(func() {
|
||||
DeleteNamespaces(namespace.Name)
|
||||
fwk3k.DeleteNamespaces(k8s, namespace.Name)
|
||||
})
|
||||
|
||||
cluster := NewCluster(namespace.Name)
|
||||
@@ -504,7 +505,7 @@ var _ = When("a shared mode cluster update its version", Label(e2eTestLabel), La
|
||||
nginxPod, _ = virtualCluster.NewNginxPod("")
|
||||
|
||||
DeferCleanup(func() {
|
||||
DeleteNamespaces(virtualCluster.Cluster.Namespace)
|
||||
fwk3k.DeleteNamespaces(k8s, virtualCluster.Cluster.Namespace)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -559,10 +560,10 @@ var _ = When("a virtual mode cluster update its version", Label(e2eTestLabel), L
|
||||
|
||||
BeforeEach(func() {
|
||||
ctx := context.Background()
|
||||
namespace := NewNamespace()
|
||||
namespace := fwk3k.CreateNamespace(k8s)
|
||||
|
||||
DeferCleanup(func() {
|
||||
DeleteNamespaces(namespace.Name)
|
||||
fwk3k.DeleteNamespaces(k8s, namespace.Name)
|
||||
})
|
||||
|
||||
cluster := NewCluster(namespace.Name)
|
||||
@@ -663,10 +664,10 @@ var _ = When("a shared mode cluster scales up servers", Label(e2eTestLabel), Lab
|
||||
|
||||
BeforeEach(func() {
|
||||
ctx := context.Background()
|
||||
namespace := NewNamespace()
|
||||
namespace := fwk3k.CreateNamespace(k8s)
|
||||
|
||||
DeferCleanup(func() {
|
||||
DeleteNamespaces(namespace.Name)
|
||||
fwk3k.DeleteNamespaces(k8s, namespace.Name)
|
||||
})
|
||||
|
||||
cluster := NewCluster(namespace.Name)
|
||||
@@ -751,10 +752,10 @@ var _ = When("a shared mode cluster scales down servers", Label(e2eTestLabel), L
|
||||
|
||||
BeforeEach(func() {
|
||||
ctx := context.Background()
|
||||
namespace := NewNamespace()
|
||||
namespace := fwk3k.CreateNamespace(k8s)
|
||||
|
||||
DeferCleanup(func() {
|
||||
DeleteNamespaces(namespace.Name)
|
||||
fwk3k.DeleteNamespaces(k8s, namespace.Name)
|
||||
})
|
||||
|
||||
cluster := NewCluster(namespace.Name)
|
||||
@@ -841,10 +842,10 @@ var _ = When("a virtual mode cluster scales up servers", Label(e2eTestLabel), La
|
||||
|
||||
BeforeEach(func() {
|
||||
ctx := context.Background()
|
||||
namespace := NewNamespace()
|
||||
namespace := fwk3k.CreateNamespace(k8s)
|
||||
|
||||
DeferCleanup(func() {
|
||||
DeleteNamespaces(namespace.Name)
|
||||
fwk3k.DeleteNamespaces(k8s, namespace.Name)
|
||||
})
|
||||
|
||||
cluster := NewCluster(namespace.Name)
|
||||
@@ -929,10 +930,10 @@ var _ = When("a virtual mode cluster scales down servers", Label(e2eTestLabel),
|
||||
|
||||
BeforeEach(func() {
|
||||
ctx := context.Background()
|
||||
namespace := NewNamespace()
|
||||
namespace := fwk3k.CreateNamespace(k8s)
|
||||
|
||||
DeferCleanup(func() {
|
||||
DeleteNamespaces(namespace.Name)
|
||||
fwk3k.DeleteNamespaces(k8s, namespace.Name)
|
||||
})
|
||||
|
||||
cluster := NewCluster(namespace.Name)
|
||||
|
||||
@@ -4,8 +4,6 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -16,8 +14,6 @@ import (
|
||||
"k8s.io/client-go/tools/remotecommand"
|
||||
"k8s.io/kubectl/pkg/scheme"
|
||||
"k8s.io/kubernetes/pkg/api/v1/pod"
|
||||
"k8s.io/utils/ptr"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
|
||||
v1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
@@ -27,6 +23,7 @@ import (
|
||||
"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
|
||||
"github.com/rancher/k3k/pkg/controller/certs"
|
||||
"github.com/rancher/k3k/pkg/controller/kubeconfig"
|
||||
fwk3k "github.com/rancher/k3k/tests/framework/k3k"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
@@ -48,7 +45,7 @@ func NewVirtualCluster() *VirtualCluster { // By default, create an ephemeral cl
|
||||
func NewVirtualClusterWithType(persistenceType v1beta1.PersistenceMode) *VirtualCluster {
|
||||
GinkgoHelper()
|
||||
|
||||
namespace := NewNamespace()
|
||||
namespace := fwk3k.CreateNamespace(k8s)
|
||||
|
||||
cluster := NewCluster(namespace.Name)
|
||||
cluster.Spec.Persistence.Type = persistenceType
|
||||
@@ -90,44 +87,6 @@ func NewVirtualClusters(n int) []*VirtualCluster {
|
||||
return clusters
|
||||
}
|
||||
|
||||
func NewNamespace() *v1.Namespace {
|
||||
GinkgoHelper()
|
||||
|
||||
namespace := &v1.Namespace{ObjectMeta: metav1.ObjectMeta{GenerateName: "ns-", Labels: map[string]string{"e2e": "true"}}}
|
||||
namespace, err := k8s.CoreV1().Namespaces().Create(context.Background(), namespace, metav1.CreateOptions{})
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
|
||||
return namespace
|
||||
}
|
||||
|
||||
func DeleteNamespaces(names ...string) {
|
||||
GinkgoHelper()
|
||||
|
||||
if _, found := os.LookupEnv("KEEP_NAMESPACES"); found {
|
||||
By(fmt.Sprintf("Keeping namespace %v", names))
|
||||
return
|
||||
}
|
||||
|
||||
wg := sync.WaitGroup{}
|
||||
wg.Add(len(names))
|
||||
|
||||
for _, name := range names {
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
defer GinkgoRecover()
|
||||
|
||||
By(fmt.Sprintf("Deleting namespace %s", name))
|
||||
|
||||
err := k8s.CoreV1().Namespaces().Delete(context.Background(), name, metav1.DeleteOptions{
|
||||
GracePeriodSeconds: ptr.To[int64](0),
|
||||
})
|
||||
Expect(client.IgnoreNotFound(err)).To(Not(HaveOccurred()))
|
||||
}()
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func NewCluster(namespace string) *v1beta1.Cluster {
|
||||
return &v1beta1.Cluster{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
@@ -464,16 +423,3 @@ func isArgFound(pod *v1.Pod, arg string) bool {
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func getServerIP(ctx context.Context, cfg *rest.Config) (string, error) {
|
||||
if k3sContainer != nil {
|
||||
return k3sContainer.ContainerIP(ctx)
|
||||
}
|
||||
|
||||
u, err := url.Parse(cfg.Host)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
// If Host includes a port, u.Hostname() extracts just the hostname part
|
||||
return u.Hostname(), nil
|
||||
}
|
||||
|
||||
@@ -1,56 +0,0 @@
|
||||
package k3k_test
|
||||
|
||||
import (
|
||||
"k8s.io/apimachinery/pkg/api/meta"
|
||||
"k8s.io/client-go/discovery"
|
||||
"k8s.io/client-go/rest"
|
||||
"k8s.io/client-go/restmapper"
|
||||
"k8s.io/client-go/tools/clientcmd"
|
||||
|
||||
memory "k8s.io/client-go/discovery/cached"
|
||||
)
|
||||
|
||||
type RESTClientGetter struct {
|
||||
clientconfig clientcmd.ClientConfig
|
||||
restConfig *rest.Config
|
||||
discoveryClient discovery.CachedDiscoveryInterface
|
||||
}
|
||||
|
||||
func NewRESTClientGetter(kubeconfig []byte) (*RESTClientGetter, error) {
|
||||
clientconfig, err := clientcmd.NewClientConfigFromBytes([]byte(kubeconfig))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
restConfig, err := clientconfig.ClientConfig()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
dc, err := discovery.NewDiscoveryClientForConfig(restConfig)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &RESTClientGetter{
|
||||
clientconfig: clientconfig,
|
||||
restConfig: restConfig,
|
||||
discoveryClient: memory.NewMemCacheClient(dc),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (r *RESTClientGetter) ToRESTConfig() (*rest.Config, error) {
|
||||
return r.restConfig, nil
|
||||
}
|
||||
|
||||
func (r *RESTClientGetter) ToDiscoveryClient() (discovery.CachedDiscoveryInterface, error) {
|
||||
return r.discoveryClient, nil
|
||||
}
|
||||
|
||||
func (r *RESTClientGetter) ToRESTMapper() (meta.RESTMapper, error) {
|
||||
return restmapper.NewDeferredDiscoveryRESTMapper(r.discoveryClient), nil
|
||||
}
|
||||
|
||||
func (r *RESTClientGetter) ToRawKubeConfigLoader() clientcmd.ClientConfig {
|
||||
return r.clientconfig
|
||||
}
|
||||
+27
-171
@@ -5,41 +5,34 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"maps"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/go-logr/zapr"
|
||||
"github.com/testcontainers/testcontainers-go"
|
||||
"github.com/testcontainers/testcontainers-go/modules/k3s"
|
||||
"go.uber.org/zap"
|
||||
"helm.sh/helm/v3/pkg/action"
|
||||
"helm.sh/helm/v3/pkg/chart/loader"
|
||||
"k8s.io/apimachinery/pkg/api/resource"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"k8s.io/apimachinery/pkg/util/intstr"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
"k8s.io/client-go/rest"
|
||||
"k8s.io/client-go/tools/clientcmd"
|
||||
"k8s.io/client-go/tools/remotecommand"
|
||||
"k8s.io/kubernetes/pkg/api/v1/pod"
|
||||
"k8s.io/utils/ptr"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
"sigs.k8s.io/controller-runtime/pkg/log"
|
||||
|
||||
appsv1 "k8s.io/api/apps/v1"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
networkingv1 "k8s.io/api/networking/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
|
||||
|
||||
"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
|
||||
fwclient "github.com/rancher/k3k/tests/framework/client"
|
||||
fwcontainer "github.com/rancher/k3k/tests/framework/container"
|
||||
fwlog "github.com/rancher/k3k/tests/framework/log"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
@@ -73,13 +66,12 @@ func TestTests(t *testing.T) {
|
||||
}
|
||||
|
||||
var (
|
||||
k3sContainer *k3s.K3sContainer
|
||||
hostIP string
|
||||
restcfg *rest.Config
|
||||
k8s *kubernetes.Clientset
|
||||
k8sClient client.Client
|
||||
kubeconfigPath string
|
||||
helmActionConfig *action.Configuration
|
||||
k3sContainer *k3s.K3sContainer
|
||||
hostIP string
|
||||
restcfg *rest.Config
|
||||
k8s *kubernetes.Clientset
|
||||
k8sClient client.Client
|
||||
kubeconfigPath string
|
||||
)
|
||||
|
||||
var _ = BeforeSuite(func() {
|
||||
@@ -105,143 +97,35 @@ var _ = BeforeSuite(func() {
|
||||
patchPVC(ctx, k8s)
|
||||
})
|
||||
|
||||
func initKubernetesClient(ctx context.Context) {
|
||||
var (
|
||||
err error
|
||||
kubeconfig []byte
|
||||
)
|
||||
|
||||
logger, err := zap.NewDevelopment()
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
log.SetLogger(zapr.NewLogger(logger))
|
||||
|
||||
kubeconfigPath := os.Getenv("KUBECONFIG")
|
||||
Expect(kubeconfigPath).To(Not(BeEmpty()))
|
||||
|
||||
kubeconfig, err = os.ReadFile(kubeconfigPath)
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
|
||||
restcfg, err = clientcmd.RESTConfigFromKubeConfig(kubeconfig)
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
|
||||
hostIP, err = getServerIP(ctx, restcfg)
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
|
||||
k8s, err = kubernetes.NewForConfig(restcfg)
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
|
||||
scheme := buildScheme()
|
||||
k8sClient, err = client.New(restcfg, client.Options{Scheme: scheme})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
}
|
||||
|
||||
func buildScheme() *runtime.Scheme {
|
||||
scheme := runtime.NewScheme()
|
||||
|
||||
err := clientgoscheme.AddToScheme(scheme)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
err = v1beta1.AddToScheme(scheme)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
return scheme
|
||||
}
|
||||
|
||||
func installK3SDocker(ctx context.Context, controllerImage, kubeletImage string) {
|
||||
var (
|
||||
err error
|
||||
kubeconfig []byte
|
||||
)
|
||||
|
||||
k3sHostVersion := os.Getenv("K3S_HOST_VERSION")
|
||||
if k3sHostVersion == "" {
|
||||
k3sHostVersion = k3sVersion
|
||||
}
|
||||
|
||||
k3sHostVersion = strings.ReplaceAll(k3sHostVersion, "+", "-")
|
||||
k3sContainer, kubeconfigPath = fwcontainer.SetupK3s(ctx, k3sHostVersion, controllerImage, kubeletImage)
|
||||
}
|
||||
|
||||
k3sContainer, err = k3s.Run(ctx, "rancher/k3s:"+k3sHostVersion)
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
func initKubernetesClient(ctx context.Context) {
|
||||
scheme := fwclient.NewScheme()
|
||||
config, err := fwclient.InitFromKubeconfig(ctx, scheme, k3sContainer)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
containerIP, err := k3sContainer.ContainerIP(ctx)
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
|
||||
GinkgoWriter.Println("K3s containerIP: " + containerIP)
|
||||
|
||||
kubeconfig, err = k3sContainer.GetKubeConfig(context.Background())
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
|
||||
tmpFile, err := os.CreateTemp("", "kubeconfig-")
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
|
||||
_, err = tmpFile.Write(kubeconfig)
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
Expect(tmpFile.Close()).To(Succeed())
|
||||
kubeconfigPath = tmpFile.Name()
|
||||
|
||||
err = k3sContainer.LoadImages(ctx, controllerImage+":dev", kubeletImage+":dev")
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
DeferCleanup(os.Remove, kubeconfigPath)
|
||||
|
||||
Expect(os.Setenv("KUBECONFIG", kubeconfigPath)).To(Succeed())
|
||||
GinkgoWriter.Printf("KUBECONFIG set to: %s\n", kubeconfigPath)
|
||||
hostIP = config.HostIP
|
||||
restcfg = config.RestConfig
|
||||
k8s = config.Clientset
|
||||
k8sClient = config.Client
|
||||
}
|
||||
|
||||
func installK3kChart(controllerImage, kubeletImage string) {
|
||||
pwd, err := os.Getwd()
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
|
||||
k3kChart, err := loader.Load(path.Join(pwd, "../charts/k3k"))
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
|
||||
helmActionConfig = new(action.Configuration)
|
||||
|
||||
installer := fwcontainer.NewHelmInstaller(controllerImage, kubeletImage, kubeconfigPath)
|
||||
kubeconfig, err := os.ReadFile(kubeconfigPath)
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
restClientGetter, err := NewRESTClientGetter(kubeconfig)
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
restClientGetter, err := fwclient.NewRESTClientGetter(kubeconfig)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
err = helmActionConfig.Init(restClientGetter, k3kNamespace, os.Getenv("HELM_DRIVER"), func(format string, v ...any) {
|
||||
GinkgoWriter.Printf("[Helm] "+format+"\n", v...)
|
||||
})
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
|
||||
iCli := action.NewInstall(helmActionConfig)
|
||||
iCli.ReleaseName = "k3k"
|
||||
iCli.Namespace = k3kNamespace
|
||||
iCli.CreateNamespace = true
|
||||
iCli.Timeout = time.Minute
|
||||
iCli.Wait = true
|
||||
|
||||
controllerMap, _ := k3kChart.Values["controller"].(map[string]any)
|
||||
|
||||
extraEnvArray, _ := controllerMap["extraEnv"].([]map[string]any)
|
||||
extraEnvArray = append(extraEnvArray, map[string]any{
|
||||
"name": "DEBUG",
|
||||
"value": "true",
|
||||
})
|
||||
controllerMap["extraEnv"] = extraEnvArray
|
||||
|
||||
imageMap, _ := controllerMap["image"].(map[string]any)
|
||||
maps.Copy(imageMap, map[string]any{
|
||||
"repository": controllerImage,
|
||||
"tag": "dev",
|
||||
"pullPolicy": "IfNotPresent",
|
||||
})
|
||||
|
||||
agentMap, _ := k3kChart.Values["agent"].(map[string]any)
|
||||
sharedAgentMap, _ := agentMap["shared"].(map[string]any)
|
||||
sharedAgentImageMap, _ := sharedAgentMap["image"].(map[string]any)
|
||||
maps.Copy(sharedAgentImageMap, map[string]any{
|
||||
"repository": kubeletImage,
|
||||
"tag": "dev",
|
||||
})
|
||||
|
||||
release, err := iCli.Run(k3kChart, k3kChart.Values)
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
|
||||
GinkgoWriter.Printf("Helm release '%s' installed in '%s' namespace\n", release.Name, release.Namespace)
|
||||
installer.InstallK3kChart(restClientGetter)
|
||||
}
|
||||
|
||||
func patchPVC(ctx context.Context, clientset *kubernetes.Clientset) {
|
||||
@@ -343,11 +227,11 @@ var _ = AfterSuite(func() {
|
||||
// dump k3s logs
|
||||
k3sLogs, err := k3sContainer.Logs(ctx)
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
writeLogs("k3s.log", k3sLogs)
|
||||
fwlog.WriteLogs("k3s.log", k3sLogs)
|
||||
|
||||
// dump k3k controller logs
|
||||
k3kLogs := getK3kLogs(ctx)
|
||||
writeLogs("k3k.log", k3kLogs)
|
||||
k3kLogs := fwlog.GetK3kPodLogs(ctx, k8sClient, k8s, k3kNamespace)
|
||||
fwlog.WriteLogs("k3k.log", k3kLogs)
|
||||
|
||||
testcontainers.CleanupContainer(GinkgoTB(), k3sContainer)
|
||||
}
|
||||
@@ -451,34 +335,6 @@ func dumpK3kCoverageData(ctx context.Context, folder string) {
|
||||
Expect(k8sClient.Delete(ctx, tarPod)).To(Succeed())
|
||||
}
|
||||
|
||||
func getK3kLogs(ctx context.Context) io.ReadCloser {
|
||||
var podList v1.PodList
|
||||
|
||||
err := k8sClient.List(ctx, &podList, &client.ListOptions{Namespace: k3kNamespace})
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
Expect(podList.Items).NotTo(BeEmpty())
|
||||
|
||||
k3kPod := podList.Items[0]
|
||||
req := k8s.CoreV1().Pods(k3kPod.Namespace).GetLogs(k3kPod.Name, &v1.PodLogOptions{Previous: true})
|
||||
podLogs, err := req.Stream(ctx)
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
|
||||
return podLogs
|
||||
}
|
||||
|
||||
func writeLogs(filename string, logs io.ReadCloser) {
|
||||
logsStr, err := io.ReadAll(logs)
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
|
||||
tempfile := path.Join(os.TempDir(), filename)
|
||||
err = os.WriteFile(tempfile, []byte(logsStr), 0o644)
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
|
||||
GinkgoWriter.Println("logs written to: " + filename)
|
||||
|
||||
_ = logs.Close()
|
||||
}
|
||||
|
||||
func readFileWithinPod(ctx context.Context, client *kubernetes.Clientset, config *rest.Config, name, namespace, path string) ([]byte, error) {
|
||||
command := []string{"cat", path}
|
||||
|
||||
|
||||
@@ -0,0 +1,105 @@
|
||||
package client
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
|
||||
"github.com/go-logr/zapr"
|
||||
"github.com/testcontainers/testcontainers-go/modules/k3s"
|
||||
"go.uber.org/zap"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
"k8s.io/client-go/rest"
|
||||
"k8s.io/client-go/tools/clientcmd"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
"sigs.k8s.io/controller-runtime/pkg/log"
|
||||
)
|
||||
|
||||
// Config holds the Kubernetes client configuration and clients.
|
||||
type Config struct {
|
||||
RestConfig *rest.Config
|
||||
Clientset *kubernetes.Clientset
|
||||
Client client.Client
|
||||
HostIP string
|
||||
}
|
||||
|
||||
// InitFromKubeconfig initializes Kubernetes clients from the KUBECONFIG environment variable.
|
||||
// It sets up logging, reads the kubeconfig file, creates REST config and clients.
|
||||
// The scheme parameter should be created using the scheme package.
|
||||
func InitFromKubeconfig(ctx context.Context, scheme *runtime.Scheme, k3sContainer *k3s.K3sContainer) (*Config, error) {
|
||||
// Setup logger
|
||||
logger, err := zap.NewDevelopment()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create logger: %w", err)
|
||||
}
|
||||
|
||||
log.SetLogger(zapr.NewLogger(logger))
|
||||
|
||||
// Get kubeconfig path from environment
|
||||
kubeconfigPath := os.Getenv("KUBECONFIG")
|
||||
if kubeconfigPath == "" {
|
||||
return nil, fmt.Errorf("KUBECONFIG environment variable is not set")
|
||||
}
|
||||
|
||||
// Read kubeconfig file
|
||||
kubeconfig, err := os.ReadFile(kubeconfigPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read kubeconfig from %s: %w", kubeconfigPath, err)
|
||||
}
|
||||
|
||||
return InitFromBytes(ctx, kubeconfig, scheme, k3sContainer)
|
||||
}
|
||||
|
||||
// InitFromBytes initializes Kubernetes clients from kubeconfig bytes.
|
||||
// The scheme parameter should be created using the scheme package.
|
||||
func InitFromBytes(ctx context.Context, kubeconfig []byte, scheme *runtime.Scheme, k3sContainer *k3s.K3sContainer) (*Config, error) {
|
||||
// Create REST config from kubeconfig
|
||||
restConfig, err := clientcmd.RESTConfigFromKubeConfig(kubeconfig)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create REST config: %w", err)
|
||||
}
|
||||
|
||||
// Extract host IP from REST config
|
||||
hostIP, err := getServerIP(ctx, restConfig, k3sContainer)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get server IP: %w", err)
|
||||
}
|
||||
|
||||
// Create Kubernetes clientset
|
||||
clientset, err := kubernetes.NewForConfig(restConfig)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create Kubernetes clientset: %w", err)
|
||||
}
|
||||
|
||||
// Create controller-runtime client
|
||||
runtimeClient, err := client.New(restConfig, client.Options{Scheme: scheme})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create controller-runtime client: %w", err)
|
||||
}
|
||||
|
||||
return &Config{
|
||||
RestConfig: restConfig,
|
||||
Clientset: clientset,
|
||||
Client: runtimeClient,
|
||||
HostIP: hostIP,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// getServerIP extracts the server IP from the REST config.
|
||||
// If running with testcontainers, it returns the container IP.
|
||||
// Otherwise, it parses the hostname from the REST config host.
|
||||
func getServerIP(ctx context.Context, cfg *rest.Config, k3sContainer *k3s.K3sContainer) (string, error) {
|
||||
if k3sContainer != nil {
|
||||
return k3sContainer.ContainerIP(ctx)
|
||||
}
|
||||
|
||||
u, err := url.Parse(cfg.Host)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to parse REST config host: %w", err)
|
||||
}
|
||||
|
||||
// If Host includes a port, u.Hostname() extracts just the hostname part
|
||||
return u.Hostname(), nil
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
package cli_test
|
||||
package client
|
||||
|
||||
import (
|
||||
"k8s.io/apimachinery/pkg/api/meta"
|
||||
@@ -10,14 +10,19 @@ import (
|
||||
memory "k8s.io/client-go/discovery/cached"
|
||||
)
|
||||
|
||||
// RESTClientGetter is a Kubernetes REST client getter implementation that satisfies
|
||||
// the genericclioptions.RESTClientGetter interface. This is used primarily for Helm
|
||||
// operations in tests.
|
||||
type RESTClientGetter struct {
|
||||
clientconfig clientcmd.ClientConfig
|
||||
restConfig *rest.Config
|
||||
discoveryClient discovery.CachedDiscoveryInterface
|
||||
}
|
||||
|
||||
// NewRESTClientGetter creates a new RESTClientGetter from kubeconfig bytes.
|
||||
// This is used for Helm operations in tests.
|
||||
func NewRESTClientGetter(kubeconfig []byte) (*RESTClientGetter, error) {
|
||||
clientconfig, err := clientcmd.NewClientConfigFromBytes([]byte(kubeconfig))
|
||||
clientconfig, err := clientcmd.NewClientConfigFromBytes(kubeconfig)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -39,18 +44,22 @@ func NewRESTClientGetter(kubeconfig []byte) (*RESTClientGetter, error) {
|
||||
}, nil
|
||||
}
|
||||
|
||||
// ToRESTConfig returns the REST config.
|
||||
func (r *RESTClientGetter) ToRESTConfig() (*rest.Config, error) {
|
||||
return r.restConfig, nil
|
||||
}
|
||||
|
||||
// ToDiscoveryClient returns the cached discovery client.
|
||||
func (r *RESTClientGetter) ToDiscoveryClient() (discovery.CachedDiscoveryInterface, error) {
|
||||
return r.discoveryClient, nil
|
||||
}
|
||||
|
||||
// ToRESTMapper returns a REST mapper from the discovery client.
|
||||
func (r *RESTClientGetter) ToRESTMapper() (meta.RESTMapper, error) {
|
||||
return restmapper.NewDeferredDiscoveryRESTMapper(r.discoveryClient), nil
|
||||
}
|
||||
|
||||
// ToRawKubeConfigLoader returns the raw kubeconfig loader.
|
||||
func (r *RESTClientGetter) ToRawKubeConfigLoader() clientcmd.ClientConfig {
|
||||
return r.clientconfig
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
package client
|
||||
|
||||
import (
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
|
||||
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
|
||||
|
||||
"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
|
||||
)
|
||||
|
||||
// NewScheme creates a new Kubernetes runtime scheme with core APIs and k3k CRDs.
|
||||
// This is suitable for most k3k test scenarios including integration and E2E tests.
|
||||
func NewScheme() *runtime.Scheme {
|
||||
scheme := runtime.NewScheme()
|
||||
|
||||
// Add core Kubernetes scheme (includes most common types)
|
||||
if err := clientgoscheme.AddToScheme(scheme); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
// Add k3k CRDs
|
||||
if err := v1beta1.AddToScheme(scheme); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
return scheme
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
package container
|
||||
|
||||
import (
|
||||
"maps"
|
||||
"os"
|
||||
"path"
|
||||
"time"
|
||||
|
||||
"helm.sh/helm/v3/pkg/action"
|
||||
"helm.sh/helm/v3/pkg/chart/loader"
|
||||
|
||||
fwclient "github.com/rancher/k3k/tests/framework/client"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
// HelmInstaller provides configuration for Helm chart installation.
|
||||
type HelmInstaller struct {
|
||||
ChartPath string
|
||||
Namespace string
|
||||
ReleaseName string
|
||||
Timeout time.Duration
|
||||
Wait bool
|
||||
ControllerImage string
|
||||
KubeletImage string
|
||||
KubeconfigPath string
|
||||
}
|
||||
|
||||
// InstallK3kChart installs the k3k Helm chart with the specified configuration.
|
||||
// It uses the provided RESTClientGetter for authentication and returns the Helm action configuration.
|
||||
func (h *HelmInstaller) InstallK3kChart(restClientGetter *fwclient.RESTClientGetter) *action.Configuration {
|
||||
GinkgoHelper()
|
||||
|
||||
// Load chart
|
||||
k3kChart, err := loader.Load(h.ChartPath)
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
|
||||
// Initialize Helm action configuration
|
||||
helmActionConfig := new(action.Configuration)
|
||||
|
||||
err = helmActionConfig.Init(restClientGetter, h.Namespace, os.Getenv("HELM_DRIVER"), func(format string, v ...any) {
|
||||
GinkgoWriter.Printf("[Helm] "+format+"\n", v...)
|
||||
})
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
|
||||
// Create install action
|
||||
iCli := action.NewInstall(helmActionConfig)
|
||||
iCli.ReleaseName = h.ReleaseName
|
||||
iCli.Namespace = h.Namespace
|
||||
iCli.CreateNamespace = true
|
||||
iCli.Timeout = h.Timeout
|
||||
iCli.Wait = h.Wait
|
||||
|
||||
// Configure controller image
|
||||
controllerMap, _ := k3kChart.Values["controller"].(map[string]any)
|
||||
|
||||
extraEnvArray, _ := controllerMap["extraEnv"].([]map[string]any)
|
||||
extraEnvArray = append(extraEnvArray, map[string]any{
|
||||
"name": "DEBUG",
|
||||
"value": "true",
|
||||
})
|
||||
controllerMap["extraEnv"] = extraEnvArray
|
||||
|
||||
imageMap, _ := controllerMap["image"].(map[string]any)
|
||||
maps.Copy(imageMap, map[string]any{
|
||||
"repository": h.ControllerImage,
|
||||
"tag": "dev",
|
||||
"pullPolicy": "IfNotPresent",
|
||||
})
|
||||
|
||||
// Configure agent image
|
||||
agentMap, _ := k3kChart.Values["agent"].(map[string]any)
|
||||
sharedAgentMap, _ := agentMap["shared"].(map[string]any)
|
||||
sharedAgentImageMap, _ := sharedAgentMap["image"].(map[string]any)
|
||||
maps.Copy(sharedAgentImageMap, map[string]any{
|
||||
"repository": h.KubeletImage,
|
||||
"tag": "dev",
|
||||
})
|
||||
|
||||
// Install chart
|
||||
release, err := iCli.Run(k3kChart, k3kChart.Values)
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
|
||||
GinkgoWriter.Printf("Helm release '%s' installed in '%s' namespace\n", release.Name, release.Namespace)
|
||||
|
||||
return helmActionConfig
|
||||
}
|
||||
|
||||
// NewHelmInstaller creates a new HelmInstaller with default values.
|
||||
func NewHelmInstaller(controllerImage, kubeletImage, kubeconfigPath string) *HelmInstaller {
|
||||
pwd, err := os.Getwd()
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
|
||||
return &HelmInstaller{
|
||||
ChartPath: path.Join(pwd, "../../charts/k3k"),
|
||||
Namespace: "k3k-system",
|
||||
ReleaseName: "k3k",
|
||||
Timeout: time.Minute,
|
||||
Wait: true,
|
||||
ControllerImage: controllerImage,
|
||||
KubeletImage: kubeletImage,
|
||||
KubeconfigPath: kubeconfigPath,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
package container
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/testcontainers/testcontainers-go/modules/k3s"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
// SetupK3s creates and starts a K3s testcontainer with the specified version and loads the provided images.
|
||||
// It returns the container instance and a temporary kubeconfig file path.
|
||||
// The kubeconfig is automatically cleaned up via DeferCleanup.
|
||||
func SetupK3s(ctx context.Context, k3sVersion, controllerImage, kubeletImage string) (*k3s.K3sContainer, string) {
|
||||
GinkgoHelper()
|
||||
|
||||
var (
|
||||
err error
|
||||
kubeconfig []byte
|
||||
)
|
||||
|
||||
// Normalize version string (replace + with -)
|
||||
k3sVersion = strings.ReplaceAll(k3sVersion, "+", "-")
|
||||
|
||||
// Start K3s container
|
||||
k3sContainer, err := k3s.Run(ctx, "rancher/k3s:"+k3sVersion)
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
|
||||
containerIP, err := k3sContainer.ContainerIP(ctx)
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
|
||||
GinkgoWriter.Println("K3s containerIP: " + containerIP)
|
||||
|
||||
// Get kubeconfig from container
|
||||
kubeconfig, err = k3sContainer.GetKubeConfig(ctx)
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
|
||||
// Write kubeconfig to temp file
|
||||
tmpFile, err := os.CreateTemp("", "kubeconfig-")
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
|
||||
_, err = tmpFile.Write(kubeconfig)
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
Expect(tmpFile.Close()).To(Succeed())
|
||||
|
||||
kubeconfigPath := tmpFile.Name()
|
||||
|
||||
// Load images into the K3s container
|
||||
err = k3sContainer.LoadImages(ctx, controllerImage+":dev", kubeletImage+":dev")
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
|
||||
// Register cleanup
|
||||
DeferCleanup(os.Remove, kubeconfigPath)
|
||||
|
||||
// Set KUBECONFIG environment variable
|
||||
Expect(os.Setenv("KUBECONFIG", kubeconfigPath)).To(Succeed())
|
||||
GinkgoWriter.Printf("KUBECONFIG set to: %s\n", kubeconfigPath)
|
||||
|
||||
return k3sContainer, kubeconfigPath
|
||||
}
|
||||
Vendored
+34
@@ -0,0 +1,34 @@
|
||||
package env
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"sigs.k8s.io/controller-runtime/pkg/envtest"
|
||||
)
|
||||
|
||||
// Config holds the envtest environment configuration.
|
||||
type Config struct {
|
||||
CRDPaths []string
|
||||
Scheme *runtime.Scheme
|
||||
}
|
||||
|
||||
// NewEnvironment creates a new envtest.Environment with CRDs from the k3k charts directory.
|
||||
// The scheme parameter should be created using the scheme package.
|
||||
func NewEnvironment(scheme *runtime.Scheme) *envtest.Environment {
|
||||
return &envtest.Environment{
|
||||
CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "charts", "k3k", "templates", "crds")},
|
||||
ErrorIfCRDPathMissing: true,
|
||||
Scheme: scheme,
|
||||
}
|
||||
}
|
||||
|
||||
// NewEnvironmentWithPaths creates a new envtest.Environment with custom CRD paths.
|
||||
// This is useful when running tests from different locations in the directory tree.
|
||||
func NewEnvironmentWithPaths(scheme *runtime.Scheme, crdPaths ...string) *envtest.Environment {
|
||||
return &envtest.Environment{
|
||||
CRDDirectoryPaths: crdPaths,
|
||||
ErrorIfCRDPathMissing: true,
|
||||
Scheme: scheme,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
package k3k
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"sync"
|
||||
|
||||
"k8s.io/client-go/kubernetes"
|
||||
"k8s.io/utils/ptr"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
|
||||
v1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
// CreateNamespace creates a new namespace with a generated name and the "e2e: true" label.
|
||||
// The namespace is created using the provided Kubernetes clientset.
|
||||
func CreateNamespace(clientset kubernetes.Interface) *v1.Namespace {
|
||||
GinkgoHelper()
|
||||
|
||||
namespace := &v1.Namespace{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
GenerateName: "ns-",
|
||||
Labels: map[string]string{
|
||||
"e2e": "true",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
namespace, err := clientset.CoreV1().Namespaces().Create(context.Background(), namespace, metav1.CreateOptions{})
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
|
||||
return namespace
|
||||
}
|
||||
|
||||
// DeleteNamespaces deletes the specified namespaces in parallel.
|
||||
// If the KEEP_NAMESPACES environment variable is set, namespaces are preserved instead.
|
||||
// This is useful for debugging test failures.
|
||||
func DeleteNamespaces(clientset kubernetes.Interface, names ...string) {
|
||||
GinkgoHelper()
|
||||
|
||||
if _, found := os.LookupEnv("KEEP_NAMESPACES"); found {
|
||||
By(fmt.Sprintf("Keeping namespaces %v", names))
|
||||
return
|
||||
}
|
||||
|
||||
wg := sync.WaitGroup{}
|
||||
wg.Add(len(names))
|
||||
|
||||
for _, name := range names {
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
defer GinkgoRecover()
|
||||
|
||||
By(fmt.Sprintf("Deleting namespace %s", name))
|
||||
|
||||
err := clientset.CoreV1().Namespaces().Delete(context.Background(), name, metav1.DeleteOptions{
|
||||
GracePeriodSeconds: ptr.To[int64](0),
|
||||
})
|
||||
Expect(client.IgnoreNotFound(err)).To(Not(HaveOccurred()))
|
||||
}()
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
package log
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"os"
|
||||
"path"
|
||||
|
||||
"k8s.io/client-go/kubernetes"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
|
||||
v1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
// GetK3kPodLogs retrieves logs from the first k3k pod in the specified namespace.
|
||||
// This is useful for debugging test failures.
|
||||
func GetK3kPodLogs(ctx context.Context, k8sClient client.Client, clientset kubernetes.Interface, namespace string) io.ReadCloser {
|
||||
GinkgoHelper()
|
||||
|
||||
var podList v1.PodList
|
||||
|
||||
err := k8sClient.List(ctx, &podList, &client.ListOptions{Namespace: namespace})
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
Expect(podList.Items).NotTo(BeEmpty())
|
||||
|
||||
k3kPod := podList.Items[0]
|
||||
|
||||
// Fetch complete pod object to access status information
|
||||
pod, err := clientset.CoreV1().Pods(k3kPod.Namespace).Get(ctx, k3kPod.Name, metav1.GetOptions{})
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
|
||||
// Detect if the container has been restarted (e.g., for coverage dumping in E2E tests)
|
||||
fetchPrevious := false
|
||||
|
||||
if len(pod.Status.ContainerStatuses) > 0 {
|
||||
containerStatus := pod.Status.ContainerStatuses[0]
|
||||
|
||||
if containerStatus.RestartCount > 0 {
|
||||
fetchPrevious = true
|
||||
|
||||
GinkgoWriter.Printf("Container has been restarted %d time(s), fetching previous logs\n", containerStatus.RestartCount)
|
||||
}
|
||||
}
|
||||
|
||||
req := clientset.CoreV1().Pods(pod.Namespace).GetLogs(pod.Name, &v1.PodLogOptions{Previous: fetchPrevious})
|
||||
podLogs, err := req.Stream(ctx)
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
|
||||
return podLogs
|
||||
}
|
||||
|
||||
// WriteLogs writes the provided logs to a temporary file with the specified filename.
|
||||
// The file is written to os.TempDir() and the full path is logged to GinkgoWriter.
|
||||
func WriteLogs(filename string, logs io.ReadCloser) {
|
||||
GinkgoHelper()
|
||||
|
||||
logsStr, err := io.ReadAll(logs)
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
|
||||
tempfile := path.Join(os.TempDir(), filename)
|
||||
err = os.WriteFile(tempfile, logsStr, 0o644)
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
|
||||
GinkgoWriter.Println("logs written to: " + tempfile)
|
||||
|
||||
_ = logs.Close()
|
||||
}
|
||||
@@ -8,19 +8,17 @@ import (
|
||||
|
||||
"github.com/go-logr/zapr"
|
||||
"go.uber.org/zap"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
"k8s.io/client-go/tools/record"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
"sigs.k8s.io/controller-runtime/pkg/envtest"
|
||||
|
||||
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
|
||||
ctrl "sigs.k8s.io/controller-runtime"
|
||||
metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
|
||||
|
||||
"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
|
||||
"github.com/rancher/k3k/pkg/controller/cluster"
|
||||
"github.com/rancher/k3k/pkg/controller/cluster/agent"
|
||||
fwclient "github.com/rancher/k3k/tests/framework/client"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
@@ -56,7 +54,7 @@ var _ = BeforeSuite(func() {
|
||||
k8s, err = kubernetes.NewForConfig(cfg)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
scheme := buildScheme()
|
||||
scheme := fwclient.NewScheme()
|
||||
k8sClient, err = client.New(cfg, client.Options{Scheme: scheme})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
@@ -97,14 +95,3 @@ var _ = AfterSuite(func() {
|
||||
err := testEnv.Stop()
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
})
|
||||
|
||||
func buildScheme() *runtime.Scheme {
|
||||
scheme := runtime.NewScheme()
|
||||
|
||||
err := clientgoscheme.AddToScheme(scheme)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
err = v1beta1.AddToScheme(scheme)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
return scheme
|
||||
}
|
||||
|
||||
@@ -10,17 +10,16 @@ import (
|
||||
|
||||
"github.com/go-logr/zapr"
|
||||
"go.uber.org/zap"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
"sigs.k8s.io/controller-runtime/pkg/envtest"
|
||||
|
||||
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
|
||||
ctrl "sigs.k8s.io/controller-runtime"
|
||||
metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
|
||||
|
||||
"github.com/rancher/k3k/k3k-kubelet/translate"
|
||||
"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
|
||||
fwclient "github.com/rancher/k3k/tests/framework/client"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
@@ -95,7 +94,7 @@ func NewTestEnv() *TestEnv {
|
||||
CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "charts", "k3k", "templates", "crds")},
|
||||
ErrorIfCRDPathMissing: true,
|
||||
BinaryAssetsDirectory: tempDir,
|
||||
Scheme: buildScheme(),
|
||||
Scheme: fwclient.NewScheme(),
|
||||
}
|
||||
|
||||
cfg, err := testEnv.Start()
|
||||
@@ -114,17 +113,6 @@ func NewTestEnv() *TestEnv {
|
||||
}
|
||||
}
|
||||
|
||||
func buildScheme() *runtime.Scheme {
|
||||
scheme := runtime.NewScheme()
|
||||
|
||||
err := clientgoscheme.AddToScheme(scheme)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
err = v1beta1.AddToScheme(scheme)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
return scheme
|
||||
}
|
||||
|
||||
var _ = Describe("Kubelet Controller", func() {
|
||||
var (
|
||||
ctx context.Context
|
||||
|
||||
@@ -7,18 +7,14 @@ import (
|
||||
|
||||
"github.com/go-logr/zapr"
|
||||
"go.uber.org/zap"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
"sigs.k8s.io/controller-runtime/pkg/envtest"
|
||||
|
||||
appsv1 "k8s.io/api/apps/v1"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
networkingv1 "k8s.io/api/networking/v1"
|
||||
ctrl "sigs.k8s.io/controller-runtime"
|
||||
metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
|
||||
|
||||
"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
|
||||
"github.com/rancher/k3k/pkg/controller/policy"
|
||||
fwclient "github.com/rancher/k3k/tests/framework/client"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
@@ -46,7 +42,7 @@ var _ = BeforeSuite(func() {
|
||||
cfg, err := testEnv.Start()
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
scheme := buildScheme()
|
||||
scheme := fwclient.NewScheme()
|
||||
k8sClient, err = client.New(cfg, client.Options{Scheme: scheme})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
@@ -75,18 +71,3 @@ var _ = AfterSuite(func() {
|
||||
err := testEnv.Stop()
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
})
|
||||
|
||||
func buildScheme() *runtime.Scheme {
|
||||
scheme := runtime.NewScheme()
|
||||
|
||||
err := corev1.AddToScheme(scheme)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
err = appsv1.AddToScheme(scheme)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
err = networkingv1.AddToScheme(scheme)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
err = v1beta1.AddToScheme(scheme)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
return scheme
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user