From 3f49593f96cd31402b04698d94ddb79d1355714c Mon Sep 17 00:00:00 2001 From: Enrico Candino Date: Tue, 21 Jan 2025 17:53:42 +0100 Subject: [PATCH] Add Cluster creation test (#192) * added k3kcli to path * test create cluster * updated ptr * added cluster creation test --- .github/workflows/test.yaml | 9 +- pkg/controller/cluster/agent/shared.go | 5 +- pkg/controller/cluster/agent/virtual.go | 5 +- pkg/controller/clusterset/clusterset_test.go | 16 ++-- tests/cluster_test.go | 88 +++++++++++++++++++ ...getter.go => k8s_restclientgetter_test.go} | 0 tests/tests_suite_test.go | 22 +++++ 7 files changed, 130 insertions(+), 15 deletions(-) create mode 100644 tests/cluster_test.go rename tests/{k8s_restclientgetter.go => k8s_restclientgetter_test.go} (100%) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index d6c09c85..69d738c0 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -81,7 +81,14 @@ jobs: run: go install github.com/onsi/ginkgo/v2/ginkgo - name: Build - run: ./scripts/build + run: | + ./scripts/build + + # add k3kcli to $PATH + echo "${{ github.workspace }}/bin" >> $GITHUB_PATH + + - name: Check k3kcli + run: k3kcli -v - name: Run tests run: ginkgo -v ./tests diff --git a/pkg/controller/cluster/agent/shared.go b/pkg/controller/cluster/agent/shared.go index 17a24494..d469b6b9 100644 --- a/pkg/controller/cluster/agent/shared.go +++ b/pkg/controller/cluster/agent/shared.go @@ -171,9 +171,8 @@ func (s *SharedAgent) podSpec(affinitySelector *metav1.LabelSelector) v1.PodSpec }, Containers: []v1.Container{ { - Name: s.Name(), - Image: s.sharedAgentImage, - ImagePullPolicy: v1.PullAlways, + Name: s.Name(), + Image: s.sharedAgentImage, Resources: v1.ResourceRequirements{ Limits: limit, }, diff --git a/pkg/controller/cluster/agent/virtual.go b/pkg/controller/cluster/agent/virtual.go index eabc6b97..5092395c 100644 --- a/pkg/controller/cluster/agent/virtual.go +++ b/pkg/controller/cluster/agent/virtual.go @@ -161,9 +161,8 @@ func (v *VirtualAgent) podSpec(image, name string, args []string, affinitySelect }, Containers: []v1.Container{ { - Name: name, - Image: image, - ImagePullPolicy: v1.PullAlways, + Name: name, + Image: image, SecurityContext: &v1.SecurityContext{ Privileged: ptr.To(true), }, diff --git a/pkg/controller/clusterset/clusterset_test.go b/pkg/controller/clusterset/clusterset_test.go index a33fdceb..b3310851 100644 --- a/pkg/controller/clusterset/clusterset_test.go +++ b/pkg/controller/clusterset/clusterset_test.go @@ -488,8 +488,8 @@ var _ = Describe("ClusterSet Controller", func() { }, Spec: v1alpha1.ClusterSpec{ Mode: v1alpha1.SharedClusterMode, - Servers: ptr.To(int32(1)), - Agents: ptr.To(int32(0)), + Servers: ptr.To[int32](1), + Agents: ptr.To[int32](0), }, } @@ -529,8 +529,8 @@ var _ = Describe("ClusterSet Controller", func() { }, Spec: v1alpha1.ClusterSpec{ Mode: v1alpha1.SharedClusterMode, - Servers: ptr.To(int32(1)), - Agents: ptr.To(int32(0)), + Servers: ptr.To[int32](1), + Agents: ptr.To[int32](0), }, } @@ -570,8 +570,8 @@ var _ = Describe("ClusterSet Controller", func() { }, Spec: v1alpha1.ClusterSpec{ Mode: v1alpha1.SharedClusterMode, - Servers: ptr.To(int32(1)), - Agents: ptr.To(int32(0)), + Servers: ptr.To[int32](1), + Agents: ptr.To[int32](0), NodeSelector: map[string]string{"label-1": "value-1"}, }, } @@ -645,8 +645,8 @@ var _ = Describe("ClusterSet Controller", func() { }, Spec: v1alpha1.ClusterSpec{ Mode: v1alpha1.SharedClusterMode, - Servers: ptr.To(int32(1)), - Agents: ptr.To(int32(0)), + Servers: ptr.To[int32](1), + Agents: ptr.To[int32](0), }, } diff --git a/tests/cluster_test.go b/tests/cluster_test.go new file mode 100644 index 00000000..730b6557 --- /dev/null +++ b/tests/cluster_test.go @@ -0,0 +1,88 @@ +package k3k_test + +import ( + "context" + "fmt" + "strings" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1" + corev1 "k8s.io/api/core/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" +) + +var _ = When("a cluster is installed", func() { + + var namespace string + + BeforeEach(func() { + createdNS := &corev1.Namespace{ObjectMeta: v1.ObjectMeta{GenerateName: "ns-"}} + createdNS, err := k8s.CoreV1().Namespaces().Create(context.Background(), createdNS, v1.CreateOptions{}) + Expect(err).To(Not(HaveOccurred())) + namespace = createdNS.Name + }) + + It("will be created in shared mode", func() { + cluster := v1alpha1.Cluster{ + ObjectMeta: v1.ObjectMeta{ + Name: "mycluster", + Namespace: namespace, + }, + Spec: v1alpha1.ClusterSpec{ + Mode: v1alpha1.SharedClusterMode, + Servers: ptr.To[int32](1), + Agents: ptr.To[int32](0), + Version: "v1.26.1-k3s1", + }, + } + + err := k8sClient.Create(context.Background(), &cluster) + Expect(err).To(Not(HaveOccurred())) + + By("checking server and kubelet readiness state") + + // check that the server Pod and the Kubelet are in Ready state + Eventually(func() bool { + podList, err := k8s.CoreV1().Pods(namespace).List(context.Background(), v1.ListOptions{}) + Expect(err).To(Not(HaveOccurred())) + + serverRunning := false + kubeletRunning := false + + for _, pod := range podList.Items { + imageName := pod.Spec.Containers[0].Image + imageName = strings.Split(imageName, ":")[0] // remove tag + + switch imageName { + case "rancher/k3s": + serverRunning = pod.Status.Phase == corev1.PodRunning + case "rancher/k3k-kubelet": + kubeletRunning = pod.Status.Phase == corev1.PodRunning + } + + if serverRunning && kubeletRunning { + return true + } + } + + return false + }). + WithTimeout(time.Minute). + WithPolling(time.Second * 5). + Should(BeTrue()) + + By("checking the existence of the bootstrap secret") + secretName := fmt.Sprintf("k3k-%s-bootstrap", cluster.Name) + + Eventually(func() error { + _, err := k8s.CoreV1().Secrets(namespace).Get(context.Background(), secretName, v1.GetOptions{}) + return err + }). + WithTimeout(time.Minute * 2). + WithPolling(time.Second * 5). + Should(BeNil()) + }) +}) diff --git a/tests/k8s_restclientgetter.go b/tests/k8s_restclientgetter_test.go similarity index 100% rename from tests/k8s_restclientgetter.go rename to tests/k8s_restclientgetter_test.go diff --git a/tests/tests_suite_test.go b/tests/tests_suite_test.go index 05d2aaaa..36601eeb 100644 --- a/tests/tests_suite_test.go +++ b/tests/tests_suite_test.go @@ -10,16 +10,22 @@ import ( "testing" "time" + "github.com/go-logr/zapr" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1" "github.com/testcontainers/testcontainers-go" "github.com/testcontainers/testcontainers-go/modules/k3s" + "go.uber.org/zap" "helm.sh/helm/v3/pkg/action" "helm.sh/helm/v3/pkg/chart/loader" corev1 "k8s.io/api/core/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/clientcmd" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" ) func TestTests(t *testing.T) { @@ -30,6 +36,7 @@ func TestTests(t *testing.T) { var ( k3sContainer *k3s.K3sContainer k8s *kubernetes.Clientset + k8sClient client.Client ) var _ = BeforeSuite(func() { @@ -52,6 +59,14 @@ func initKubernetesClient(kubeconfig []byte) { k8s, err = kubernetes.NewForConfig(restcfg) Expect(err).To(Not(HaveOccurred())) + + scheme := buildScheme() + k8sClient, err = client.New(restcfg, client.Options{Scheme: scheme}) + Expect(err).NotTo(HaveOccurred()) + + logger, err := zap.NewDevelopment() + Expect(err).NotTo(HaveOccurred()) + log.SetLogger(zapr.NewLogger(logger)) } func installK3kChart(kubeconfig []byte) { @@ -155,3 +170,10 @@ var _ = When("k3k is installed", func() { Should(BeTrue()) }) }) + +func buildScheme() *runtime.Scheme { + scheme := runtime.NewScheme() + err := v1alpha1.AddToScheme(scheme) + Expect(err).NotTo(HaveOccurred()) + return scheme +}