Add Cluster creation test (#192)

* added k3kcli to path

* test create cluster

* updated ptr

* added cluster creation test

.github/workflows/test.yaml

@@ -81,7 +81,14 @@ jobs:
       run: go install github.com/onsi/ginkgo/v2/ginkgo
     
     - name: Build
-      run: ./scripts/build
+      run: |
+        ./scripts/build
+
+        # add k3kcli to $PATH
+        echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
+
+    - name: Check k3kcli
+      run: k3kcli -v
 
     - name: Run tests
       run: ginkgo -v ./tests

pkg/controller/cluster/agent/shared.go

@@ -171,9 +171,8 @@ func (s *SharedAgent) podSpec(affinitySelector *metav1.LabelSelector) v1.PodSpec
 		},
 		Containers: []v1.Container{
 			{
-				Name:            s.Name(),
-				Image:           s.sharedAgentImage,
-				ImagePullPolicy: v1.PullAlways,
+				Name:  s.Name(),
+				Image: s.sharedAgentImage,
 				Resources: v1.ResourceRequirements{
 					Limits: limit,
 				},

pkg/controller/cluster/agent/virtual.go

@@ -161,9 +161,8 @@ func (v *VirtualAgent) podSpec(image, name string, args []string, affinitySelect
 		},
 		Containers: []v1.Container{
 			{
-				Name:            name,
-				Image:           image,
-				ImagePullPolicy: v1.PullAlways,
+				Name:  name,
+				Image: image,
 				SecurityContext: &v1.SecurityContext{
 					Privileged: ptr.To(true),
 				},

pkg/controller/clusterset/clusterset_test.go

@@ -488,8 +488,8 @@ var _ = Describe("ClusterSet Controller", func() {
 					},
 					Spec: v1alpha1.ClusterSpec{
 						Mode:    v1alpha1.SharedClusterMode,
-						Servers: ptr.To(int32(1)),
-						Agents:  ptr.To(int32(0)),
+						Servers: ptr.To[int32](1),
+						Agents:  ptr.To[int32](0),
 					},
 				}
 
@@ -529,8 +529,8 @@ var _ = Describe("ClusterSet Controller", func() {
 					},
 					Spec: v1alpha1.ClusterSpec{
 						Mode:    v1alpha1.SharedClusterMode,
-						Servers: ptr.To(int32(1)),
-						Agents:  ptr.To(int32(0)),
+						Servers: ptr.To[int32](1),
+						Agents:  ptr.To[int32](0),
 					},
 				}
 
@@ -570,8 +570,8 @@ var _ = Describe("ClusterSet Controller", func() {
 					},
 					Spec: v1alpha1.ClusterSpec{
 						Mode:         v1alpha1.SharedClusterMode,
-						Servers:      ptr.To(int32(1)),
-						Agents:       ptr.To(int32(0)),
+						Servers:      ptr.To[int32](1),
+						Agents:       ptr.To[int32](0),
 						NodeSelector: map[string]string{"label-1": "value-1"},
 					},
 				}
@@ -645,8 +645,8 @@ var _ = Describe("ClusterSet Controller", func() {
 					},
 					Spec: v1alpha1.ClusterSpec{
 						Mode:    v1alpha1.SharedClusterMode,
-						Servers: ptr.To(int32(1)),
-						Agents:  ptr.To(int32(0)),
+						Servers: ptr.To[int32](1),
+						Agents:  ptr.To[int32](0),
 					},
 				}
 

tests/cluster_test.go

@@ -0,0 +1,88 @@
+package k3k_test
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
+	corev1 "k8s.io/api/core/v1"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/utils/ptr"
+)
+
+var _ = When("a cluster is installed", func() {
+
+	var namespace string
+
+	BeforeEach(func() {
+		createdNS := &corev1.Namespace{ObjectMeta: v1.ObjectMeta{GenerateName: "ns-"}}
+		createdNS, err := k8s.CoreV1().Namespaces().Create(context.Background(), createdNS, v1.CreateOptions{})
+		Expect(err).To(Not(HaveOccurred()))
+		namespace = createdNS.Name
+	})
+
+	It("will be created in shared mode", func() {
+		cluster := v1alpha1.Cluster{
+			ObjectMeta: v1.ObjectMeta{
+				Name:      "mycluster",
+				Namespace: namespace,
+			},
+			Spec: v1alpha1.ClusterSpec{
+				Mode:    v1alpha1.SharedClusterMode,
+				Servers: ptr.To[int32](1),
+				Agents:  ptr.To[int32](0),
+				Version: "v1.26.1-k3s1",
+			},
+		}
+
+		err := k8sClient.Create(context.Background(), &cluster)
+		Expect(err).To(Not(HaveOccurred()))
+
+		By("checking server and kubelet readiness state")
+
+		// check that the server Pod and the Kubelet are in Ready state
+		Eventually(func() bool {
+			podList, err := k8s.CoreV1().Pods(namespace).List(context.Background(), v1.ListOptions{})
+			Expect(err).To(Not(HaveOccurred()))
+
+			serverRunning := false
+			kubeletRunning := false
+
+			for _, pod := range podList.Items {
+				imageName := pod.Spec.Containers[0].Image
+				imageName = strings.Split(imageName, ":")[0] // remove tag
+
+				switch imageName {
+				case "rancher/k3s":
+					serverRunning = pod.Status.Phase == corev1.PodRunning
+				case "rancher/k3k-kubelet":
+					kubeletRunning = pod.Status.Phase == corev1.PodRunning
+				}
+
+				if serverRunning && kubeletRunning {
+					return true
+				}
+			}
+
+			return false
+		}).
+			WithTimeout(time.Minute).
+			WithPolling(time.Second * 5).
+			Should(BeTrue())
+
+		By("checking the existence of the bootstrap secret")
+		secretName := fmt.Sprintf("k3k-%s-bootstrap", cluster.Name)
+
+		Eventually(func() error {
+			_, err := k8s.CoreV1().Secrets(namespace).Get(context.Background(), secretName, v1.GetOptions{})
+			return err
+		}).
+			WithTimeout(time.Minute * 2).
+			WithPolling(time.Second * 5).
+			Should(BeNil())
+	})
+})

tests/tests_suite_test.go

@@ -10,16 +10,22 @@ import (
 	"testing"
 	"time"
 
+	"github.com/go-logr/zapr"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
 	"github.com/testcontainers/testcontainers-go"
 	"github.com/testcontainers/testcontainers-go/modules/k3s"
+	"go.uber.org/zap"
 	"helm.sh/helm/v3/pkg/action"
 	"helm.sh/helm/v3/pkg/chart/loader"
 	corev1 "k8s.io/api/core/v1"
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/clientcmd"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/log"
 )
 
 func TestTests(t *testing.T) {
@@ -30,6 +36,7 @@ func TestTests(t *testing.T) {
 var (
 	k3sContainer *k3s.K3sContainer
 	k8s          *kubernetes.Clientset
+	k8sClient    client.Client
 )
 
 var _ = BeforeSuite(func() {
@@ -52,6 +59,14 @@ func initKubernetesClient(kubeconfig []byte) {
 
 	k8s, err = kubernetes.NewForConfig(restcfg)
 	Expect(err).To(Not(HaveOccurred()))
+
+	scheme := buildScheme()
+	k8sClient, err = client.New(restcfg, client.Options{Scheme: scheme})
+	Expect(err).NotTo(HaveOccurred())
+
+	logger, err := zap.NewDevelopment()
+	Expect(err).NotTo(HaveOccurred())
+	log.SetLogger(zapr.NewLogger(logger))
 }
 
 func installK3kChart(kubeconfig []byte) {
@@ -155,3 +170,10 @@ var _ = When("k3k is installed", func() {
 			Should(BeTrue())
 	})
 })
+
+func buildScheme() *runtime.Scheme {
+	scheme := runtime.NewScheme()
+	err := v1alpha1.AddToScheme(scheme)
+	Expect(err).NotTo(HaveOccurred())
+	return scheme
+}