k3k/pkg/controller/cluster/cluster_test.go

package cluster_test

import (
	"context"
	"strings"
	"time"

	"k8s.io/apimachinery/pkg/api/resource"
	"k8s.io/utils/ptr"
	"sigs.k8s.io/controller-runtime/pkg/client"

	appsv1 "k8s.io/api/apps/v1"
	corev1 "k8s.io/api/core/v1"
	networkingv1 "k8s.io/api/networking/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
	k3kcontroller "github.com/rancher/k3k/pkg/controller"
	"github.com/rancher/k3k/pkg/controller/cluster/server"

	. "github.com/onsi/ginkgo/v2"
	. "github.com/onsi/gomega"
)

var _ = Describe("Cluster Controller", Label("controller"), Label("Cluster"), func() {
	Context("creating a Cluster", func() {
		var (
			namespace string
			ctx       context.Context
		)

		BeforeEach(func() {
			ctx = context.Background()

			createdNS := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{GenerateName: "ns-"}}
			err := k8sClient.Create(context.Background(), createdNS)
			Expect(err).To(Not(HaveOccurred()))

			namespace = createdNS.Name
		})

		When("creating a Cluster", func() {
			It("will be created with some defaults", func() {
				cluster := &v1beta1.Cluster{
					ObjectMeta: metav1.ObjectMeta{
						GenerateName: "cluster-",
						Namespace:    namespace,
					},
				}

				err := k8sClient.Create(ctx, cluster)
				Expect(err).To(Not(HaveOccurred()))

				Expect(cluster.Spec.Mode).To(Equal(v1beta1.SharedClusterMode))
				Expect(cluster.Spec.Agents).To(Equal(ptr.To[int32](0)))
				Expect(cluster.Spec.Servers).To(Equal(ptr.To[int32](1)))
				Expect(cluster.Spec.Version).To(BeEmpty())

				Expect(cluster.Spec.CustomCAs).To(BeNil())

				// sync
				// enabled by default
				Expect(cluster.Spec.Sync).To(Not(BeNil()))
				Expect(cluster.Spec.Sync.ConfigMaps.Enabled).To(BeTrue())
				Expect(cluster.Spec.Sync.PersistentVolumeClaims.Enabled).To(BeTrue())
				Expect(cluster.Spec.Sync.Secrets.Enabled).To(BeTrue())
				Expect(cluster.Spec.Sync.Services.Enabled).To(BeTrue())
				// disabled by default
				Expect(cluster.Spec.Sync.Ingresses.Enabled).To(BeFalse())
				Expect(cluster.Spec.Sync.PriorityClasses.Enabled).To(BeFalse())

				Expect(cluster.Spec.Persistence.Type).To(Equal(v1beta1.DynamicPersistenceMode))
				Expect(cluster.Spec.Persistence.StorageRequestSize.Equal(resource.MustParse("2G"))).To(BeTrue())

				Expect(cluster.Status.Phase).To(Equal(v1beta1.ClusterUnknown))

				serverVersion, err := k8s.ServerVersion()
				Expect(err).To(Not(HaveOccurred()))

				Eventually(func() string {
					err := k8sClient.Get(ctx, client.ObjectKeyFromObject(cluster), cluster)
					Expect(err).To(Not(HaveOccurred()))

					return cluster.Status.HostVersion
				}).
					WithTimeout(time.Second * 30).
					WithPolling(time.Second).
					Should(Equal(serverVersion.GitVersion))

				// check NetworkPolicy
				expectedNetworkPolicy := &networkingv1.NetworkPolicy{
					ObjectMeta: metav1.ObjectMeta{
						Name:      k3kcontroller.SafeConcatNameWithPrefix(cluster.Name),
						Namespace: cluster.Namespace,
					},
				}

				err = k8sClient.Get(ctx, client.ObjectKeyFromObject(expectedNetworkPolicy), expectedNetworkPolicy)
				Expect(err).To(Not(HaveOccurred()))

				spec := expectedNetworkPolicy.Spec
				Expect(spec.PolicyTypes).To(HaveLen(2))
				Expect(spec.PolicyTypes).To(ContainElement(networkingv1.PolicyTypeEgress))
				Expect(spec.PolicyTypes).To(ContainElement(networkingv1.PolicyTypeIngress))

				Expect(spec.Ingress).To(Equal([]networkingv1.NetworkPolicyIngressRule{{}}))
			})

			When("exposing the cluster with nodePort", func() {
				It("will have a NodePort service", func() {
					cluster := &v1beta1.Cluster{
						ObjectMeta: metav1.ObjectMeta{
							GenerateName: "cluster-",
							Namespace:    namespace,
						},
						Spec: v1beta1.ClusterSpec{
							Expose: &v1beta1.ExposeConfig{
								NodePort: &v1beta1.NodePortConfig{},
							},
						},
					}

					Expect(k8sClient.Create(ctx, cluster)).To(Succeed())

					var service corev1.Service

					Eventually(func() corev1.ServiceType {
						serviceKey := client.ObjectKey{
							Name:      server.ServiceName(cluster.Name),
							Namespace: cluster.Namespace,
						}

						err := k8sClient.Get(ctx, serviceKey, &service)
						Expect(client.IgnoreNotFound(err)).To(Not(HaveOccurred()))

						return service.Spec.Type
					}).
						WithTimeout(time.Second * 30).
						WithPolling(time.Second).
						Should(Equal(corev1.ServiceTypeNodePort))
				})

				It("will have the specified ports exposed when specified", func() {
					cluster := &v1beta1.Cluster{
						ObjectMeta: metav1.ObjectMeta{
							GenerateName: "cluster-",
							Namespace:    namespace,
						},
						Spec: v1beta1.ClusterSpec{
							Expose: &v1beta1.ExposeConfig{
								NodePort: &v1beta1.NodePortConfig{
									ServerPort: ptr.To[int32](30010),
									ETCDPort:   ptr.To[int32](30011),
								},
							},
						},
					}

					Expect(k8sClient.Create(ctx, cluster)).To(Succeed())

					var service corev1.Service

					Eventually(func() corev1.ServiceType {
						serviceKey := client.ObjectKey{
							Name:      server.ServiceName(cluster.Name),
							Namespace: cluster.Namespace,
						}

						err := k8sClient.Get(ctx, serviceKey, &service)
						Expect(client.IgnoreNotFound(err)).To(Not(HaveOccurred()))

						return service.Spec.Type
					}).
						WithTimeout(time.Second * 30).
						WithPolling(time.Second).
						Should(Equal(corev1.ServiceTypeNodePort))

					servicePorts := service.Spec.Ports
					Expect(servicePorts).NotTo(BeEmpty())
					Expect(servicePorts).To(HaveLen(2))

					serverPort := servicePorts[0]
					Expect(serverPort.Name).To(Equal("k3s-server-port"))
					Expect(serverPort.Port).To(BeEquivalentTo(443))
					Expect(serverPort.NodePort).To(BeEquivalentTo(30010))

					etcdPort := servicePorts[1]
					Expect(etcdPort.Name).To(Equal("k3s-etcd-port"))
					Expect(etcdPort.Port).To(BeEquivalentTo(2379))
					Expect(etcdPort.NodePort).To(BeEquivalentTo(30011))
				})

				It("will not expose the port when out of range", func() {
					cluster := &v1beta1.Cluster{
						ObjectMeta: metav1.ObjectMeta{
							GenerateName: "cluster-",
							Namespace:    namespace,
						},
						Spec: v1beta1.ClusterSpec{
							Expose: &v1beta1.ExposeConfig{
								NodePort: &v1beta1.NodePortConfig{
									ETCDPort: ptr.To[int32](2222),
								},
							},
						},
					}

					Expect(k8sClient.Create(ctx, cluster)).To(Succeed())

					var service corev1.Service

					Eventually(func() corev1.ServiceType {
						serviceKey := client.ObjectKey{
							Name:      server.ServiceName(cluster.Name),
							Namespace: cluster.Namespace,
						}

						err := k8sClient.Get(ctx, serviceKey, &service)
						Expect(client.IgnoreNotFound(err)).To(Not(HaveOccurred()))

						return service.Spec.Type
					}).
						WithTimeout(time.Second * 30).
						WithPolling(time.Second).
						Should(Equal(corev1.ServiceTypeNodePort))

					servicePorts := service.Spec.Ports
					Expect(servicePorts).NotTo(BeEmpty())
					Expect(servicePorts).To(HaveLen(1))

					serverPort := servicePorts[0]
					Expect(serverPort.Name).To(Equal("k3s-server-port"))
					Expect(serverPort.Port).To(BeEquivalentTo(443))
					Expect(serverPort.TargetPort.IntValue()).To(BeEquivalentTo(6443))
				})
			})

			When("exposing the cluster with loadbalancer", func() {
				It("will have a LoadBalancer service with the default ports exposed", func() {
					cluster := &v1beta1.Cluster{
						ObjectMeta: metav1.ObjectMeta{
							GenerateName: "cluster-",
							Namespace:    namespace,
						},
						Spec: v1beta1.ClusterSpec{
							Expose: &v1beta1.ExposeConfig{
								LoadBalancer: &v1beta1.LoadBalancerConfig{},
							},
						},
					}

					Expect(k8sClient.Create(ctx, cluster)).To(Succeed())

					var service corev1.Service

					Eventually(func() error {
						serviceKey := client.ObjectKey{
							Name:      server.ServiceName(cluster.Name),
							Namespace: cluster.Namespace,
						}

						return k8sClient.Get(ctx, serviceKey, &service)
					}).
						WithTimeout(time.Second * 30).
						WithPolling(time.Second).
						Should(Succeed())

					Expect(service.Spec.Type).To(Equal(corev1.ServiceTypeLoadBalancer))

					servicePorts := service.Spec.Ports
					Expect(servicePorts).NotTo(BeEmpty())
					Expect(servicePorts).To(HaveLen(2))

					serverPort := servicePorts[0]
					Expect(serverPort.Name).To(Equal("k3s-server-port"))
					Expect(serverPort.Port).To(BeEquivalentTo(443))
					Expect(serverPort.TargetPort.IntValue()).To(BeEquivalentTo(6443))

					etcdPort := servicePorts[1]
					Expect(etcdPort.Name).To(Equal("k3s-etcd-port"))
					Expect(etcdPort.Port).To(BeEquivalentTo(2379))
					Expect(etcdPort.TargetPort.IntValue()).To(BeEquivalentTo(2379))
				})
			})

			When("exposing the cluster with nodePort and loadbalancer", func() {
				It("will fail", func() {
					cluster := &v1beta1.Cluster{
						ObjectMeta: metav1.ObjectMeta{
							GenerateName: "cluster-",
							Namespace:    namespace,
						},
						Spec: v1beta1.ClusterSpec{
							Expose: &v1beta1.ExposeConfig{
								LoadBalancer: &v1beta1.LoadBalancerConfig{},
								NodePort:     &v1beta1.NodePortConfig{},
							},
						},
					}

					err := k8sClient.Create(ctx, cluster)
					Expect(err).To(HaveOccurred())
				})
			})
			When("adding addons to the cluster", func() {
				It("will create a statefulset with the correct addon volumes and volume mounts", func() {
					// Create the addon secret first
					addonSecret := &corev1.Secret{
						ObjectMeta: metav1.ObjectMeta{
							Name:      "test-addon",
							Namespace: namespace,
						},
						Data: map[string][]byte{
							"manifest.yaml": []byte("apiVersion: v1\nkind: ConfigMap\nmetadata:\n  name: test-cm\n"),
						},
					}
					Expect(k8sClient.Create(ctx, addonSecret)).To(Succeed())

					// Create the cluster with an addon referencing the secret
					cluster := &v1beta1.Cluster{
						ObjectMeta: metav1.ObjectMeta{
							GenerateName: "cluster-",
							Namespace:    namespace,
						},
						Spec: v1beta1.ClusterSpec{
							Addons: []v1beta1.Addon{
								{
									SecretRef: "test-addon",
								},
							},
						},
					}
					Expect(k8sClient.Create(ctx, cluster)).To(Succeed())

					// Wait for the statefulset to be created and verify volumes/mounts
					var statefulSet appsv1.StatefulSet

					statefulSetName := k3kcontroller.SafeConcatNameWithPrefix(cluster.Name, "server")

					Eventually(func() error {
						return k8sClient.Get(ctx, client.ObjectKey{
							Name:      statefulSetName,
							Namespace: cluster.Namespace,
						}, &statefulSet)
					}).
						WithTimeout(time.Second * 30).
						WithPolling(time.Second).
						Should(Succeed())

					// Verify the addon volume exists
					var addonVolume *corev1.Volume

					for i := range statefulSet.Spec.Template.Spec.Volumes {
						v := &statefulSet.Spec.Template.Spec.Volumes[i]
						if v.Name == "addon-test-addon" {
							addonVolume = v
							break
						}
					}

					Expect(addonVolume).NotTo(BeNil(), "addon volume should exist")
					Expect(addonVolume.VolumeSource.Secret).NotTo(BeNil())
					Expect(addonVolume.VolumeSource.Secret.SecretName).To(Equal("test-addon"))

					// Verify the addon volume mount exists in the first container
					containers := statefulSet.Spec.Template.Spec.Containers
					Expect(containers).NotTo(BeEmpty())

					var addonMount *corev1.VolumeMount

					for i := range containers[0].VolumeMounts {
						m := &containers[0].VolumeMounts[i]
						if m.Name == "addon-test-addon" {
							addonMount = m
							break
						}
					}

					Expect(addonMount).NotTo(BeNil(), "addon volume mount should exist")
					Expect(addonMount.MountPath).To(Equal("/var/lib/rancher/k3s/server/manifests/test-addon"))
					Expect(addonMount.ReadOnly).To(BeTrue())
				})

				It("will create volumes for multiple addons in the correct order", func() {
					// Create multiple addon secrets
					addonSecret1 := &corev1.Secret{
						ObjectMeta: metav1.ObjectMeta{
							Name:      "addon-one",
							Namespace: namespace,
						},
						Data: map[string][]byte{
							"manifest.yaml": []byte("apiVersion: v1\nkind: ConfigMap\nmetadata:\n  name: cm-one\n"),
						},
					}
					addonSecret2 := &corev1.Secret{
						ObjectMeta: metav1.ObjectMeta{
							Name:      "addon-two",
							Namespace: namespace,
						},
						Data: map[string][]byte{
							"manifest.yaml": []byte("apiVersion: v1\nkind: ConfigMap\nmetadata:\n  name: cm-two\n"),
						},
					}

					Expect(k8sClient.Create(ctx, addonSecret1)).To(Succeed())
					Expect(k8sClient.Create(ctx, addonSecret2)).To(Succeed())

					// Create the cluster with multiple addons in specific order
					cluster := &v1beta1.Cluster{
						ObjectMeta: metav1.ObjectMeta{
							GenerateName: "cluster-",
							Namespace:    namespace,
						},
						Spec: v1beta1.ClusterSpec{
							Addons: []v1beta1.Addon{
								{SecretRef: "addon-one"},
								{SecretRef: "addon-two"},
							},
						},
					}
					Expect(k8sClient.Create(ctx, cluster)).To(Succeed())

					// Wait for the statefulset to be created
					var statefulSet appsv1.StatefulSet

					statefulSetName := k3kcontroller.SafeConcatNameWithPrefix(cluster.Name, "server")

					Eventually(func() error {
						return k8sClient.Get(ctx, client.ObjectKey{
							Name:      statefulSetName,
							Namespace: cluster.Namespace,
						}, &statefulSet)
					}).
						WithTimeout(time.Second * 30).
						WithPolling(time.Second).
						Should(Succeed())

					// Verify both addon volumes exist and are in the correct order
					volumes := statefulSet.Spec.Template.Spec.Volumes

					// Extract only addon volumes (those starting with "addon-")
					var addonVolumes []corev1.Volume

					for _, v := range volumes {
						if strings.HasPrefix(v.Name, "addon-") {
							addonVolumes = append(addonVolumes, v)
						}
					}

					Expect(addonVolumes).To(HaveLen(2))
					Expect(addonVolumes[0].Name).To(Equal("addon-addon-one"))
					Expect(addonVolumes[1].Name).To(Equal("addon-addon-two"))

					// Verify both addon volume mounts exist and are in the correct order
					containers := statefulSet.Spec.Template.Spec.Containers
					Expect(containers).NotTo(BeEmpty())

					// Extract only addon mounts (those starting with "addon-")
					var addonMounts []corev1.VolumeMount

					for _, m := range containers[0].VolumeMounts {
						if strings.HasPrefix(m.Name, "addon-") {
							addonMounts = append(addonMounts, m)
						}
					}

					Expect(addonMounts).To(HaveLen(2))
					Expect(addonMounts[0].Name).To(Equal("addon-addon-one"))
					Expect(addonMounts[0].MountPath).To(Equal("/var/lib/rancher/k3s/server/manifests/addon-one"))
					Expect(addonMounts[1].Name).To(Equal("addon-addon-two"))
					Expect(addonMounts[1].MountPath).To(Equal("/var/lib/rancher/k3s/server/manifests/addon-two"))
				})
			})
		})
	})
})