diff --git a/charts/k3k/crds/k3k.io_clusters.yaml b/charts/k3k/crds/k3k.io_clusters.yaml index 308bafd6..d0b3b39f 100644 --- a/charts/k3k/crds/k3k.io_clusters.yaml +++ b/charts/k3k/crds/k3k.io_clusters.yaml @@ -36,6 +36,7 @@ spec: metadata: type: object spec: + default: {} properties: addons: description: Addons is a list of secrets containing raw YAML which @@ -55,6 +56,7 @@ spec: type: string type: array agents: + default: 0 description: Agents is the number of K3s pods to run in agent (worker) mode. format: int32 @@ -179,6 +181,7 @@ spec: type: string type: array servers: + default: 1 description: Servers is the number of K3s pods to run in server (controlplane) mode. format: int32 @@ -218,11 +221,6 @@ spec: description: Version is a string representing the Kubernetes version to be used by the virtual nodes. type: string - required: - - agents - - mode - - servers - - version type: object status: properties: @@ -230,6 +228,8 @@ spec: type: string clusterDNS: type: string + hostVersion: + type: string persistence: properties: storageClassName: @@ -250,8 +250,6 @@ spec: type: string type: array type: object - required: - - spec type: object served: true storage: true diff --git a/pkg/apis/k3k.io/v1alpha1/types.go b/pkg/apis/k3k.io/v1alpha1/types.go index ef3df10d..738b0b83 100644 --- a/pkg/apis/k3k.io/v1alpha1/types.go +++ b/pkg/apis/k3k.io/v1alpha1/types.go @@ -14,24 +14,35 @@ type Cluster struct { metav1.ObjectMeta `json:"metadata,omitempty"` metav1.TypeMeta `json:",inline"` + // +kubebuilder:default={} + // +optional Spec ClusterSpec `json:"spec"` Status ClusterStatus `json:"status,omitempty"` } type ClusterSpec struct { // Version is a string representing the Kubernetes version to be used by the virtual nodes. + // + // +optional Version string `json:"version"` // Servers is the number of K3s pods to run in server (controlplane) mode. + // + // +kubebuilder:default=1 // +kubebuilder:validation:XValidation:message="cluster must have at least one server",rule="self >= 1" + // +optional Servers *int32 `json:"servers"` // Agents is the number of K3s pods to run in agent (worker) mode. + // + // +kubebuilder:default=0 // +kubebuilder:validation:XValidation:message="invalid value for agents",rule="self >= 0" + // +optional Agents *int32 `json:"agents"` // NodeSelector is the node selector that will be applied to all server/agent pods. // In "shared" mode the node selector will be applied also to the workloads. + // // +optional NodeSelector map[string]string `json:"nodeSelector,omitempty"` @@ -73,10 +84,12 @@ type ClusterSpec struct { Addons []Addon `json:"addons,omitempty"` // Mode is the cluster provisioning mode which can be either "shared" or "virtual". Defaults to "shared" + // // +kubebuilder:default="shared" // +kubebuilder:validation:Enum=shared;virtual // +kubebuilder:validation:XValidation:message="mode is immutable",rule="self == oldSelf" - Mode ClusterMode `json:"mode"` + // +optional + Mode ClusterMode `json:"mode,omitempty"` // Persistence contains options controlling how the etcd data of the virtual cluster is persisted. By default, no data // persistence is guaranteed, so restart of a virtual cluster pod may result in data loss without this field. @@ -151,6 +164,7 @@ type NodePortConfig struct { } type ClusterStatus struct { + HostVersion string `json:"hostVersion,omitempty"` ClusterCIDR string `json:"clusterCIDR,omitempty"` ServiceCIDR string `json:"serviceCIDR,omitempty"` ClusterDNS string `json:"clusterDNS,omitempty"` diff --git a/pkg/controller/cluster/cluster.go b/pkg/controller/cluster/cluster.go index f7936da2..08adab1e 100644 --- a/pkg/controller/cluster/cluster.go +++ b/pkg/controller/cluster/cluster.go @@ -20,6 +20,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/discovery" ctrl "sigs.k8s.io/controller-runtime" ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" ctrlruntimecontroller "sigs.k8s.io/controller-runtime/pkg/controller" @@ -44,6 +45,7 @@ const ( ) type ClusterReconciler struct { + DiscoveryClient *discovery.DiscoveryClient Client ctrlruntimeclient.Client Scheme *runtime.Scheme SharedAgentImage string @@ -52,13 +54,21 @@ type ClusterReconciler struct { // Add adds a new controller to the manager func Add(ctx context.Context, mgr manager.Manager, sharedAgentImage string, logger *log.Logger) error { + + discoveryClient, err := discovery.NewDiscoveryClientForConfig(mgr.GetConfig()) + if err != nil { + return err + } + // initialize a new Reconciler reconciler := ClusterReconciler{ + DiscoveryClient: discoveryClient, Client: mgr.GetClient(), Scheme: mgr.GetScheme(), SharedAgentImage: sharedAgentImage, logger: logger.Named(clusterController), } + return ctrl.NewControllerManagedBy(mgr). For(&v1alpha1.Cluster{}). WithOptions(ctrlruntimecontroller.Options{ @@ -76,6 +86,22 @@ func (c *ClusterReconciler) Reconcile(ctx context.Context, req reconcile.Request if err := c.Client.Get(ctx, req.NamespacedName, &cluster); err != nil { return reconcile.Result{}, ctrlruntimeclient.IgnoreNotFound(err) } + + // if the Version is not specified we will try to use the same Kubernetes version of the host. + // This version is stored in the Status object, and it will not be updated if already set. + if cluster.Spec.Version == "" && cluster.Status.HostVersion == "" { + hostVersion, err := c.DiscoveryClient.ServerVersion() + if err != nil { + return reconcile.Result{}, err + } + + // update Status HostVersion + cluster.Status.HostVersion = fmt.Sprintf("v%s.%s.0-k3s1", hostVersion.Major, hostVersion.Minor) + if err := c.Client.Status().Update(ctx, &cluster); err != nil { + return reconcile.Result{}, err + } + } + if cluster.DeletionTimestamp.IsZero() { if !controllerutil.ContainsFinalizer(&cluster, clusterFinalizerName) { controllerutil.AddFinalizer(&cluster, clusterFinalizerName) diff --git a/pkg/controller/cluster/cluster_suite_test.go b/pkg/controller/cluster/cluster_suite_test.go new file mode 100644 index 00000000..94ff9bd1 --- /dev/null +++ b/pkg/controller/cluster/cluster_suite_test.go @@ -0,0 +1,91 @@ +package cluster_test + +import ( + "context" + "path/filepath" + "testing" + + "github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1" + "github.com/rancher/k3k/pkg/controller/cluster" + "github.com/rancher/k3k/pkg/log" + + "go.uber.org/zap" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestController(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Cluster Controller Suite") +} + +var ( + testEnv *envtest.Environment + k8s *kubernetes.Clientset + k8sClient client.Client + ctx context.Context + cancel context.CancelFunc +) + +var _ = BeforeSuite(func() { + + By("bootstrapping test environment") + testEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "charts", "k3k", "crds")}, + ErrorIfCRDPathMissing: true, + } + cfg, err := testEnv.Start() + Expect(err).NotTo(HaveOccurred()) + + k8s, err = kubernetes.NewForConfig(cfg) + Expect(err).NotTo(HaveOccurred()) + + scheme := buildScheme() + k8sClient, err = client.New(cfg, client.Options{Scheme: scheme}) + Expect(err).NotTo(HaveOccurred()) + + mgr, err := ctrl.NewManager(cfg, ctrl.Options{Scheme: scheme}) + Expect(err).NotTo(HaveOccurred()) + + ctx, cancel = context.WithCancel(context.Background()) + err = cluster.Add(ctx, mgr, "", &log.Logger{SugaredLogger: zap.NewNop().Sugar()}) + Expect(err).NotTo(HaveOccurred()) + + go func() { + defer GinkgoRecover() + err = mgr.Start(ctx) + Expect(err).NotTo(HaveOccurred(), "failed to run manager") + }() +}) + +var _ = AfterSuite(func() { + cancel() + + By("tearing down the test environment") + err := testEnv.Stop() + Expect(err).NotTo(HaveOccurred()) +}) + +func buildScheme() *runtime.Scheme { + scheme := runtime.NewScheme() + + err := corev1.AddToScheme(scheme) + Expect(err).NotTo(HaveOccurred()) + err = appsv1.AddToScheme(scheme) + Expect(err).NotTo(HaveOccurred()) + err = networkingv1.AddToScheme(scheme) + Expect(err).NotTo(HaveOccurred()) + err = v1alpha1.AddToScheme(scheme) + Expect(err).NotTo(HaveOccurred()) + + return scheme +} diff --git a/pkg/controller/cluster/cluster_test.go b/pkg/controller/cluster/cluster_test.go new file mode 100644 index 00000000..65bd70cf --- /dev/null +++ b/pkg/controller/cluster/cluster_test.go @@ -0,0 +1,68 @@ +package cluster_test + +import ( + "context" + "fmt" + "time" + + "github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1" + "sigs.k8s.io/controller-runtime/pkg/client" + + corev1 "k8s.io/api/core/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Cluster Controller", func() { + + Context("creating a Cluster", func() { + + var ( + namespace string + ) + + BeforeEach(func() { + createdNS := &corev1.Namespace{ObjectMeta: v1.ObjectMeta{GenerateName: "ns-"}} + err := k8sClient.Create(context.Background(), createdNS) + Expect(err).To(Not(HaveOccurred())) + namespace = createdNS.Name + }) + + When("created with a default spec", func() { + + It("should have been created with some defaults", func() { + cluster := &v1alpha1.Cluster{ + ObjectMeta: v1.ObjectMeta{ + GenerateName: "clusterset-", + Namespace: namespace, + }, + } + + err := k8sClient.Create(ctx, cluster) + Expect(err).To(Not(HaveOccurred())) + + Expect(cluster.Spec.Mode).To(Equal(v1alpha1.SharedClusterMode)) + Expect(cluster.Spec.Agents).To(Equal(ptr.To[int32](0))) + Expect(cluster.Spec.Servers).To(Equal(ptr.To[int32](1))) + Expect(cluster.Spec.Version).To(BeEmpty()) + + serverVersion, err := k8s.DiscoveryClient.ServerVersion() + Expect(err).To(Not(HaveOccurred())) + expectedHostVersion := fmt.Sprintf("v%s.%s.0-k3s1", serverVersion.Major, serverVersion.Minor) + + Eventually(func() string { + err := k8sClient.Get(ctx, client.ObjectKeyFromObject(cluster), cluster) + Expect(err).To(Not(HaveOccurred())) + return cluster.Status.HostVersion + + }). + WithTimeout(time.Second * 30). + WithPolling(time.Second). + Should(Equal(expectedHostVersion)) + }) + }) + }) +}) diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 2d179b66..b6f37471 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -27,9 +27,21 @@ var Backoff = wait.Backoff{ Jitter: 0.1, } +// K3SImage returns the rancher/k3s image tagged with the specified Version. +// If Version is empty it will use with the same k8s version of the host cluster, +// stored in the Status object. It will return the untagged version as last fallback. func K3SImage(cluster *v1alpha1.Cluster) string { - return k3SImageName + ":" + cluster.Spec.Version + if cluster.Spec.Version != "" { + return k3SImageName + ":" + cluster.Spec.Version + } + + if cluster.Status.HostVersion != "" { + return k3SImageName + ":" + cluster.Status.HostVersion + } + + return k3SImageName } + func nodeAddress(node *v1.Node) string { var externalIP string var internalIP string