add back cluster and service cidr

Signed-off-by: Brian Downs <brian.downs@gmail.com>

charts/k3k/crds/cluster.yaml

@@ -53,6 +53,15 @@ spec:
                       type: string
                     storageRequestSize:
                       type: string
+                addons:
+                  type: array
+                  items:
+                    type: object
+                    properties:
+                      secretNamespace:
+                        type: string
+                      secretRef:
+                        type: string
                 expose:
                   type: object
                   properties:

charts/k3k/values.yaml

@@ -2,10 +2,10 @@ replicaCount: 1
 namespace: k3k-system
 
 image:
-  repository: rancher/k3k
+  repository: briandowns/k3k
   pullPolicy: Always
   # Overrides the image tag whose default is the chart appVersion.
-  tag: "v0.0.0-alpha6"
+  tag: "dev"
 
 imagePullSecrets: []
 nameOverride: ""

pkg/apis/k3k.io/v1alpha1/types.go

@@ -27,11 +27,17 @@ type ClusterSpec struct {
 	ServerArgs  []string `json:"serverArgs,omitempty"`
 	AgentArgs   []string `json:"agentArgs,omitempty"`
 	TLSSANs     []string `json:"tlsSANs,omitempty"`
+	Addons      []Addon  `json:"addons,omitempty"`
 
 	Persistence *PersistenceConfig `json:"persistence,omitempty"`
 	Expose      *ExposeConfig      `json:"expose,omitempty"`
 }
 
+type Addon struct {
+	SecretNamespace string `json:"secretNamespace,omitempty"`
+	SecretRef       string `json:"secretRef,omitempty"`
+}
+
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
 
 type ClusterList struct {
@@ -72,36 +78,3 @@ type ClusterStatus struct {
 	ServiceCIDR string `json:"serviceCIDR,omitempty"`
 	ClusterDNS  string `json:"clusterDNS,omitempty"`
 }
-
-type Allocation struct {
-	ClusterName string `json:"clusterName"`
-	Issued      int64  `json:"issued"`
-	IPNet       string `json:"ipNet"`
-}
-
-// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
-
-type CIDRAllocationPool struct {
-	metav1.ObjectMeta `json:"metadata,omitempty"`
-	metav1.TypeMeta   `json:",inline"`
-
-	Spec   CIDRAllocationPoolSpec   `json:"spec"`
-	Status CIDRAllocationPoolStatus `json:"status"`
-}
-
-type CIDRAllocationPoolSpec struct {
-	DefaultClusterCIDR string `json:"defaultClusterCIDR"`
-}
-
-type CIDRAllocationPoolStatus struct {
-	Pool []Allocation `json:"pool"`
-}
-
-// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
-
-type CIDRAllocationPoolList struct {
-	metav1.ListMeta `json:"metadata,omitempty"`
-	metav1.TypeMeta `json:",inline"`
-
-	Items []CIDRAllocationPool `json:"items"`
-}

pkg/apis/k3k.io/v1alpha1/zz_generated.deepcopy.go

@@ -10,115 +10,17 @@ import (
 )
 
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *Allocation) DeepCopyInto(out *Allocation) {
+func (in *Addon) DeepCopyInto(out *Addon) {
 	*out = *in
 	return
 }
 
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Allocation.
-func (in *Allocation) DeepCopy() *Allocation {
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Addon.
+func (in *Addon) DeepCopy() *Addon {
 	if in == nil {
 		return nil
 	}
-	out := new(Allocation)
-	in.DeepCopyInto(out)
-	return out
-}
-
-// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *CIDRAllocationPool) DeepCopyInto(out *CIDRAllocationPool) {
-	*out = *in
-	in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
-	out.TypeMeta = in.TypeMeta
-	out.Spec = in.Spec
-	in.Status.DeepCopyInto(&out.Status)
-	return
-}
-
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CIDRAllocationPool.
-func (in *CIDRAllocationPool) DeepCopy() *CIDRAllocationPool {
-	if in == nil {
-		return nil
-	}
-	out := new(CIDRAllocationPool)
-	in.DeepCopyInto(out)
-	return out
-}
-
-// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
-func (in *CIDRAllocationPool) DeepCopyObject() runtime.Object {
-	if c := in.DeepCopy(); c != nil {
-		return c
-	}
-	return nil
-}
-
-// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *CIDRAllocationPoolList) DeepCopyInto(out *CIDRAllocationPoolList) {
-	*out = *in
-	in.ListMeta.DeepCopyInto(&out.ListMeta)
-	out.TypeMeta = in.TypeMeta
-	if in.Items != nil {
-		in, out := &in.Items, &out.Items
-		*out = make([]CIDRAllocationPool, len(*in))
-		for i := range *in {
-			(*in)[i].DeepCopyInto(&(*out)[i])
-		}
-	}
-	return
-}
-
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CIDRAllocationPoolList.
-func (in *CIDRAllocationPoolList) DeepCopy() *CIDRAllocationPoolList {
-	if in == nil {
-		return nil
-	}
-	out := new(CIDRAllocationPoolList)
-	in.DeepCopyInto(out)
-	return out
-}
-
-// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
-func (in *CIDRAllocationPoolList) DeepCopyObject() runtime.Object {
-	if c := in.DeepCopy(); c != nil {
-		return c
-	}
-	return nil
-}
-
-// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *CIDRAllocationPoolSpec) DeepCopyInto(out *CIDRAllocationPoolSpec) {
-	*out = *in
-	return
-}
-
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CIDRAllocationPoolSpec.
-func (in *CIDRAllocationPoolSpec) DeepCopy() *CIDRAllocationPoolSpec {
-	if in == nil {
-		return nil
-	}
-	out := new(CIDRAllocationPoolSpec)
-	in.DeepCopyInto(out)
-	return out
-}
-
-// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *CIDRAllocationPoolStatus) DeepCopyInto(out *CIDRAllocationPoolStatus) {
-	*out = *in
-	if in.Pool != nil {
-		in, out := &in.Pool, &out.Pool
-		*out = make([]Allocation, len(*in))
-		copy(*out, *in)
-	}
-	return
-}
-
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CIDRAllocationPoolStatus.
-func (in *CIDRAllocationPoolStatus) DeepCopy() *CIDRAllocationPoolStatus {
-	if in == nil {
-		return nil
-	}
-	out := new(CIDRAllocationPoolStatus)
+	out := new(Addon)
 	in.DeepCopyInto(out)
 	return out
 }
@@ -212,6 +114,11 @@ func (in *ClusterSpec) DeepCopyInto(out *ClusterSpec) {
 		*out = make([]string, len(*in))
 		copy(*out, *in)
 	}
+	if in.Addons != nil {
+		in, out := &in.Addons, &out.Addons
+		*out = make([]Addon, len(*in))
+		copy(*out, *in)
+	}
 	if in.Persistence != nil {
 		in, out := &in.Persistence, &out.Persistence
 		*out = new(PersistenceConfig)

pkg/controller/cluster/agent/agent.go

@@ -12,39 +12,52 @@ import (
 
 const agentName = "k3k-agent"
 
-func Agent(cluster *v1alpha1.Cluster) *apps.Deployment {
-	image := util.K3SImage(cluster)
+type Agent struct {
+	cluster *v1alpha1.Cluster
+}
+
+func New(cluster *v1alpha1.Cluster) *Agent {
+	return &Agent{
+		cluster: cluster,
+	}
+}
+
+func (a *Agent) Deploy() *apps.Deployment {
+	image := util.K3SImage(a.cluster)
+
+	const name = "k3k-agent"
+
 	return &apps.Deployment{
 		TypeMeta: metav1.TypeMeta{
 			Kind:       "Deployment",
 			APIVersion: "apps/v1",
 		},
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      cluster.Name + "-" + agentName,
-			Namespace: util.ClusterNamespace(cluster),
+			Name:      a.cluster.Name + "-" + name,
+			Namespace: util.ClusterNamespace(a.cluster),
 		},
 		Spec: apps.DeploymentSpec{
-			Replicas: cluster.Spec.Agents,
+			Replicas: a.cluster.Spec.Agents,
 			Selector: &metav1.LabelSelector{
 				MatchLabels: map[string]string{
-					"cluster": cluster.Name,
+					"cluster": a.cluster.Name,
 					"type":    "agent",
 				},
 			},
 			Template: v1.PodTemplateSpec{
 				ObjectMeta: metav1.ObjectMeta{
 					Labels: map[string]string{
-						"cluster": cluster.Name,
+						"cluster": a.cluster.Name,
 						"type":    "agent",
 					},
 				},
-				Spec: agentPodSpec(image, agentName, cluster.Spec.AgentArgs, false),
+				Spec: a.podSpec(image, name, a.cluster.Spec.AgentArgs, false),
 			},
 		},
 	}
 }
 
-func StatefulAgent(cluster *v1alpha1.Cluster) *apps.StatefulSet {
+func (a *Agent) StatefulAgent(cluster *v1alpha1.Cluster) *apps.StatefulSet {
 	image := util.K3SImage(cluster)
 
 	return &apps.StatefulSet{
@@ -112,13 +125,13 @@ func StatefulAgent(cluster *v1alpha1.Cluster) *apps.StatefulSet {
 						"type":    "agent",
 					},
 				},
-				Spec: agentPodSpec(image, agentName, cluster.Spec.AgentArgs, true),
+				Spec: a.podSpec(image, agentName, cluster.Spec.AgentArgs, true),
 			},
 		},
 	}
 }
 
-func agentPodSpec(image, name string, args []string, statefulSet bool) v1.PodSpec {
+func (a *Agent) podSpec(image, name string, args []string, statefulSet bool) v1.PodSpec {
 	args = append([]string{"agent", "--config", "/opt/rancher/k3s/config.yaml"}, args...)
 	podSpec := v1.PodSpec{
 		Volumes: []v1.Volume{
@@ -212,9 +225,9 @@ func agentPodSpec(image, name string, args []string, statefulSet bool) v1.PodSpe
 			},
 		},
 	}
+
 	if !statefulSet {
 		podSpec.Volumes = append(podSpec.Volumes, v1.Volume{
-
 			Name: "varlibkubelet",
 			VolumeSource: v1.VolumeSource{
 				EmptyDir: &v1.EmptyDirVolumeSource{},
@@ -228,5 +241,6 @@ func agentPodSpec(image, name string, args []string, statefulSet bool) v1.PodSpe
 		},
 		)
 	}
+
 	return podSpec
 }

pkg/controller/cluster/agent/service.go

@@ -7,7 +7,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
-func StatefulAgentService(cluster *v1alpha1.Cluster) *v1.Service {
+func (a *Agent) StatefulAgentService(cluster *v1alpha1.Cluster) *v1.Service {
 	return &v1.Service{
 		TypeMeta: metav1.TypeMeta{
 			Kind:       "Service",

pkg/controller/cluster/cluster.go

@@ -2,6 +2,7 @@ package cluster
 
 import (
 	"context"
+	"fmt"
 
 	"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
 	"github.com/rancher/k3k/pkg/controller/cluster/agent"
@@ -27,6 +28,11 @@ const (
 	clusterFinalizerName = "cluster.k3k.io/finalizer"
 	EphermalNodesType    = "ephermal"
 	DynamicNodesType     = "dynamic"
+
+	maxConcurrentReconciles = 1
+
+	defaultClusterCIDR        = "10.44.0.0/16"
+	defaultClusterServiceCIDR = "10.45.0.0/16"
 )
 
 type ClusterReconciler struct {
@@ -46,7 +52,7 @@ func Add(ctx context.Context, mgr manager.Manager) error {
 	//this can be replaced by the new builder functionality in controller-runtime
 	controller, err := controller.New(clusterController, mgr, controller.Options{
 		Reconciler:              &reconciler,
-		MaxConcurrentReconciles: 1,
+		MaxConcurrentReconciles: maxConcurrentReconciles,
 	})
 	if err != nil {
 		return err
@@ -77,7 +83,7 @@ func (c *ClusterReconciler) Reconcile(ctx context.Context, req reconcile.Request
 		}
 		if err := c.Client.Get(ctx, objKey, &ns); err != nil {
 			if !apierrors.IsNotFound(err) {
-				return reconcile.Result{}, util.WrapErr("failed to get cluster namespace "+util.ClusterNamespace(&cluster), err)
+				return reconcile.Result{}, util.LogAndReturnErr("failed to get cluster namespace "+util.ClusterNamespace(&cluster), err)
 			}
 		}
 
@@ -99,6 +105,9 @@ func (c *ClusterReconciler) Reconcile(ctx context.Context, req reconcile.Request
 }
 
 func (c *ClusterReconciler) createCluster(ctx context.Context, cluster *v1alpha1.Cluster) error {
+	server := server.New(cluster, c.Client)
+	agent := agent.New(cluster)
+
 	if cluster.Spec.Persistence == nil {
 		// default to ephermal nodes
 		cluster.Spec.Persistence = &v1alpha1.PersistenceConfig{
@@ -106,21 +115,31 @@ func (c *ClusterReconciler) createCluster(ctx context.Context, cluster *v1alpha1
 		}
 	}
 	if err := c.Client.Update(ctx, cluster); err != nil {
-		return util.WrapErr("failed to update cluster with persistence type", err)
+		return util.LogAndReturnErr("failed to update cluster with persistence type", err)
 	}
 	// create a new namespace for the cluster
 	if err := c.createNamespace(ctx, cluster); err != nil {
-		return util.WrapErr("failed to create ns", err)
+		return util.LogAndReturnErr("failed to create ns", err)
+	}
+
+	cluster.Status.ClusterCIDR = cluster.Spec.ClusterCIDR
+	if cluster.Status.ClusterCIDR == "" {
+		cluster.Status.ClusterCIDR = defaultClusterCIDR
+	}
+
+	cluster.Status.ServiceCIDR = cluster.Spec.ServiceCIDR
+	if cluster.Status.ServiceCIDR == "" {
+		cluster.Status.ServiceCIDR = defaultClusterServiceCIDR
 	}
 
 	klog.Infof("creating cluster service")
-	serviceIP, err := c.createClusterService(ctx, cluster)
+	serviceIP, err := c.createClusterService(ctx, cluster, server)
 	if err != nil {
-		return util.WrapErr("failed to create cluster service", err)
+		return util.LogAndReturnErr("failed to create cluster service", err)
 	}
 
 	if err := c.createClusterConfigs(ctx, cluster, serviceIP); err != nil {
-		return util.WrapErr("failed to create cluster configs", err)
+		return util.LogAndReturnErr("failed to create cluster configs", err)
 	}
 
 	// creating statefulsets in case the user chose a persistence type other than ephermal
@@ -129,38 +148,38 @@ func (c *ClusterReconciler) createCluster(ctx context.Context, cluster *v1alpha1
 			// default to 1G of request size
 			cluster.Spec.Persistence.StorageRequestSize = "1G"
 		}
-		if err := c.createStatefulSets(ctx, cluster); err != nil {
-			return util.WrapErr("failed to create servers and agents statefulsets", err)
+		if err := c.createStatefulSets(ctx, cluster, server, agent); err != nil {
+			return util.LogAndReturnErr("failed to create servers and agents statefulsets", err)
 		}
 	} else {
-		if err := c.createDeployments(ctx, cluster); err != nil {
-			return util.WrapErr("failed to create servers and agents deployment", err)
+		if err := c.createDeployments(ctx, cluster, server); err != nil {
+			return util.LogAndReturnErr("failed to create servers and agents deployment", err)
 		}
 	}
 
 	if cluster.Spec.Expose != nil {
 		if cluster.Spec.Expose.Ingress != nil {
-			serverIngress, err := server.Ingress(ctx, cluster, c.Client)
+			serverIngress, err := server.Ingress(ctx, c.Client)
 			if err != nil {
-				return util.WrapErr("failed to create ingress object", err)
+				return util.LogAndReturnErr("failed to create ingress object", err)
 			}
 
 			if err := c.Client.Create(ctx, serverIngress); err != nil {
 				if !apierrors.IsAlreadyExists(err) {
-					return util.WrapErr("failed to create server ingress", err)
+					return util.LogAndReturnErr("failed to create server ingress", err)
 				}
 			}
 		}
 	}
 
-	kubeconfigSecret, err := server.GenerateNewKubeConfig(ctx, cluster, serviceIP)
+	kubeconfigSecret, err := server.GenerateNewKubeConfig(ctx, serviceIP)
 	if err != nil {
-		return util.WrapErr("failed to generate new kubeconfig", err)
+		return util.LogAndReturnErr("failed to generate new kubeconfig", err)
 	}
 
 	if err := c.Client.Create(ctx, kubeconfigSecret); err != nil {
 		if !apierrors.IsAlreadyExists(err) {
-			return util.WrapErr("failed to create kubeconfig secret", err)
+			return util.LogAndReturnErr("failed to create kubeconfig secret", err)
 		}
 	}
 
@@ -180,7 +199,7 @@ func (c *ClusterReconciler) createNamespace(ctx context.Context, cluster *v1alph
 
 	if err := c.Client.Create(ctx, &namespace); err != nil {
 		if !apierrors.IsAlreadyExists(err) {
-			return util.WrapErr("failed to create ns", err)
+			return util.LogAndReturnErr("failed to create ns", err)
 		}
 	}
 
@@ -219,7 +238,7 @@ func (c *ClusterReconciler) createClusterConfigs(ctx context.Context, cluster *v
 	}
 
 	// create agents configuration
-	agentsConfig := config.Agent(cluster, serviceIP)
+	agentsConfig := agentConfig(cluster, serviceIP)
 	if err := controllerutil.SetControllerReference(cluster, &agentsConfig, c.Scheme); err != nil {
 		return err
 	}
@@ -232,7 +251,7 @@ func (c *ClusterReconciler) createClusterConfigs(ctx context.Context, cluster *v
 	return nil
 }
 
-func (c *ClusterReconciler) createClusterService(ctx context.Context, cluster *v1alpha1.Cluster) (string, error) {
+func (c *ClusterReconciler) createClusterService(ctx context.Context, cluster *v1alpha1.Cluster, server *server.Server) (string, error) {
 	// create cluster service
 	clusterService := server.Service(cluster)
 
@@ -258,10 +277,13 @@ func (c *ClusterReconciler) createClusterService(ctx context.Context, cluster *v
 	return service.Spec.ClusterIP, nil
 }
 
-func (c *ClusterReconciler) createDeployments(ctx context.Context, cluster *v1alpha1.Cluster) error {
+func (c *ClusterReconciler) createDeployments(ctx context.Context, cluster *v1alpha1.Cluster, server *server.Server) error {
 	// create deployment for the init server
 	// the init deployment must have only 1 replica
-	initServerDeployment := server.Server(cluster, true)
+	initServerDeployment, err := server.Deploy(ctx, true)
+	if err != nil {
+		return err
+	}
 
 	if err := controllerutil.SetControllerReference(cluster, initServerDeployment, c.Scheme); err != nil {
 		return err
@@ -274,7 +296,10 @@ func (c *ClusterReconciler) createDeployments(ctx context.Context, cluster *v1al
 	}
 
 	// create deployment for the rest of the servers
-	serversDeployment := server.Server(cluster, false)
+	serversDeployment, err := server.Deploy(ctx, false)
+	if err != nil {
+		return err
+	}
 
 	if err := controllerutil.SetControllerReference(cluster, serversDeployment, c.Scheme); err != nil {
 		return err
@@ -286,7 +311,9 @@ func (c *ClusterReconciler) createDeployments(ctx context.Context, cluster *v1al
 		}
 	}
 
-	agentsDeployment := agent.Agent(cluster)
+	agent := agent.New(cluster)
+
+	agentsDeployment := agent.Deploy()
 	if err := controllerutil.SetControllerReference(cluster, agentsDeployment, c.Scheme); err != nil {
 		return err
 	}
@@ -300,7 +327,7 @@ func (c *ClusterReconciler) createDeployments(ctx context.Context, cluster *v1al
 	return nil
 }
 
-func (c *ClusterReconciler) createStatefulSets(ctx context.Context, cluster *v1alpha1.Cluster) error {
+func (c *ClusterReconciler) createStatefulSets(ctx context.Context, cluster *v1alpha1.Cluster, server *server.Server, agent *agent.Agent) error {
 	// create headless service for the init statefulset
 	initServerStatefulService := server.StatefulServerService(cluster, true)
 	if err := controllerutil.SetControllerReference(cluster, initServerStatefulService, c.Scheme); err != nil {
@@ -314,7 +341,7 @@ func (c *ClusterReconciler) createStatefulSets(ctx context.Context, cluster *v1a
 
 	// create statefulsets for the init server
 	// the init statefulset must have only 1 replica
-	initServerStatefulSet := server.StatefulServer(cluster, true)
+	initServerStatefulSet := server.StatefulServer(ctx, cluster, true)
 
 	if err := controllerutil.SetControllerReference(cluster, initServerStatefulSet, c.Scheme); err != nil {
 		return err
@@ -337,7 +364,7 @@ func (c *ClusterReconciler) createStatefulSets(ctx context.Context, cluster *v1a
 			return err
 		}
 	}
-	serversStatefulSet := server.StatefulServer(cluster, false)
+	serversStatefulSet := server.StatefulServer(ctx, cluster, false)
 
 	if err := controllerutil.SetControllerReference(cluster, serversStatefulSet, c.Scheme); err != nil {
 		return err
@@ -373,22 +400,60 @@ func (c *ClusterReconciler) createStatefulSets(ctx context.Context, cluster *v1a
 	return nil
 }
 
-func (c *ClusterReconciler) createCIDRPools(ctx context.Context) error {
-	if err := c.Client.Create(ctx, &v1alpha1.CIDRAllocationPool{}); err != nil {
-		if !apierrors.IsAlreadyExists(err) {
-			// return nil since the resource has
-			// already been created
-			return err
-		}
-	}
-
-	if err := c.Client.Create(ctx, &v1alpha1.CIDRAllocationPool{}); err != nil {
-		if !apierrors.IsAlreadyExists(err) {
-			// return nil since the resource has
-			// already been created
-			return err
-		}
-	}
-
-	return nil
+func serverData(serviceIP string, cluster *v1alpha1.Cluster) string {
+	return "cluster-init: true\nserver: https://" + serviceIP + ":6443" + serverOptions(cluster)
+}
+
+func initConfigData(cluster *v1alpha1.Cluster) string {
+	return "cluster-init: true\n" + serverOptions(cluster)
+}
+
+func serverOptions(cluster *v1alpha1.Cluster) string {
+	var opts string
+
+	// TODO: generate token if not found
+	if cluster.Spec.Token != "" {
+		opts = "token: " + cluster.Spec.Token + "\n"
+	}
+	if cluster.Status.ClusterCIDR != "" {
+		opts = opts + "cluster-cidr: " + cluster.Status.ClusterCIDR + "\n"
+	}
+	if cluster.Status.ServiceCIDR != "" {
+		opts = opts + "service-cidr: " + cluster.Status.ServiceCIDR + "\n"
+	}
+	if cluster.Spec.ClusterDNS != "" {
+		opts = opts + "cluster-dns: " + cluster.Spec.ClusterDNS + "\n"
+	}
+	if len(cluster.Spec.TLSSANs) > 0 {
+		opts = opts + "tls-san:\n"
+		for _, addr := range cluster.Spec.TLSSANs {
+			opts = opts + "- " + addr + "\n"
+		}
+	}
+	// TODO: Add extra args to the options
+
+	return opts
+}
+
+func agentConfig(cluster *v1alpha1.Cluster, serviceIP string) v1.Secret {
+	config := agentData(serviceIP, cluster.Spec.Token)
+
+	return v1.Secret{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Secret",
+			APIVersion: "v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "k3k-agent-config",
+			Namespace: util.ClusterNamespace(cluster),
+		},
+		Data: map[string][]byte{
+			"config.yaml": []byte(config),
+		},
+	}
+}
+
+func agentData(serviceIP, token string) string {
+	return fmt.Sprintf(`server: https://%s:6443
+token: %s`, serviceIP, token)
 }

pkg/controller/cluster/server/ingress.go

@@ -3,7 +3,6 @@ package server
 import (
 	"context"
 
-	"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
 	"github.com/rancher/k3k/pkg/controller/util"
 	networkingv1 "k8s.io/api/networking/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -18,40 +17,40 @@ const (
 	nginxSSLRedirectAnnotation     = "nginx.ingress.kubernetes.io/ssl-redirect"
 )
 
-func Ingress(ctx context.Context, cluster *v1alpha1.Cluster, client client.Client) (*networkingv1.Ingress, error) {
+func (s *Server) Ingress(ctx context.Context, client client.Client) (*networkingv1.Ingress, error) {
 	addresses, err := util.Addresses(ctx, client)
 	if err != nil {
 		return nil, err
 	}
 
-	ingressRules := ingressRules(cluster, addresses)
+	ingressRules := s.ingressRules(addresses)
 	ingress := &networkingv1.Ingress{
 		TypeMeta: metav1.TypeMeta{
 			Kind:       "Ingress",
 			APIVersion: "networking.k8s.io/v1",
 		},
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      cluster.Name + "-server-ingress",
-			Namespace: util.ClusterNamespace(cluster),
+			Name:      s.cluster.Name + "-server-ingress",
+			Namespace: util.ClusterNamespace(s.cluster),
 		},
 		Spec: networkingv1.IngressSpec{
-			IngressClassName: &cluster.Spec.Expose.Ingress.IngressClassName,
+			IngressClassName: &s.cluster.Spec.Expose.Ingress.IngressClassName,
 			Rules:            ingressRules,
 		},
 	}
 
-	configureIngressOptions(ingress, cluster.Spec.Expose.Ingress.IngressClassName)
+	configureIngressOptions(ingress, s.cluster.Spec.Expose.Ingress.IngressClassName)
 
 	return ingress, nil
 }
 
-func ingressRules(cluster *v1alpha1.Cluster, addresses []string) []networkingv1.IngressRule {
+func (s *Server) ingressRules(addresses []string) []networkingv1.IngressRule {
 	var ingressRules []networkingv1.IngressRule
 	pathTypePrefix := networkingv1.PathTypePrefix
 
 	for _, address := range addresses {
 		rule := networkingv1.IngressRule{
-			Host: cluster.Name + "." + address + wildcardDNS,
+			Host: s.cluster.Name + "." + address + wildcardDNS,
 			IngressRuleValue: networkingv1.IngressRuleValue{
 				HTTP: &networkingv1.HTTPIngressRuleValue{
 					Paths: []networkingv1.HTTPIngressPath{
@@ -62,7 +61,7 @@ func ingressRules(cluster *v1alpha1.Cluster, addresses []string) []networkingv1.
 								Service: &networkingv1.IngressServiceBackend{
 									Name: "k3k-server-service",
 									Port: networkingv1.ServiceBackendPort{
-										Number: 6443,
+										Number: port,
 									},
 								},
 							},

pkg/controller/cluster/server/kubeconfig.go

@@ -12,7 +12,6 @@ import (
 	"time"
 
 	certutil "github.com/rancher/dynamiclistener/cert"
-	"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
 	"github.com/rancher/k3k/pkg/controller/util"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -44,8 +43,8 @@ type content struct {
 // 2- generate client admin cert/key
 // 3- use the ca cert from the bootstrap data & admin cert/key to write a new kubeconfig
 // 4- save the new kubeconfig as a secret
-func GenerateNewKubeConfig(ctx context.Context, cluster *v1alpha1.Cluster, ip string) (*v1.Secret, error) {
-	token := cluster.Spec.Token
+func (s *Server) GenerateNewKubeConfig(ctx context.Context, ip string) (*v1.Secret, error) {
+	token := s.cluster.Spec.Token
 
 	var bootstrap *controlRuntimeBootstrap
 	if err := retry.OnError(retry.DefaultBackoff, func(err error) bool {
@@ -83,8 +82,8 @@ func GenerateNewKubeConfig(ctx context.Context, cluster *v1alpha1.Cluster, ip st
 			APIVersion: "v1",
 		},
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      cluster.Name + "-kubeconfig",
-			Namespace: util.ClusterNamespace(cluster),
+			Name:      s.cluster.Name + "-kubeconfig",
+			Namespace: util.ClusterNamespace(s.cluster),
 		},
 		Data: map[string][]byte{
 			"kubeconfig.yaml": kubeconfigData,

pkg/controller/cluster/server/server.go

@@ -1,6 +1,8 @@
 package server
 
 import (
+	"context"
+	"fmt"
 	"strconv"
 
 	"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
@@ -9,42 +11,123 @@ import (
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/utils/pointer"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 const (
-	serverName     = "k3k-server"
-	initServerName = "k3k-init-server"
+	serverName         = "k3k-"
+	k3kSystemNamespace = serverName + "system"
+	initServerName     = serverName + "init-server"
 )
 
-func Server(cluster *v1alpha1.Cluster, init bool) *apps.Deployment {
-	var replicas int32
-	image := util.K3SImage(cluster)
+// Server
+type Server struct {
+	cluster *v1alpha1.Cluster
+	client  client.Client
+}
 
-	name := serverName
+func New(cluster *v1alpha1.Cluster, client client.Client) *Server {
+	return &Server{
+		cluster: cluster,
+		client:  client,
+	}
+}
+
+func (s *Server) Deploy(ctx context.Context, init bool) (*apps.Deployment, error) {
+	var replicas int32
+	image := util.K3SImage(s.cluster)
+
+	name := serverName + "server"
 	if init {
-		name = initServerName
+		name = serverName + "init-server"
 	}
 
-	replicas = *cluster.Spec.Servers - 1
+	replicas = *s.cluster.Spec.Servers - 1
 	if init {
 		replicas = 1
 	}
 
+	var volumes []v1.Volume
+	var volumeMounts []v1.VolumeMount
+
+	for _, addon := range s.cluster.Spec.Addons {
+		namespace := k3kSystemNamespace
+		if addon.SecretNamespace != "" {
+			namespace = addon.SecretNamespace
+		}
+
+		nn := types.NamespacedName{
+			Name:      addon.SecretRef,
+			Namespace: namespace,
+		}
+
+		var addons v1.Secret
+		if err := s.client.Get(ctx, nn, &addons); err != nil {
+			return nil, err
+		}
+
+		clusterAddons := v1.Secret{
+			TypeMeta: metav1.TypeMeta{
+				Kind:       "Secret",
+				APIVersion: "v1",
+			},
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      addons.Name,
+				Namespace: util.ClusterNamespace(s.cluster),
+			},
+			Data: make(map[string][]byte, len(addons.Data)),
+		}
+		for k, v := range addons.Data {
+			clusterAddons.Data[k] = v
+		}
+
+		if err := s.client.Create(ctx, &clusterAddons); err != nil {
+			return nil, err
+		}
+
+		name := "varlibrancherk3smanifests" + addon.SecretRef
+		volume := v1.Volume{
+			Name: name,
+			VolumeSource: v1.VolumeSource{
+				Secret: &v1.SecretVolumeSource{
+					SecretName: addon.SecretRef,
+				},
+			},
+		}
+		volumes = append(volumes, volume)
+
+		volumeMount := v1.VolumeMount{
+			Name:      name,
+			MountPath: "/var/lib/rancher/k3s/server/manifests/" + addon.SecretRef,
+			// changes to this part of the filesystem shouldn't be done manually. The secret should be updated instead.
+			ReadOnly: true,
+		}
+		volumeMounts = append(volumeMounts, volumeMount)
+	}
+
+	podSpec := s.podSpec(ctx, image, name, false)
+
+	podSpec.Volumes = append(podSpec.Volumes, volumes...)
+	podSpec.Containers[0].VolumeMounts = append(podSpec.Containers[0].VolumeMounts, volumeMounts...)
+
+	fmt.Printf("XXX - Pod Spec\n %#v\n", podSpec)
+
 	return &apps.Deployment{
 		TypeMeta: metav1.TypeMeta{
 			Kind:       "Deployment",
 			APIVersion: "apps/v1",
 		},
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      cluster.Name + "-" + name,
-			Namespace: util.ClusterNamespace(cluster),
+			Name:      s.cluster.Name + "-" + name,
+			Namespace: util.ClusterNamespace(s.cluster),
 		},
 		Spec: apps.DeploymentSpec{
 			Replicas: &replicas,
 			Selector: &metav1.LabelSelector{
 				MatchLabels: map[string]string{
-					"cluster": cluster.Name,
+					"cluster": s.cluster.Name,
 					"role":    "server",
 					"init":    strconv.FormatBool(init),
 				},
@@ -52,19 +135,20 @@ func Server(cluster *v1alpha1.Cluster, init bool) *apps.Deployment {
 			Template: v1.PodTemplateSpec{
 				ObjectMeta: metav1.ObjectMeta{
 					Labels: map[string]string{
-						"cluster": cluster.Name,
+						"cluster": s.cluster.Name,
 						"role":    "server",
 						"init":    strconv.FormatBool(init),
 					},
 				},
-				Spec: serverPodSpec(image, name, cluster.Spec.ServerArgs, false),
+				Spec: podSpec,
 			},
 		},
-	}
+	}, nil
 }
 
-func serverPodSpec(image, name string, args []string, statefulSet bool) v1.PodSpec {
-	args = append([]string{"server", "--config", "/opt/rancher/k3s/config.yaml"}, args...)
+func (s *Server) podSpec(ctx context.Context, image, name string, statefulSet bool) v1.PodSpec {
+	args := append([]string{"server", "--config", "/opt/rancher/k3s/config.yaml"}, s.cluster.Spec.ServerArgs...)
+
 	podSpec := v1.PodSpec{
 		Volumes: []v1.Volume{
 			{
@@ -157,6 +241,7 @@ func serverPodSpec(image, name string, args []string, statefulSet bool) v1.PodSp
 			},
 		},
 	}
+
 	if !statefulSet {
 		podSpec.Volumes = append(podSpec.Volumes, v1.Volume{
 
@@ -176,7 +261,7 @@ func serverPodSpec(image, name string, args []string, statefulSet bool) v1.PodSp
 	return podSpec
 }
 
-func StatefulServer(cluster *v1alpha1.Cluster, init bool) *apps.StatefulSet {
+func (s *Server) StatefulServer(ctx context.Context, cluster *v1alpha1.Cluster, init bool) *apps.StatefulSet {
 	var replicas int32
 	image := util.K3SImage(cluster)
 
@@ -257,7 +342,7 @@ func StatefulServer(cluster *v1alpha1.Cluster, init bool) *apps.StatefulSet {
 						"init":    strconv.FormatBool(init),
 					},
 				},
-				Spec: serverPodSpec(image, name, cluster.Spec.ServerArgs, true),
+				Spec: s.podSpec(ctx, image, name, true),
 			},
 		},
 	}

pkg/controller/cluster/server/service.go

@@ -9,7 +9,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
-func Service(cluster *v1alpha1.Cluster) *v1.Service {
+func (s *Server) Service(cluster *v1alpha1.Cluster) *v1.Service {
 	serviceType := v1.ServiceTypeClusterIP
 	if cluster.Spec.Expose != nil {
 		if cluster.Spec.Expose.NodePort != nil {
@@ -38,14 +38,14 @@ func Service(cluster *v1alpha1.Cluster) *v1.Service {
 				{
 					Name:     "k3s-server-port",
 					Protocol: v1.ProtocolTCP,
-					Port:     6443,
+					Port:     port,
 				},
 			},
 		},
 	}
 }
 
-func StatefulServerService(cluster *v1alpha1.Cluster, init bool) *v1.Service {
+func (s *Server) StatefulServerService(cluster *v1alpha1.Cluster, init bool) *v1.Service {
 	name := serverName
 	if init {
 		name = initServerName

pkg/controller/util/util.go

@@ -14,6 +14,10 @@ const (
 	k3SImageName    = "rancher/k3s"
 )
 
+const (
+	K3kSystemNamespace = namespacePrefix + "system"
+)
+
 func ClusterNamespace(cluster *v1alpha1.Cluster) string {
 	return namespacePrefix + cluster.Name
 }
@@ -22,7 +26,7 @@ func K3SImage(cluster *v1alpha1.Cluster) string {
 	return k3SImageName + ":" + cluster.Spec.Version
 }
 
-func WrapErr(errString string, err error) error {
+func LogAndReturnErr(errString string, err error) error {
 	klog.Errorf("%s: %v", errString, err)
 	return err
 }