Add serverArgs and agentArgs

Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>

crd/cluster.yaml

@@ -25,6 +25,20 @@ spec:
                   type: integer
                 token:
                   type: string
+                clusterCIDR:
+                  type: string
+                serviceCIDR:
+                  type: string
+                clusterDNS:
+                  type: string
+                serverArgs:
+                  type: array
+                  items:
+                    type: string
+                agentArgs:
+                  type: array
+                  items:
+                    type: string
                 expose:
                   type: object
                   properties:

examples/multiple-servers.yaml

@@ -1,13 +1,18 @@
 apiVersion: k3k.io/v1alpha1
 kind: Cluster
 metadata:
-  name: multiple-servers
+  name: example1
   namespace: default
 spec:
-  servers: 2
+  servers: 1
   agents: 3
   token: test
   version: v1.26.0-k3s2
+  clusterCIDR: 10.30.0.0/16
+  serviceCIDR: 10.31.0.0/16
+  clusterDNS: 10.30.0.10
+  serverArgs:
+  - "--write-kubeconfig-mode=777"
   expose:
     ingress:
       enabled: true

examples/single-server.yaml

@@ -0,0 +1,19 @@
+apiVersion: k3k.io/v1alpha1
+kind: Cluster
+metadata:
+  name: single-server
+  namespace: default
+spec:
+  servers: 1
+  agents: 3
+  token: test
+  version: v1.26.0-k3s2
+  clusterCIDR: 10.30.0.0/16
+  serviceCIDR: 10.31.0.0/16
+  clusterDNS: 10.30.0.10
+  serverArgs:
+  - "--write-kubeconfig-mode=777"
+  expose:
+    ingress:
+      enabled: true
+      ingressClassName: "nginx"

examples/single-server/single-server.yaml

@@ -1,11 +0,0 @@
-apiVersion: k3k.io/v1alpha1
-kind: Cluster
-metadata:
-  name: single-server
-  namespace: default
-spec:
-  servers: 1
-  agents: 3
-  token: test
-  version: v1.26.0-k3s2
-  ingressClassName: traefik

pkg/apis/k3k.io/v1alpha1/types.go

@@ -15,13 +15,19 @@ type Cluster struct {
 }
 
 type ClusterSpec struct {
-	Name    string `json:"name"`
-	Version string `json:"version"`
-	Servers *int32 `json:"servers"`
-	Agents  *int32 `json:"agents"`
-	Token   string `json:"token"`
+	Name        string `json:"name"`
+	Version     string `json:"version"`
+	Servers     *int32 `json:"servers"`
+	Agents      *int32 `json:"agents"`
+	Token       string `json:"token"`
+	ClusterCIDR string `json:"clusterCIDR,omitempty"`
+	ServiceCIDR string `json:"serviceCIDR,omitempty"`
+	ClusterDNS  string `json:"clusterDNS,omitempty"`
 
-	Expose ExposeConfig `json:"expose"`
+	ServerArgs []string `json:"serverArgs,omitempty"`
+	AgentArgs  []string `json:"agentArgs,omitempty"`
+
+	Expose ExposeConfig `json:"expose,omitempty"`
 }
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

pkg/apis/k3k.io/v1alpha1/zz_generated.deepcopy.go

@@ -82,6 +82,16 @@ func (in *ClusterSpec) DeepCopyInto(out *ClusterSpec) {
 		*out = new(int32)
 		**out = **in
 	}
+	if in.ServerArgs != nil {
+		in, out := &in.ServerArgs, &out.ServerArgs
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
+	if in.AgentArgs != nil {
+		in, out := &in.AgentArgs, &out.AgentArgs
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
 	in.Expose.DeepCopyInto(&out.Expose)
 	return
 }

pkg/controller/cluster/agent/agent.go

@@ -1,6 +1,8 @@
 package agent
 
 import (
+	"strings"
+
 	"github.com/galal-hussein/k3k/pkg/apis/k3k.io/v1alpha1"
 	"github.com/galal-hussein/k3k/pkg/controller/util"
 	apps "k8s.io/api/apps/v1"
@@ -37,13 +39,13 @@ func Agent(cluster *v1alpha1.Cluster) *apps.Deployment {
 						"type":    "agent",
 					},
 				},
-				Spec: agentPodSpec(image, name),
+				Spec: agentPodSpec(image, name, cluster.Spec.AgentArgs),
 			},
 		},
 	}
 }
 
-func agentPodSpec(image, name string) v1.PodSpec {
+func agentPodSpec(image, name string, args []string) v1.PodSpec {
 	privileged := true
 	return v1.PodSpec{
 		Volumes: []v1.Volume{
@@ -110,7 +112,9 @@ func agentPodSpec(image, name string) v1.PodSpec {
 				},
 				Args: []string{
 					"-c",
-					"/bin/k3s agent --config /opt/rancher/k3s/config.yaml && true",
+					"/bin/k3s agent --config /opt/rancher/k3s/config.yaml " +
+						strings.Join(args, " ") +
+						" && true",
 				},
 				VolumeMounts: []v1.VolumeMount{
 					{

pkg/controller/cluster/config/server.go

@@ -9,17 +9,17 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
-func ServerConfig(cluster *v1alpha1.Cluster, init bool, serviceIP string) v1.Secret {
+func ServerConfig(cluster *v1alpha1.Cluster, init bool, serviceIP string) (*v1.Secret, error) {
 	name := "k3k-server-config"
 	if init {
 		name = "k3k-init-server-config"
 	}
 
-	config := serverConfigData(serviceIP, cluster.Spec.Token)
+	config := serverConfigData(serviceIP, cluster)
 	if init {
-		config = initConfigData(cluster.Spec.Token)
+		config = initConfigData(cluster)
 	}
-	return v1.Secret{
+	return &v1.Secret{
 		TypeMeta: metav1.TypeMeta{
 			Kind:       "Secret",
 			APIVersion: "v1",
@@ -31,26 +31,36 @@ func ServerConfig(cluster *v1alpha1.Cluster, init bool, serviceIP string) v1.Sec
 		Data: map[string][]byte{
 			"config.yaml": []byte(config),
 		},
-	}
+	}, nil
 }
 
-func serverConfigData(serviceIP, token string) string {
+func serverConfigData(serviceIP string, cluster *v1alpha1.Cluster) string {
+	opts := serverOptions(cluster)
 	return fmt.Sprintf(`cluster-init: true
 server: https://%s:6443
-token: %s
-cluster-cidr: 10.40.0.0/16
-service-cidr: 10.44.0.0/16
-cluster-dns: 10.44.0.10
-tls-san:
-- 0.0.0.0`, serviceIP, token)
+%s`, serviceIP, opts)
 }
 
-func initConfigData(token string) string {
+func initConfigData(cluster *v1alpha1.Cluster) string {
+	opts := serverOptions(cluster)
 	return fmt.Sprintf(`cluster-init: true
-token: %s
-cluster-cidr: 10.40.0.0/16
-service-cidr: 10.44.0.0/16
-cluster-dns: 10.44.0.10
-tls-san:
-- 0.0.0.0`, token)
+%s`, opts)
+}
+
+func serverOptions(cluster *v1alpha1.Cluster) string {
+	opts := ""
+	// TODO: generate token if not found
+	if cluster.Spec.Token != "" {
+		opts = fmt.Sprintf("token: %s\n", cluster.Spec.Token)
+	}
+	if cluster.Spec.ClusterCIDR != "" {
+		opts = fmt.Sprintf("%scluster-cidr: %s\n", opts, cluster.Spec.ClusterCIDR)
+	}
+	if cluster.Spec.ServiceCIDR != "" {
+		opts = fmt.Sprintf("%sservice-cidr: %s\n", opts, cluster.Spec.ServiceCIDR)
+	}
+	if cluster.Spec.ClusterDNS != "" {
+		opts = fmt.Sprintf("%scluster-dns: %s\n", opts, cluster.Spec.ClusterDNS)
+	}
+	return opts
 }

pkg/controller/cluster/controller.go

@@ -119,16 +119,23 @@ func (r *ClusterReconciler) createCluster(ctx context.Context, cluster *v1alpha1
 	}
 
 	// create init node config
-	initServerConfigMap := config.ServerConfig(cluster, true, service.Spec.ClusterIP)
-	if err := r.Client.Create(ctx, &initServerConfigMap); err != nil {
+	initServerConfigMap, err := config.ServerConfig(cluster, true, service.Spec.ClusterIP)
+	if err != nil {
+		return util.WrapErr("failed to get init server config", err)
+	}
+	if err := r.Client.Create(ctx, initServerConfigMap); err != nil {
 		if !apierrors.IsAlreadyExists(err) {
 			return util.WrapErr("failed to create init configmap", err)
 		}
 	}
 
 	// create servers configuration
-	serverConfigMap := config.ServerConfig(cluster, false, service.Spec.ClusterIP)
-	if err := r.Client.Create(ctx, &serverConfigMap); err != nil {
+	serverConfigMap, err := config.ServerConfig(cluster, false, service.Spec.ClusterIP)
+	if err != nil {
+		return util.WrapErr("failed to get server config", err)
+
+	}
+	if err := r.Client.Create(ctx, serverConfigMap); err != nil {
 		if !apierrors.IsAlreadyExists(err) {
 			return util.WrapErr("failed to create configmap", err)
 		}

pkg/controller/cluster/server/ingress.go

@@ -5,7 +5,6 @@ import (
 
 	"github.com/galal-hussein/k3k/pkg/apis/k3k.io/v1alpha1"
 	"github.com/galal-hussein/k3k/pkg/controller/util"
-	v1 "k8s.io/api/core/v1"
 	networkingv1 "k8s.io/api/networking/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -20,7 +19,7 @@ const (
 )
 
 func Ingress(ctx context.Context, cluster *v1alpha1.Cluster, client client.Client) (*networkingv1.Ingress, error) {
-	addresses, err := addresses(ctx, client)
+	addresses, err := util.Addresses(ctx, client)
 	if err != nil {
 		return nil, err
 	}
@@ -46,39 +45,6 @@ func Ingress(ctx context.Context, cluster *v1alpha1.Cluster, client client.Clien
 	return ingress, nil
 }
 
-// return all the nodes external addresses, if not found then return internal addresses
-func addresses(ctx context.Context, client client.Client) ([]string, error) {
-	addresses := []string{}
-	nodeList := v1.NodeList{}
-	if err := client.List(ctx, &nodeList); err != nil {
-		return nil, err
-	}
-
-	for _, node := range nodeList.Items {
-		addresses = append(addresses, getNodeAddress(&node))
-	}
-
-	return addresses, nil
-}
-
-func getNodeAddress(node *v1.Node) string {
-	externalIP := ""
-	internalIP := ""
-	for _, ip := range node.Status.Addresses {
-		if ip.Type == "ExternalIP" && ip.Address != "" {
-			externalIP = ip.Address
-			break
-		} else if ip.Type == "InternalIP" && ip.Address != "" {
-			internalIP = ip.Address
-		}
-	}
-	if externalIP != "" {
-		return externalIP
-	}
-
-	return internalIP
-}
-
 func ingressRules(cluster *v1alpha1.Cluster, addresses []string) []networkingv1.IngressRule {
 	ingressRules := []networkingv1.IngressRule{}
 	pathTypePrefix := networkingv1.PathTypePrefix

pkg/controller/cluster/server/server.go

@@ -2,6 +2,7 @@ package server
 
 import (
 	"strconv"
+	"strings"
 
 	"github.com/galal-hussein/k3k/pkg/apis/k3k.io/v1alpha1"
 	"github.com/galal-hussein/k3k/pkg/controller/util"
@@ -49,13 +50,13 @@ func Server(cluster *v1alpha1.Cluster, init bool) *apps.Deployment {
 						"init":    strconv.FormatBool(init),
 					},
 				},
-				Spec: serverPodSpec(image, name),
+				Spec: serverPodSpec(image, name, cluster.Spec.ServerArgs),
 			},
 		},
 	}
 }
 
-func serverPodSpec(image, name string) v1.PodSpec {
+func serverPodSpec(image, name string, args []string) v1.PodSpec {
 	privileged := true
 	return v1.PodSpec{
 		Volumes: []v1.Volume{
@@ -122,7 +123,9 @@ func serverPodSpec(image, name string) v1.PodSpec {
 				},
 				Args: []string{
 					"-c",
-					"/bin/k3s server --config /opt/rancher/k3s/config.yaml && true",
+					"/bin/k3s server --config /opt/rancher/k3s/config.yaml " +
+						strings.Join(args, " ") +
+						" && true",
 				},
 				VolumeMounts: []v1.VolumeMount{
 					{

pkg/controller/util/util.go

@@ -1,8 +1,12 @@
 package util
 
 import (
+	"context"
+
 	"github.com/galal-hussein/k3k/pkg/apis/k3k.io/v1alpha1"
+	v1 "k8s.io/api/core/v1"
 	"k8s.io/klog"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 const (
@@ -22,3 +26,36 @@ func WrapErr(errString string, err error) error {
 	klog.Errorf("%s: %v", errString, err)
 	return err
 }
+
+// return all the nodes external addresses, if not found then return internal addresses
+func Addresses(ctx context.Context, client client.Client) ([]string, error) {
+	addresses := []string{}
+	nodeList := v1.NodeList{}
+	if err := client.List(ctx, &nodeList); err != nil {
+		return nil, err
+	}
+
+	for _, node := range nodeList.Items {
+		addresses = append(addresses, getNodeAddress(&node))
+	}
+
+	return addresses, nil
+}
+
+func getNodeAddress(node *v1.Node) string {
+	externalIP := ""
+	internalIP := ""
+	for _, ip := range node.Status.Addresses {
+		if ip.Type == "ExternalIP" && ip.Address != "" {
+			externalIP = ip.Address
+			break
+		} else if ip.Type == "InternalIP" && ip.Address != "" {
+			internalIP = ip.Address
+		}
+	}
+	if externalIP != "" {
+		return externalIP
+	}
+
+	return internalIP
+}