diff --git a/crd/cluster.yaml b/crd/cluster.yaml index c833b3d..c2f6a9f 100644 --- a/crd/cluster.yaml +++ b/crd/cluster.yaml @@ -25,6 +25,20 @@ spec: type: integer token: type: string + clusterCIDR: + type: string + serviceCIDR: + type: string + clusterDNS: + type: string + serverArgs: + type: array + items: + type: string + agentArgs: + type: array + items: + type: string expose: type: object properties: diff --git a/examples/multiple-servers/multiple-servers.yaml b/examples/multiple-servers.yaml similarity index 56% rename from examples/multiple-servers/multiple-servers.yaml rename to examples/multiple-servers.yaml index d06bd97..11276e1 100644 --- a/examples/multiple-servers/multiple-servers.yaml +++ b/examples/multiple-servers.yaml @@ -1,13 +1,18 @@ apiVersion: k3k.io/v1alpha1 kind: Cluster metadata: - name: multiple-servers + name: example1 namespace: default spec: - servers: 2 + servers: 1 agents: 3 token: test version: v1.26.0-k3s2 + clusterCIDR: 10.30.0.0/16 + serviceCIDR: 10.31.0.0/16 + clusterDNS: 10.30.0.10 + serverArgs: + - "--write-kubeconfig-mode=777" expose: ingress: enabled: true diff --git a/examples/single-server.yaml b/examples/single-server.yaml new file mode 100644 index 0000000..78c9a79 --- /dev/null +++ b/examples/single-server.yaml @@ -0,0 +1,19 @@ +apiVersion: k3k.io/v1alpha1 +kind: Cluster +metadata: + name: single-server + namespace: default +spec: + servers: 1 + agents: 3 + token: test + version: v1.26.0-k3s2 + clusterCIDR: 10.30.0.0/16 + serviceCIDR: 10.31.0.0/16 + clusterDNS: 10.30.0.10 + serverArgs: + - "--write-kubeconfig-mode=777" + expose: + ingress: + enabled: true + ingressClassName: "nginx" diff --git a/examples/single-server/single-server.yaml b/examples/single-server/single-server.yaml deleted file mode 100644 index 8e350c6..0000000 --- a/examples/single-server/single-server.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: k3k.io/v1alpha1 -kind: Cluster -metadata: - name: single-server - namespace: default -spec: - servers: 1 - agents: 3 - token: test - version: v1.26.0-k3s2 - ingressClassName: traefik diff --git a/pkg/apis/k3k.io/v1alpha1/types.go b/pkg/apis/k3k.io/v1alpha1/types.go index 7a7aeed..7100220 100644 --- a/pkg/apis/k3k.io/v1alpha1/types.go +++ b/pkg/apis/k3k.io/v1alpha1/types.go @@ -15,13 +15,19 @@ type Cluster struct { } type ClusterSpec struct { - Name string `json:"name"` - Version string `json:"version"` - Servers *int32 `json:"servers"` - Agents *int32 `json:"agents"` - Token string `json:"token"` + Name string `json:"name"` + Version string `json:"version"` + Servers *int32 `json:"servers"` + Agents *int32 `json:"agents"` + Token string `json:"token"` + ClusterCIDR string `json:"clusterCIDR,omitempty"` + ServiceCIDR string `json:"serviceCIDR,omitempty"` + ClusterDNS string `json:"clusterDNS,omitempty"` - Expose ExposeConfig `json:"expose"` + ServerArgs []string `json:"serverArgs,omitempty"` + AgentArgs []string `json:"agentArgs,omitempty"` + + Expose ExposeConfig `json:"expose,omitempty"` } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object diff --git a/pkg/apis/k3k.io/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/k3k.io/v1alpha1/zz_generated.deepcopy.go index c8555b0..3e8b5c9 100644 --- a/pkg/apis/k3k.io/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/k3k.io/v1alpha1/zz_generated.deepcopy.go @@ -82,6 +82,16 @@ func (in *ClusterSpec) DeepCopyInto(out *ClusterSpec) { *out = new(int32) **out = **in } + if in.ServerArgs != nil { + in, out := &in.ServerArgs, &out.ServerArgs + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.AgentArgs != nil { + in, out := &in.AgentArgs, &out.AgentArgs + *out = make([]string, len(*in)) + copy(*out, *in) + } in.Expose.DeepCopyInto(&out.Expose) return } diff --git a/pkg/controller/cluster/agent/agent.go b/pkg/controller/cluster/agent/agent.go index a805aba..b7c1522 100644 --- a/pkg/controller/cluster/agent/agent.go +++ b/pkg/controller/cluster/agent/agent.go @@ -1,6 +1,8 @@ package agent import ( + "strings" + "github.com/galal-hussein/k3k/pkg/apis/k3k.io/v1alpha1" "github.com/galal-hussein/k3k/pkg/controller/util" apps "k8s.io/api/apps/v1" @@ -37,13 +39,13 @@ func Agent(cluster *v1alpha1.Cluster) *apps.Deployment { "type": "agent", }, }, - Spec: agentPodSpec(image, name), + Spec: agentPodSpec(image, name, cluster.Spec.AgentArgs), }, }, } } -func agentPodSpec(image, name string) v1.PodSpec { +func agentPodSpec(image, name string, args []string) v1.PodSpec { privileged := true return v1.PodSpec{ Volumes: []v1.Volume{ @@ -110,7 +112,9 @@ func agentPodSpec(image, name string) v1.PodSpec { }, Args: []string{ "-c", - "/bin/k3s agent --config /opt/rancher/k3s/config.yaml && true", + "/bin/k3s agent --config /opt/rancher/k3s/config.yaml " + + strings.Join(args, " ") + + " && true", }, VolumeMounts: []v1.VolumeMount{ { diff --git a/pkg/controller/cluster/config/server.go b/pkg/controller/cluster/config/server.go index d705559..3ebe89f 100644 --- a/pkg/controller/cluster/config/server.go +++ b/pkg/controller/cluster/config/server.go @@ -9,17 +9,17 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -func ServerConfig(cluster *v1alpha1.Cluster, init bool, serviceIP string) v1.Secret { +func ServerConfig(cluster *v1alpha1.Cluster, init bool, serviceIP string) (*v1.Secret, error) { name := "k3k-server-config" if init { name = "k3k-init-server-config" } - config := serverConfigData(serviceIP, cluster.Spec.Token) + config := serverConfigData(serviceIP, cluster) if init { - config = initConfigData(cluster.Spec.Token) + config = initConfigData(cluster) } - return v1.Secret{ + return &v1.Secret{ TypeMeta: metav1.TypeMeta{ Kind: "Secret", APIVersion: "v1", @@ -31,26 +31,36 @@ func ServerConfig(cluster *v1alpha1.Cluster, init bool, serviceIP string) v1.Sec Data: map[string][]byte{ "config.yaml": []byte(config), }, - } + }, nil } -func serverConfigData(serviceIP, token string) string { +func serverConfigData(serviceIP string, cluster *v1alpha1.Cluster) string { + opts := serverOptions(cluster) return fmt.Sprintf(`cluster-init: true server: https://%s:6443 -token: %s -cluster-cidr: 10.40.0.0/16 -service-cidr: 10.44.0.0/16 -cluster-dns: 10.44.0.10 -tls-san: -- 0.0.0.0`, serviceIP, token) +%s`, serviceIP, opts) } -func initConfigData(token string) string { +func initConfigData(cluster *v1alpha1.Cluster) string { + opts := serverOptions(cluster) return fmt.Sprintf(`cluster-init: true -token: %s -cluster-cidr: 10.40.0.0/16 -service-cidr: 10.44.0.0/16 -cluster-dns: 10.44.0.10 -tls-san: -- 0.0.0.0`, token) +%s`, opts) +} + +func serverOptions(cluster *v1alpha1.Cluster) string { + opts := "" + // TODO: generate token if not found + if cluster.Spec.Token != "" { + opts = fmt.Sprintf("token: %s\n", cluster.Spec.Token) + } + if cluster.Spec.ClusterCIDR != "" { + opts = fmt.Sprintf("%scluster-cidr: %s\n", opts, cluster.Spec.ClusterCIDR) + } + if cluster.Spec.ServiceCIDR != "" { + opts = fmt.Sprintf("%sservice-cidr: %s\n", opts, cluster.Spec.ServiceCIDR) + } + if cluster.Spec.ClusterDNS != "" { + opts = fmt.Sprintf("%scluster-dns: %s\n", opts, cluster.Spec.ClusterDNS) + } + return opts } diff --git a/pkg/controller/cluster/controller.go b/pkg/controller/cluster/controller.go index 1f7445e..11ce47f 100644 --- a/pkg/controller/cluster/controller.go +++ b/pkg/controller/cluster/controller.go @@ -119,16 +119,23 @@ func (r *ClusterReconciler) createCluster(ctx context.Context, cluster *v1alpha1 } // create init node config - initServerConfigMap := config.ServerConfig(cluster, true, service.Spec.ClusterIP) - if err := r.Client.Create(ctx, &initServerConfigMap); err != nil { + initServerConfigMap, err := config.ServerConfig(cluster, true, service.Spec.ClusterIP) + if err != nil { + return util.WrapErr("failed to get init server config", err) + } + if err := r.Client.Create(ctx, initServerConfigMap); err != nil { if !apierrors.IsAlreadyExists(err) { return util.WrapErr("failed to create init configmap", err) } } // create servers configuration - serverConfigMap := config.ServerConfig(cluster, false, service.Spec.ClusterIP) - if err := r.Client.Create(ctx, &serverConfigMap); err != nil { + serverConfigMap, err := config.ServerConfig(cluster, false, service.Spec.ClusterIP) + if err != nil { + return util.WrapErr("failed to get server config", err) + + } + if err := r.Client.Create(ctx, serverConfigMap); err != nil { if !apierrors.IsAlreadyExists(err) { return util.WrapErr("failed to create configmap", err) } diff --git a/pkg/controller/cluster/server/ingress.go b/pkg/controller/cluster/server/ingress.go index 27b0cfc..064662b 100644 --- a/pkg/controller/cluster/server/ingress.go +++ b/pkg/controller/cluster/server/ingress.go @@ -5,7 +5,6 @@ import ( "github.com/galal-hussein/k3k/pkg/apis/k3k.io/v1alpha1" "github.com/galal-hussein/k3k/pkg/controller/util" - v1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -20,7 +19,7 @@ const ( ) func Ingress(ctx context.Context, cluster *v1alpha1.Cluster, client client.Client) (*networkingv1.Ingress, error) { - addresses, err := addresses(ctx, client) + addresses, err := util.Addresses(ctx, client) if err != nil { return nil, err } @@ -46,39 +45,6 @@ func Ingress(ctx context.Context, cluster *v1alpha1.Cluster, client client.Clien return ingress, nil } -// return all the nodes external addresses, if not found then return internal addresses -func addresses(ctx context.Context, client client.Client) ([]string, error) { - addresses := []string{} - nodeList := v1.NodeList{} - if err := client.List(ctx, &nodeList); err != nil { - return nil, err - } - - for _, node := range nodeList.Items { - addresses = append(addresses, getNodeAddress(&node)) - } - - return addresses, nil -} - -func getNodeAddress(node *v1.Node) string { - externalIP := "" - internalIP := "" - for _, ip := range node.Status.Addresses { - if ip.Type == "ExternalIP" && ip.Address != "" { - externalIP = ip.Address - break - } else if ip.Type == "InternalIP" && ip.Address != "" { - internalIP = ip.Address - } - } - if externalIP != "" { - return externalIP - } - - return internalIP -} - func ingressRules(cluster *v1alpha1.Cluster, addresses []string) []networkingv1.IngressRule { ingressRules := []networkingv1.IngressRule{} pathTypePrefix := networkingv1.PathTypePrefix diff --git a/pkg/controller/cluster/server/server.go b/pkg/controller/cluster/server/server.go index 9b4c5e8..eb14a9c 100644 --- a/pkg/controller/cluster/server/server.go +++ b/pkg/controller/cluster/server/server.go @@ -2,6 +2,7 @@ package server import ( "strconv" + "strings" "github.com/galal-hussein/k3k/pkg/apis/k3k.io/v1alpha1" "github.com/galal-hussein/k3k/pkg/controller/util" @@ -49,13 +50,13 @@ func Server(cluster *v1alpha1.Cluster, init bool) *apps.Deployment { "init": strconv.FormatBool(init), }, }, - Spec: serverPodSpec(image, name), + Spec: serverPodSpec(image, name, cluster.Spec.ServerArgs), }, }, } } -func serverPodSpec(image, name string) v1.PodSpec { +func serverPodSpec(image, name string, args []string) v1.PodSpec { privileged := true return v1.PodSpec{ Volumes: []v1.Volume{ @@ -122,7 +123,9 @@ func serverPodSpec(image, name string) v1.PodSpec { }, Args: []string{ "-c", - "/bin/k3s server --config /opt/rancher/k3s/config.yaml && true", + "/bin/k3s server --config /opt/rancher/k3s/config.yaml " + + strings.Join(args, " ") + + " && true", }, VolumeMounts: []v1.VolumeMount{ { diff --git a/pkg/controller/util/util.go b/pkg/controller/util/util.go index 0fe8b2a..d262ba3 100644 --- a/pkg/controller/util/util.go +++ b/pkg/controller/util/util.go @@ -1,8 +1,12 @@ package util import ( + "context" + "github.com/galal-hussein/k3k/pkg/apis/k3k.io/v1alpha1" + v1 "k8s.io/api/core/v1" "k8s.io/klog" + "sigs.k8s.io/controller-runtime/pkg/client" ) const ( @@ -22,3 +26,36 @@ func WrapErr(errString string, err error) error { klog.Errorf("%s: %v", errString, err) return err } + +// return all the nodes external addresses, if not found then return internal addresses +func Addresses(ctx context.Context, client client.Client) ([]string, error) { + addresses := []string{} + nodeList := v1.NodeList{} + if err := client.List(ctx, &nodeList); err != nil { + return nil, err + } + + for _, node := range nodeList.Items { + addresses = append(addresses, getNodeAddress(&node)) + } + + return addresses, nil +} + +func getNodeAddress(node *v1.Node) string { + externalIP := "" + internalIP := "" + for _, ip := range node.Status.Addresses { + if ip.Type == "ExternalIP" && ip.Address != "" { + externalIP = ip.Address + break + } else if ip.Type == "InternalIP" && ip.Address != "" { + internalIP = ip.Address + } + } + if externalIP != "" { + return externalIP + } + + return internalIP +}