diff --git a/crd/cluster.yaml b/crd/cluster.yaml index 1e1d5daa..c2f6a9ff 100644 --- a/crd/cluster.yaml +++ b/crd/cluster.yaml @@ -25,8 +25,35 @@ spec: type: integer token: type: string - ingressClassName: + clusterCIDR: type: string + serviceCIDR: + type: string + clusterDNS: + type: string + serverArgs: + type: array + items: + type: string + agentArgs: + type: array + items: + type: string + expose: + type: object + properties: + ingress: + type: object + properties: + enabled: + type: boolean + ingressClassName: + type: string + loadbalancer: + type: object + properties: + enabled: + type: boolean scope: Namespaced names: plural: clusters diff --git a/examples/multiple-servers.yaml b/examples/multiple-servers.yaml new file mode 100644 index 00000000..11276e1d --- /dev/null +++ b/examples/multiple-servers.yaml @@ -0,0 +1,19 @@ +apiVersion: k3k.io/v1alpha1 +kind: Cluster +metadata: + name: example1 + namespace: default +spec: + servers: 1 + agents: 3 + token: test + version: v1.26.0-k3s2 + clusterCIDR: 10.30.0.0/16 + serviceCIDR: 10.31.0.0/16 + clusterDNS: 10.30.0.10 + serverArgs: + - "--write-kubeconfig-mode=777" + expose: + ingress: + enabled: true + ingressClassName: "nginx" diff --git a/examples/multiple-servers/multiple-servers.yaml b/examples/multiple-servers/multiple-servers.yaml deleted file mode 100644 index 0d1ac614..00000000 --- a/examples/multiple-servers/multiple-servers.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: k3k.io/v1alpha1 -kind: Cluster -metadata: - name: multiple-servers - namespace: default -spec: - servers: 2 - agents: 3 - token: test - version: v1.26.0-k3s2 - ingressClassName: traefik diff --git a/examples/single-server.yaml b/examples/single-server.yaml new file mode 100644 index 00000000..78c9a79e --- /dev/null +++ b/examples/single-server.yaml @@ -0,0 +1,19 @@ +apiVersion: k3k.io/v1alpha1 +kind: Cluster +metadata: + name: single-server + namespace: default +spec: + servers: 1 + agents: 3 + token: test + version: v1.26.0-k3s2 + clusterCIDR: 10.30.0.0/16 + serviceCIDR: 10.31.0.0/16 + clusterDNS: 10.30.0.10 + serverArgs: + - "--write-kubeconfig-mode=777" + expose: + ingress: + enabled: true + ingressClassName: "nginx" diff --git a/examples/single-server/single-server.yaml b/examples/single-server/single-server.yaml deleted file mode 100644 index 8e350c6f..00000000 --- a/examples/single-server/single-server.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: k3k.io/v1alpha1 -kind: Cluster -metadata: - name: single-server - namespace: default -spec: - servers: 1 - agents: 3 - token: test - version: v1.26.0-k3s2 - ingressClassName: traefik diff --git a/pkg/apis/k3k.io/v1alpha1/types.go b/pkg/apis/k3k.io/v1alpha1/types.go index afd173d1..71002208 100644 --- a/pkg/apis/k3k.io/v1alpha1/types.go +++ b/pkg/apis/k3k.io/v1alpha1/types.go @@ -15,12 +15,19 @@ type Cluster struct { } type ClusterSpec struct { - Name string `json:"name"` - Version string `json:"version"` - Servers *int32 `json:"servers"` - Agents *int32 `json:"agents"` - Token string `json:"token"` - IngressClassName string `json:"ingressClassName"` + Name string `json:"name"` + Version string `json:"version"` + Servers *int32 `json:"servers"` + Agents *int32 `json:"agents"` + Token string `json:"token"` + ClusterCIDR string `json:"clusterCIDR,omitempty"` + ServiceCIDR string `json:"serviceCIDR,omitempty"` + ClusterDNS string `json:"clusterDNS,omitempty"` + + ServerArgs []string `json:"serverArgs,omitempty"` + AgentArgs []string `json:"agentArgs,omitempty"` + + Expose ExposeConfig `json:"expose,omitempty"` } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object @@ -31,3 +38,17 @@ type ClusterList struct { Items []Cluster `json:"items"` } + +type ExposeConfig struct { + Ingress *IngressConfig `json:"ingress"` + LoadBalancer *LoadBalancerConfig `json:"loadbalancer"` +} + +type IngressConfig struct { + Enabled bool `json:"enabled"` + IngressClassName string `json:"ingressClassName"` +} + +type LoadBalancerConfig struct { + Enabled bool `json:"enabled"` +} diff --git a/pkg/apis/k3k.io/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/k3k.io/v1alpha1/zz_generated.deepcopy.go index 49797e26..3e8b5c99 100644 --- a/pkg/apis/k3k.io/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/k3k.io/v1alpha1/zz_generated.deepcopy.go @@ -82,6 +82,17 @@ func (in *ClusterSpec) DeepCopyInto(out *ClusterSpec) { *out = new(int32) **out = **in } + if in.ServerArgs != nil { + in, out := &in.ServerArgs, &out.ServerArgs + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.AgentArgs != nil { + in, out := &in.AgentArgs, &out.AgentArgs + *out = make([]string, len(*in)) + copy(*out, *in) + } + in.Expose.DeepCopyInto(&out.Expose) return } @@ -94,3 +105,61 @@ func (in *ClusterSpec) DeepCopy() *ClusterSpec { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ExposeConfig) DeepCopyInto(out *ExposeConfig) { + *out = *in + if in.Ingress != nil { + in, out := &in.Ingress, &out.Ingress + *out = new(IngressConfig) + **out = **in + } + if in.LoadBalancer != nil { + in, out := &in.LoadBalancer, &out.LoadBalancer + *out = new(LoadBalancerConfig) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExposeConfig. +func (in *ExposeConfig) DeepCopy() *ExposeConfig { + if in == nil { + return nil + } + out := new(ExposeConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *IngressConfig) DeepCopyInto(out *IngressConfig) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IngressConfig. +func (in *IngressConfig) DeepCopy() *IngressConfig { + if in == nil { + return nil + } + out := new(IngressConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LoadBalancerConfig) DeepCopyInto(out *LoadBalancerConfig) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LoadBalancerConfig. +func (in *LoadBalancerConfig) DeepCopy() *LoadBalancerConfig { + if in == nil { + return nil + } + out := new(LoadBalancerConfig) + in.DeepCopyInto(out) + return out +} diff --git a/pkg/controller/cluster/agent/agent.go b/pkg/controller/cluster/agent/agent.go index a805aba3..b7c15229 100644 --- a/pkg/controller/cluster/agent/agent.go +++ b/pkg/controller/cluster/agent/agent.go @@ -1,6 +1,8 @@ package agent import ( + "strings" + "github.com/galal-hussein/k3k/pkg/apis/k3k.io/v1alpha1" "github.com/galal-hussein/k3k/pkg/controller/util" apps "k8s.io/api/apps/v1" @@ -37,13 +39,13 @@ func Agent(cluster *v1alpha1.Cluster) *apps.Deployment { "type": "agent", }, }, - Spec: agentPodSpec(image, name), + Spec: agentPodSpec(image, name, cluster.Spec.AgentArgs), }, }, } } -func agentPodSpec(image, name string) v1.PodSpec { +func agentPodSpec(image, name string, args []string) v1.PodSpec { privileged := true return v1.PodSpec{ Volumes: []v1.Volume{ @@ -110,7 +112,9 @@ func agentPodSpec(image, name string) v1.PodSpec { }, Args: []string{ "-c", - "/bin/k3s agent --config /opt/rancher/k3s/config.yaml && true", + "/bin/k3s agent --config /opt/rancher/k3s/config.yaml " + + strings.Join(args, " ") + + " && true", }, VolumeMounts: []v1.VolumeMount{ { diff --git a/pkg/controller/cluster/config/server.go b/pkg/controller/cluster/config/server.go index d7055594..3ebe89fe 100644 --- a/pkg/controller/cluster/config/server.go +++ b/pkg/controller/cluster/config/server.go @@ -9,17 +9,17 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -func ServerConfig(cluster *v1alpha1.Cluster, init bool, serviceIP string) v1.Secret { +func ServerConfig(cluster *v1alpha1.Cluster, init bool, serviceIP string) (*v1.Secret, error) { name := "k3k-server-config" if init { name = "k3k-init-server-config" } - config := serverConfigData(serviceIP, cluster.Spec.Token) + config := serverConfigData(serviceIP, cluster) if init { - config = initConfigData(cluster.Spec.Token) + config = initConfigData(cluster) } - return v1.Secret{ + return &v1.Secret{ TypeMeta: metav1.TypeMeta{ Kind: "Secret", APIVersion: "v1", @@ -31,26 +31,36 @@ func ServerConfig(cluster *v1alpha1.Cluster, init bool, serviceIP string) v1.Sec Data: map[string][]byte{ "config.yaml": []byte(config), }, - } + }, nil } -func serverConfigData(serviceIP, token string) string { +func serverConfigData(serviceIP string, cluster *v1alpha1.Cluster) string { + opts := serverOptions(cluster) return fmt.Sprintf(`cluster-init: true server: https://%s:6443 -token: %s -cluster-cidr: 10.40.0.0/16 -service-cidr: 10.44.0.0/16 -cluster-dns: 10.44.0.10 -tls-san: -- 0.0.0.0`, serviceIP, token) +%s`, serviceIP, opts) } -func initConfigData(token string) string { +func initConfigData(cluster *v1alpha1.Cluster) string { + opts := serverOptions(cluster) return fmt.Sprintf(`cluster-init: true -token: %s -cluster-cidr: 10.40.0.0/16 -service-cidr: 10.44.0.0/16 -cluster-dns: 10.44.0.10 -tls-san: -- 0.0.0.0`, token) +%s`, opts) +} + +func serverOptions(cluster *v1alpha1.Cluster) string { + opts := "" + // TODO: generate token if not found + if cluster.Spec.Token != "" { + opts = fmt.Sprintf("token: %s\n", cluster.Spec.Token) + } + if cluster.Spec.ClusterCIDR != "" { + opts = fmt.Sprintf("%scluster-cidr: %s\n", opts, cluster.Spec.ClusterCIDR) + } + if cluster.Spec.ServiceCIDR != "" { + opts = fmt.Sprintf("%sservice-cidr: %s\n", opts, cluster.Spec.ServiceCIDR) + } + if cluster.Spec.ClusterDNS != "" { + opts = fmt.Sprintf("%scluster-dns: %s\n", opts, cluster.Spec.ClusterDNS) + } + return opts } diff --git a/pkg/controller/cluster/controller.go b/pkg/controller/cluster/controller.go index 41efbd98..11ce47f3 100644 --- a/pkg/controller/cluster/controller.go +++ b/pkg/controller/cluster/controller.go @@ -119,16 +119,23 @@ func (r *ClusterReconciler) createCluster(ctx context.Context, cluster *v1alpha1 } // create init node config - initServerConfigMap := config.ServerConfig(cluster, true, service.Spec.ClusterIP) - if err := r.Client.Create(ctx, &initServerConfigMap); err != nil { + initServerConfigMap, err := config.ServerConfig(cluster, true, service.Spec.ClusterIP) + if err != nil { + return util.WrapErr("failed to get init server config", err) + } + if err := r.Client.Create(ctx, initServerConfigMap); err != nil { if !apierrors.IsAlreadyExists(err) { return util.WrapErr("failed to create init configmap", err) } } // create servers configuration - serverConfigMap := config.ServerConfig(cluster, false, service.Spec.ClusterIP) - if err := r.Client.Create(ctx, &serverConfigMap); err != nil { + serverConfigMap, err := config.ServerConfig(cluster, false, service.Spec.ClusterIP) + if err != nil { + return util.WrapErr("failed to get server config", err) + + } + if err := r.Client.Create(ctx, serverConfigMap); err != nil { if !apierrors.IsAlreadyExists(err) { return util.WrapErr("failed to create configmap", err) } @@ -165,14 +172,15 @@ func (r *ClusterReconciler) createCluster(ctx context.Context, cluster *v1alpha1 } } - // create ingress with random port for the server - serverIngress, err := server.Ingress(ctx, cluster, r.Client) - if err != nil { - return util.WrapErr("failed to create ingress object", err) - } - if err := r.Client.Create(ctx, serverIngress); err != nil { - if !apierrors.IsAlreadyExists(err) { - return util.WrapErr("failed to create server ingress", err) + if cluster.Spec.Expose.Ingress.Enabled { + serverIngress, err := server.Ingress(ctx, cluster, r.Client) + if err != nil { + return util.WrapErr("failed to create ingress object", err) + } + if err := r.Client.Create(ctx, serverIngress); err != nil { + if !apierrors.IsAlreadyExists(err) { + return util.WrapErr("failed to create server ingress", err) + } } } diff --git a/pkg/controller/cluster/server/ingress.go b/pkg/controller/cluster/server/ingress.go index b3224453..064662b3 100644 --- a/pkg/controller/cluster/server/ingress.go +++ b/pkg/controller/cluster/server/ingress.go @@ -5,25 +5,27 @@ import ( "github.com/galal-hussein/k3k/pkg/apis/k3k.io/v1alpha1" "github.com/galal-hussein/k3k/pkg/controller/util" - v1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" ) -var ( - pathType = networkingv1.PathTypePrefix +const ( wildcardDNS = ".sslip.io" + + nginxSSLPassthroughAnnotation = "nginx.ingress.kubernetes.io/ssl-passthrough" + nginxBackendProtocolAnnotation = "nginx.ingress.kubernetes.io/backend-protocol" + nginxSSLRedirectAnnotation = "nginx.ingress.kubernetes.io/ssl-redirect" ) func Ingress(ctx context.Context, cluster *v1alpha1.Cluster, client client.Client) (*networkingv1.Ingress, error) { - addresses, err := addresses(ctx, client) + addresses, err := util.Addresses(ctx, client) if err != nil { return nil, err } ingressRules := ingressRules(cluster, addresses) - return &networkingv1.Ingress{ + ingress := &networkingv1.Ingress{ TypeMeta: metav1.TypeMeta{ Kind: "Ingress", APIVersion: "networking.k8s.io/v1", @@ -33,47 +35,19 @@ func Ingress(ctx context.Context, cluster *v1alpha1.Cluster, client client.Clien Namespace: util.ClusterNamespace(cluster), }, Spec: networkingv1.IngressSpec{ - IngressClassName: &cluster.Spec.IngressClassName, + IngressClassName: &cluster.Spec.Expose.Ingress.IngressClassName, Rules: ingressRules, }, - }, nil -} - -// return all the nodes external addresses, if not found then return internal addresses -func addresses(ctx context.Context, client client.Client) ([]string, error) { - addresses := []string{} - nodeList := v1.NodeList{} - if err := client.List(ctx, &nodeList); err != nil { - return nil, err } - for _, node := range nodeList.Items { - addresses = append(addresses, GetNodeAddress(&node)) - } + configureIngressOptions(ingress, cluster.Spec.Expose.Ingress.IngressClassName) - return addresses, nil -} - -func GetNodeAddress(node *v1.Node) string { - externalIP := "" - internalIP := "" - for _, ip := range node.Status.Addresses { - if ip.Type == "ExternalIP" && ip.Address != "" { - externalIP = ip.Address - break - } else if ip.Type == "InternalIP" && ip.Address != "" { - internalIP = ip.Address - } - } - if externalIP != "" { - return externalIP - } - - return internalIP + return ingress, nil } func ingressRules(cluster *v1alpha1.Cluster, addresses []string) []networkingv1.IngressRule { ingressRules := []networkingv1.IngressRule{} + pathTypePrefix := networkingv1.PathTypePrefix for _, address := range addresses { rule := networkingv1.IngressRule{ Host: cluster.Name + "." + address + wildcardDNS, @@ -82,7 +56,7 @@ func ingressRules(cluster *v1alpha1.Cluster, addresses []string) []networkingv1. Paths: []networkingv1.HTTPIngressPath{ { Path: "/", - PathType: &pathType, + PathType: &pathTypePrefix, Backend: networkingv1.IngressBackend{ Service: &networkingv1.IngressServiceBackend{ Name: "k3k-server-service", @@ -100,3 +74,17 @@ func ingressRules(cluster *v1alpha1.Cluster, addresses []string) []networkingv1. } return ingressRules } + +// configureIngressOptions will configure the ingress object by +// adding tls passthrough capabilities and TLS needed annotations +// it depends on the ingressclassname to configure each ingress +// TODO: add treafik support through ingresstcproutes +func configureIngressOptions(ingress *networkingv1.Ingress, ingressClassName string) { + // initial support for nginx ingress via annotations + if ingressClassName == "nginx" { + ingress.Annotations = make(map[string]string) + ingress.Annotations[nginxSSLPassthroughAnnotation] = "true" + ingress.Annotations[nginxSSLRedirectAnnotation] = "true" + ingress.Annotations[nginxBackendProtocolAnnotation] = "HTTPS" + } +} diff --git a/pkg/controller/cluster/server/kubeconfig.go b/pkg/controller/cluster/server/kubeconfig.go index 2f31dc19..22fdfd14 100644 --- a/pkg/controller/cluster/server/kubeconfig.go +++ b/pkg/controller/cluster/server/kubeconfig.go @@ -40,7 +40,7 @@ type content struct { Content string } -// GenerateNewKubeConfig generates the kubeconfig for the server: +// GenerateNewKubeConfig generates the kubeconfig for the cluster: // 1- use the server token to get the bootstrap data from k3s // 2- generate client admin cert/key // 3- use the ca cert from the bootstrap data & admin cert/key to write a new kubeconfig diff --git a/pkg/controller/cluster/server/server.go b/pkg/controller/cluster/server/server.go index 9b4c5e8e..eb14a9cb 100644 --- a/pkg/controller/cluster/server/server.go +++ b/pkg/controller/cluster/server/server.go @@ -2,6 +2,7 @@ package server import ( "strconv" + "strings" "github.com/galal-hussein/k3k/pkg/apis/k3k.io/v1alpha1" "github.com/galal-hussein/k3k/pkg/controller/util" @@ -49,13 +50,13 @@ func Server(cluster *v1alpha1.Cluster, init bool) *apps.Deployment { "init": strconv.FormatBool(init), }, }, - Spec: serverPodSpec(image, name), + Spec: serverPodSpec(image, name, cluster.Spec.ServerArgs), }, }, } } -func serverPodSpec(image, name string) v1.PodSpec { +func serverPodSpec(image, name string, args []string) v1.PodSpec { privileged := true return v1.PodSpec{ Volumes: []v1.Volume{ @@ -122,7 +123,9 @@ func serverPodSpec(image, name string) v1.PodSpec { }, Args: []string{ "-c", - "/bin/k3s server --config /opt/rancher/k3s/config.yaml && true", + "/bin/k3s server --config /opt/rancher/k3s/config.yaml " + + strings.Join(args, " ") + + " && true", }, VolumeMounts: []v1.VolumeMount{ { diff --git a/pkg/controller/util/util.go b/pkg/controller/util/util.go index 0fe8b2a1..d262ba3c 100644 --- a/pkg/controller/util/util.go +++ b/pkg/controller/util/util.go @@ -1,8 +1,12 @@ package util import ( + "context" + "github.com/galal-hussein/k3k/pkg/apis/k3k.io/v1alpha1" + v1 "k8s.io/api/core/v1" "k8s.io/klog" + "sigs.k8s.io/controller-runtime/pkg/client" ) const ( @@ -22,3 +26,36 @@ func WrapErr(errString string, err error) error { klog.Errorf("%s: %v", errString, err) return err } + +// return all the nodes external addresses, if not found then return internal addresses +func Addresses(ctx context.Context, client client.Client) ([]string, error) { + addresses := []string{} + nodeList := v1.NodeList{} + if err := client.List(ctx, &nodeList); err != nil { + return nil, err + } + + for _, node := range nodeList.Items { + addresses = append(addresses, getNodeAddress(&node)) + } + + return addresses, nil +} + +func getNodeAddress(node *v1.Node) string { + externalIP := "" + internalIP := "" + for _, ip := range node.Status.Addresses { + if ip.Type == "ExternalIP" && ip.Address != "" { + externalIP = ip.Address + break + } else if ip.Type == "InternalIP" && ip.Address != "" { + internalIP = ip.Address + } + } + if externalIP != "" { + return externalIP + } + + return internalIP +}