From 7794188415024fcdd02a0081676983c217293b4c Mon Sep 17 00:00:00 2001 From: galal-hussein Date: Tue, 24 Jan 2023 23:39:41 +0200 Subject: [PATCH 1/2] Add expose options to the cluster --- crd/cluster.yaml | 17 +++++- .../multiple-servers/multiple-servers.yaml | 5 +- pkg/apis/k3k.io/v1alpha1/types.go | 27 +++++++-- .../k3k.io/v1alpha1/zz_generated.deepcopy.go | 59 +++++++++++++++++++ pkg/controller/cluster/controller.go | 17 +++--- pkg/controller/cluster/server/ingress.go | 38 +++++++++--- pkg/controller/cluster/server/kubeconfig.go | 2 +- 7 files changed, 139 insertions(+), 26 deletions(-) diff --git a/crd/cluster.yaml b/crd/cluster.yaml index 1e1d5daa..c833b3df 100644 --- a/crd/cluster.yaml +++ b/crd/cluster.yaml @@ -25,8 +25,21 @@ spec: type: integer token: type: string - ingressClassName: - type: string + expose: + type: object + properties: + ingress: + type: object + properties: + enabled: + type: boolean + ingressClassName: + type: string + loadbalancer: + type: object + properties: + enabled: + type: boolean scope: Namespaced names: plural: clusters diff --git a/examples/multiple-servers/multiple-servers.yaml b/examples/multiple-servers/multiple-servers.yaml index 0d1ac614..d06bd971 100644 --- a/examples/multiple-servers/multiple-servers.yaml +++ b/examples/multiple-servers/multiple-servers.yaml @@ -8,4 +8,7 @@ spec: agents: 3 token: test version: v1.26.0-k3s2 - ingressClassName: traefik + expose: + ingress: + enabled: true + ingressClassName: "nginx" diff --git a/pkg/apis/k3k.io/v1alpha1/types.go b/pkg/apis/k3k.io/v1alpha1/types.go index afd173d1..7a7aeed7 100644 --- a/pkg/apis/k3k.io/v1alpha1/types.go +++ b/pkg/apis/k3k.io/v1alpha1/types.go @@ -15,12 +15,13 @@ type Cluster struct { } type ClusterSpec struct { - Name string `json:"name"` - Version string `json:"version"` - Servers *int32 `json:"servers"` - Agents *int32 `json:"agents"` - Token string `json:"token"` - IngressClassName string `json:"ingressClassName"` + Name string `json:"name"` + Version string `json:"version"` + Servers *int32 `json:"servers"` + Agents *int32 `json:"agents"` + Token string `json:"token"` + + Expose ExposeConfig `json:"expose"` } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object @@ -31,3 +32,17 @@ type ClusterList struct { Items []Cluster `json:"items"` } + +type ExposeConfig struct { + Ingress *IngressConfig `json:"ingress"` + LoadBalancer *LoadBalancerConfig `json:"loadbalancer"` +} + +type IngressConfig struct { + Enabled bool `json:"enabled"` + IngressClassName string `json:"ingressClassName"` +} + +type LoadBalancerConfig struct { + Enabled bool `json:"enabled"` +} diff --git a/pkg/apis/k3k.io/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/k3k.io/v1alpha1/zz_generated.deepcopy.go index 49797e26..c8555b09 100644 --- a/pkg/apis/k3k.io/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/k3k.io/v1alpha1/zz_generated.deepcopy.go @@ -82,6 +82,7 @@ func (in *ClusterSpec) DeepCopyInto(out *ClusterSpec) { *out = new(int32) **out = **in } + in.Expose.DeepCopyInto(&out.Expose) return } @@ -94,3 +95,61 @@ func (in *ClusterSpec) DeepCopy() *ClusterSpec { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ExposeConfig) DeepCopyInto(out *ExposeConfig) { + *out = *in + if in.Ingress != nil { + in, out := &in.Ingress, &out.Ingress + *out = new(IngressConfig) + **out = **in + } + if in.LoadBalancer != nil { + in, out := &in.LoadBalancer, &out.LoadBalancer + *out = new(LoadBalancerConfig) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExposeConfig. +func (in *ExposeConfig) DeepCopy() *ExposeConfig { + if in == nil { + return nil + } + out := new(ExposeConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *IngressConfig) DeepCopyInto(out *IngressConfig) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IngressConfig. +func (in *IngressConfig) DeepCopy() *IngressConfig { + if in == nil { + return nil + } + out := new(IngressConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LoadBalancerConfig) DeepCopyInto(out *LoadBalancerConfig) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LoadBalancerConfig. +func (in *LoadBalancerConfig) DeepCopy() *LoadBalancerConfig { + if in == nil { + return nil + } + out := new(LoadBalancerConfig) + in.DeepCopyInto(out) + return out +} diff --git a/pkg/controller/cluster/controller.go b/pkg/controller/cluster/controller.go index 41efbd98..1f7445ee 100644 --- a/pkg/controller/cluster/controller.go +++ b/pkg/controller/cluster/controller.go @@ -165,14 +165,15 @@ func (r *ClusterReconciler) createCluster(ctx context.Context, cluster *v1alpha1 } } - // create ingress with random port for the server - serverIngress, err := server.Ingress(ctx, cluster, r.Client) - if err != nil { - return util.WrapErr("failed to create ingress object", err) - } - if err := r.Client.Create(ctx, serverIngress); err != nil { - if !apierrors.IsAlreadyExists(err) { - return util.WrapErr("failed to create server ingress", err) + if cluster.Spec.Expose.Ingress.Enabled { + serverIngress, err := server.Ingress(ctx, cluster, r.Client) + if err != nil { + return util.WrapErr("failed to create ingress object", err) + } + if err := r.Client.Create(ctx, serverIngress); err != nil { + if !apierrors.IsAlreadyExists(err) { + return util.WrapErr("failed to create server ingress", err) + } } } diff --git a/pkg/controller/cluster/server/ingress.go b/pkg/controller/cluster/server/ingress.go index b3224453..27b0cfcd 100644 --- a/pkg/controller/cluster/server/ingress.go +++ b/pkg/controller/cluster/server/ingress.go @@ -11,9 +11,12 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -var ( - pathType = networkingv1.PathTypePrefix +const ( wildcardDNS = ".sslip.io" + + nginxSSLPassthroughAnnotation = "nginx.ingress.kubernetes.io/ssl-passthrough" + nginxBackendProtocolAnnotation = "nginx.ingress.kubernetes.io/backend-protocol" + nginxSSLRedirectAnnotation = "nginx.ingress.kubernetes.io/ssl-redirect" ) func Ingress(ctx context.Context, cluster *v1alpha1.Cluster, client client.Client) (*networkingv1.Ingress, error) { @@ -23,7 +26,7 @@ func Ingress(ctx context.Context, cluster *v1alpha1.Cluster, client client.Clien } ingressRules := ingressRules(cluster, addresses) - return &networkingv1.Ingress{ + ingress := &networkingv1.Ingress{ TypeMeta: metav1.TypeMeta{ Kind: "Ingress", APIVersion: "networking.k8s.io/v1", @@ -33,10 +36,14 @@ func Ingress(ctx context.Context, cluster *v1alpha1.Cluster, client client.Clien Namespace: util.ClusterNamespace(cluster), }, Spec: networkingv1.IngressSpec{ - IngressClassName: &cluster.Spec.IngressClassName, + IngressClassName: &cluster.Spec.Expose.Ingress.IngressClassName, Rules: ingressRules, }, - }, nil + } + + configureIngressOptions(ingress, cluster.Spec.Expose.Ingress.IngressClassName) + + return ingress, nil } // return all the nodes external addresses, if not found then return internal addresses @@ -48,13 +55,13 @@ func addresses(ctx context.Context, client client.Client) ([]string, error) { } for _, node := range nodeList.Items { - addresses = append(addresses, GetNodeAddress(&node)) + addresses = append(addresses, getNodeAddress(&node)) } return addresses, nil } -func GetNodeAddress(node *v1.Node) string { +func getNodeAddress(node *v1.Node) string { externalIP := "" internalIP := "" for _, ip := range node.Status.Addresses { @@ -74,6 +81,7 @@ func GetNodeAddress(node *v1.Node) string { func ingressRules(cluster *v1alpha1.Cluster, addresses []string) []networkingv1.IngressRule { ingressRules := []networkingv1.IngressRule{} + pathTypePrefix := networkingv1.PathTypePrefix for _, address := range addresses { rule := networkingv1.IngressRule{ Host: cluster.Name + "." + address + wildcardDNS, @@ -82,7 +90,7 @@ func ingressRules(cluster *v1alpha1.Cluster, addresses []string) []networkingv1. Paths: []networkingv1.HTTPIngressPath{ { Path: "/", - PathType: &pathType, + PathType: &pathTypePrefix, Backend: networkingv1.IngressBackend{ Service: &networkingv1.IngressServiceBackend{ Name: "k3k-server-service", @@ -100,3 +108,17 @@ func ingressRules(cluster *v1alpha1.Cluster, addresses []string) []networkingv1. } return ingressRules } + +// configureIngressOptions will configure the ingress object by +// adding tls passthrough capabilities and TLS needed annotations +// it depends on the ingressclassname to configure each ingress +// TODO: add treafik support through ingresstcproutes +func configureIngressOptions(ingress *networkingv1.Ingress, ingressClassName string) { + // initial support for nginx ingress via annotations + if ingressClassName == "nginx" { + ingress.Annotations = make(map[string]string) + ingress.Annotations[nginxSSLPassthroughAnnotation] = "true" + ingress.Annotations[nginxSSLRedirectAnnotation] = "true" + ingress.Annotations[nginxBackendProtocolAnnotation] = "HTTPS" + } +} diff --git a/pkg/controller/cluster/server/kubeconfig.go b/pkg/controller/cluster/server/kubeconfig.go index 2f31dc19..22fdfd14 100644 --- a/pkg/controller/cluster/server/kubeconfig.go +++ b/pkg/controller/cluster/server/kubeconfig.go @@ -40,7 +40,7 @@ type content struct { Content string } -// GenerateNewKubeConfig generates the kubeconfig for the server: +// GenerateNewKubeConfig generates the kubeconfig for the cluster: // 1- use the server token to get the bootstrap data from k3s // 2- generate client admin cert/key // 3- use the ca cert from the bootstrap data & admin cert/key to write a new kubeconfig From e31ab4102cf77414758264474b202aceebdeade3 Mon Sep 17 00:00:00 2001 From: galal-hussein Date: Tue, 31 Jan 2023 02:52:13 +0200 Subject: [PATCH 2/2] Add serverArgs and agentArgs Signed-off-by: galal-hussein --- crd/cluster.yaml | 14 ++++++ .../multiple-servers.yaml | 9 +++- examples/single-server.yaml | 19 ++++++++ examples/single-server/single-server.yaml | 11 ----- pkg/apis/k3k.io/v1alpha1/types.go | 18 ++++--- .../k3k.io/v1alpha1/zz_generated.deepcopy.go | 10 ++++ pkg/controller/cluster/agent/agent.go | 10 ++-- pkg/controller/cluster/config/server.go | 48 +++++++++++-------- pkg/controller/cluster/controller.go | 15 ++++-- pkg/controller/cluster/server/ingress.go | 36 +------------- pkg/controller/cluster/server/server.go | 9 ++-- pkg/controller/util/util.go | 37 ++++++++++++++ 12 files changed, 153 insertions(+), 83 deletions(-) rename examples/{multiple-servers => }/multiple-servers.yaml (56%) create mode 100644 examples/single-server.yaml delete mode 100644 examples/single-server/single-server.yaml diff --git a/crd/cluster.yaml b/crd/cluster.yaml index c833b3df..c2f6a9ff 100644 --- a/crd/cluster.yaml +++ b/crd/cluster.yaml @@ -25,6 +25,20 @@ spec: type: integer token: type: string + clusterCIDR: + type: string + serviceCIDR: + type: string + clusterDNS: + type: string + serverArgs: + type: array + items: + type: string + agentArgs: + type: array + items: + type: string expose: type: object properties: diff --git a/examples/multiple-servers/multiple-servers.yaml b/examples/multiple-servers.yaml similarity index 56% rename from examples/multiple-servers/multiple-servers.yaml rename to examples/multiple-servers.yaml index d06bd971..11276e1d 100644 --- a/examples/multiple-servers/multiple-servers.yaml +++ b/examples/multiple-servers.yaml @@ -1,13 +1,18 @@ apiVersion: k3k.io/v1alpha1 kind: Cluster metadata: - name: multiple-servers + name: example1 namespace: default spec: - servers: 2 + servers: 1 agents: 3 token: test version: v1.26.0-k3s2 + clusterCIDR: 10.30.0.0/16 + serviceCIDR: 10.31.0.0/16 + clusterDNS: 10.30.0.10 + serverArgs: + - "--write-kubeconfig-mode=777" expose: ingress: enabled: true diff --git a/examples/single-server.yaml b/examples/single-server.yaml new file mode 100644 index 00000000..78c9a79e --- /dev/null +++ b/examples/single-server.yaml @@ -0,0 +1,19 @@ +apiVersion: k3k.io/v1alpha1 +kind: Cluster +metadata: + name: single-server + namespace: default +spec: + servers: 1 + agents: 3 + token: test + version: v1.26.0-k3s2 + clusterCIDR: 10.30.0.0/16 + serviceCIDR: 10.31.0.0/16 + clusterDNS: 10.30.0.10 + serverArgs: + - "--write-kubeconfig-mode=777" + expose: + ingress: + enabled: true + ingressClassName: "nginx" diff --git a/examples/single-server/single-server.yaml b/examples/single-server/single-server.yaml deleted file mode 100644 index 8e350c6f..00000000 --- a/examples/single-server/single-server.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: k3k.io/v1alpha1 -kind: Cluster -metadata: - name: single-server - namespace: default -spec: - servers: 1 - agents: 3 - token: test - version: v1.26.0-k3s2 - ingressClassName: traefik diff --git a/pkg/apis/k3k.io/v1alpha1/types.go b/pkg/apis/k3k.io/v1alpha1/types.go index 7a7aeed7..71002208 100644 --- a/pkg/apis/k3k.io/v1alpha1/types.go +++ b/pkg/apis/k3k.io/v1alpha1/types.go @@ -15,13 +15,19 @@ type Cluster struct { } type ClusterSpec struct { - Name string `json:"name"` - Version string `json:"version"` - Servers *int32 `json:"servers"` - Agents *int32 `json:"agents"` - Token string `json:"token"` + Name string `json:"name"` + Version string `json:"version"` + Servers *int32 `json:"servers"` + Agents *int32 `json:"agents"` + Token string `json:"token"` + ClusterCIDR string `json:"clusterCIDR,omitempty"` + ServiceCIDR string `json:"serviceCIDR,omitempty"` + ClusterDNS string `json:"clusterDNS,omitempty"` - Expose ExposeConfig `json:"expose"` + ServerArgs []string `json:"serverArgs,omitempty"` + AgentArgs []string `json:"agentArgs,omitempty"` + + Expose ExposeConfig `json:"expose,omitempty"` } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object diff --git a/pkg/apis/k3k.io/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/k3k.io/v1alpha1/zz_generated.deepcopy.go index c8555b09..3e8b5c99 100644 --- a/pkg/apis/k3k.io/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/k3k.io/v1alpha1/zz_generated.deepcopy.go @@ -82,6 +82,16 @@ func (in *ClusterSpec) DeepCopyInto(out *ClusterSpec) { *out = new(int32) **out = **in } + if in.ServerArgs != nil { + in, out := &in.ServerArgs, &out.ServerArgs + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.AgentArgs != nil { + in, out := &in.AgentArgs, &out.AgentArgs + *out = make([]string, len(*in)) + copy(*out, *in) + } in.Expose.DeepCopyInto(&out.Expose) return } diff --git a/pkg/controller/cluster/agent/agent.go b/pkg/controller/cluster/agent/agent.go index a805aba3..b7c15229 100644 --- a/pkg/controller/cluster/agent/agent.go +++ b/pkg/controller/cluster/agent/agent.go @@ -1,6 +1,8 @@ package agent import ( + "strings" + "github.com/galal-hussein/k3k/pkg/apis/k3k.io/v1alpha1" "github.com/galal-hussein/k3k/pkg/controller/util" apps "k8s.io/api/apps/v1" @@ -37,13 +39,13 @@ func Agent(cluster *v1alpha1.Cluster) *apps.Deployment { "type": "agent", }, }, - Spec: agentPodSpec(image, name), + Spec: agentPodSpec(image, name, cluster.Spec.AgentArgs), }, }, } } -func agentPodSpec(image, name string) v1.PodSpec { +func agentPodSpec(image, name string, args []string) v1.PodSpec { privileged := true return v1.PodSpec{ Volumes: []v1.Volume{ @@ -110,7 +112,9 @@ func agentPodSpec(image, name string) v1.PodSpec { }, Args: []string{ "-c", - "/bin/k3s agent --config /opt/rancher/k3s/config.yaml && true", + "/bin/k3s agent --config /opt/rancher/k3s/config.yaml " + + strings.Join(args, " ") + + " && true", }, VolumeMounts: []v1.VolumeMount{ { diff --git a/pkg/controller/cluster/config/server.go b/pkg/controller/cluster/config/server.go index d7055594..3ebe89fe 100644 --- a/pkg/controller/cluster/config/server.go +++ b/pkg/controller/cluster/config/server.go @@ -9,17 +9,17 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -func ServerConfig(cluster *v1alpha1.Cluster, init bool, serviceIP string) v1.Secret { +func ServerConfig(cluster *v1alpha1.Cluster, init bool, serviceIP string) (*v1.Secret, error) { name := "k3k-server-config" if init { name = "k3k-init-server-config" } - config := serverConfigData(serviceIP, cluster.Spec.Token) + config := serverConfigData(serviceIP, cluster) if init { - config = initConfigData(cluster.Spec.Token) + config = initConfigData(cluster) } - return v1.Secret{ + return &v1.Secret{ TypeMeta: metav1.TypeMeta{ Kind: "Secret", APIVersion: "v1", @@ -31,26 +31,36 @@ func ServerConfig(cluster *v1alpha1.Cluster, init bool, serviceIP string) v1.Sec Data: map[string][]byte{ "config.yaml": []byte(config), }, - } + }, nil } -func serverConfigData(serviceIP, token string) string { +func serverConfigData(serviceIP string, cluster *v1alpha1.Cluster) string { + opts := serverOptions(cluster) return fmt.Sprintf(`cluster-init: true server: https://%s:6443 -token: %s -cluster-cidr: 10.40.0.0/16 -service-cidr: 10.44.0.0/16 -cluster-dns: 10.44.0.10 -tls-san: -- 0.0.0.0`, serviceIP, token) +%s`, serviceIP, opts) } -func initConfigData(token string) string { +func initConfigData(cluster *v1alpha1.Cluster) string { + opts := serverOptions(cluster) return fmt.Sprintf(`cluster-init: true -token: %s -cluster-cidr: 10.40.0.0/16 -service-cidr: 10.44.0.0/16 -cluster-dns: 10.44.0.10 -tls-san: -- 0.0.0.0`, token) +%s`, opts) +} + +func serverOptions(cluster *v1alpha1.Cluster) string { + opts := "" + // TODO: generate token if not found + if cluster.Spec.Token != "" { + opts = fmt.Sprintf("token: %s\n", cluster.Spec.Token) + } + if cluster.Spec.ClusterCIDR != "" { + opts = fmt.Sprintf("%scluster-cidr: %s\n", opts, cluster.Spec.ClusterCIDR) + } + if cluster.Spec.ServiceCIDR != "" { + opts = fmt.Sprintf("%sservice-cidr: %s\n", opts, cluster.Spec.ServiceCIDR) + } + if cluster.Spec.ClusterDNS != "" { + opts = fmt.Sprintf("%scluster-dns: %s\n", opts, cluster.Spec.ClusterDNS) + } + return opts } diff --git a/pkg/controller/cluster/controller.go b/pkg/controller/cluster/controller.go index 1f7445ee..11ce47f3 100644 --- a/pkg/controller/cluster/controller.go +++ b/pkg/controller/cluster/controller.go @@ -119,16 +119,23 @@ func (r *ClusterReconciler) createCluster(ctx context.Context, cluster *v1alpha1 } // create init node config - initServerConfigMap := config.ServerConfig(cluster, true, service.Spec.ClusterIP) - if err := r.Client.Create(ctx, &initServerConfigMap); err != nil { + initServerConfigMap, err := config.ServerConfig(cluster, true, service.Spec.ClusterIP) + if err != nil { + return util.WrapErr("failed to get init server config", err) + } + if err := r.Client.Create(ctx, initServerConfigMap); err != nil { if !apierrors.IsAlreadyExists(err) { return util.WrapErr("failed to create init configmap", err) } } // create servers configuration - serverConfigMap := config.ServerConfig(cluster, false, service.Spec.ClusterIP) - if err := r.Client.Create(ctx, &serverConfigMap); err != nil { + serverConfigMap, err := config.ServerConfig(cluster, false, service.Spec.ClusterIP) + if err != nil { + return util.WrapErr("failed to get server config", err) + + } + if err := r.Client.Create(ctx, serverConfigMap); err != nil { if !apierrors.IsAlreadyExists(err) { return util.WrapErr("failed to create configmap", err) } diff --git a/pkg/controller/cluster/server/ingress.go b/pkg/controller/cluster/server/ingress.go index 27b0cfcd..064662b3 100644 --- a/pkg/controller/cluster/server/ingress.go +++ b/pkg/controller/cluster/server/ingress.go @@ -5,7 +5,6 @@ import ( "github.com/galal-hussein/k3k/pkg/apis/k3k.io/v1alpha1" "github.com/galal-hussein/k3k/pkg/controller/util" - v1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -20,7 +19,7 @@ const ( ) func Ingress(ctx context.Context, cluster *v1alpha1.Cluster, client client.Client) (*networkingv1.Ingress, error) { - addresses, err := addresses(ctx, client) + addresses, err := util.Addresses(ctx, client) if err != nil { return nil, err } @@ -46,39 +45,6 @@ func Ingress(ctx context.Context, cluster *v1alpha1.Cluster, client client.Clien return ingress, nil } -// return all the nodes external addresses, if not found then return internal addresses -func addresses(ctx context.Context, client client.Client) ([]string, error) { - addresses := []string{} - nodeList := v1.NodeList{} - if err := client.List(ctx, &nodeList); err != nil { - return nil, err - } - - for _, node := range nodeList.Items { - addresses = append(addresses, getNodeAddress(&node)) - } - - return addresses, nil -} - -func getNodeAddress(node *v1.Node) string { - externalIP := "" - internalIP := "" - for _, ip := range node.Status.Addresses { - if ip.Type == "ExternalIP" && ip.Address != "" { - externalIP = ip.Address - break - } else if ip.Type == "InternalIP" && ip.Address != "" { - internalIP = ip.Address - } - } - if externalIP != "" { - return externalIP - } - - return internalIP -} - func ingressRules(cluster *v1alpha1.Cluster, addresses []string) []networkingv1.IngressRule { ingressRules := []networkingv1.IngressRule{} pathTypePrefix := networkingv1.PathTypePrefix diff --git a/pkg/controller/cluster/server/server.go b/pkg/controller/cluster/server/server.go index 9b4c5e8e..eb14a9cb 100644 --- a/pkg/controller/cluster/server/server.go +++ b/pkg/controller/cluster/server/server.go @@ -2,6 +2,7 @@ package server import ( "strconv" + "strings" "github.com/galal-hussein/k3k/pkg/apis/k3k.io/v1alpha1" "github.com/galal-hussein/k3k/pkg/controller/util" @@ -49,13 +50,13 @@ func Server(cluster *v1alpha1.Cluster, init bool) *apps.Deployment { "init": strconv.FormatBool(init), }, }, - Spec: serverPodSpec(image, name), + Spec: serverPodSpec(image, name, cluster.Spec.ServerArgs), }, }, } } -func serverPodSpec(image, name string) v1.PodSpec { +func serverPodSpec(image, name string, args []string) v1.PodSpec { privileged := true return v1.PodSpec{ Volumes: []v1.Volume{ @@ -122,7 +123,9 @@ func serverPodSpec(image, name string) v1.PodSpec { }, Args: []string{ "-c", - "/bin/k3s server --config /opt/rancher/k3s/config.yaml && true", + "/bin/k3s server --config /opt/rancher/k3s/config.yaml " + + strings.Join(args, " ") + + " && true", }, VolumeMounts: []v1.VolumeMount{ { diff --git a/pkg/controller/util/util.go b/pkg/controller/util/util.go index 0fe8b2a1..d262ba3c 100644 --- a/pkg/controller/util/util.go +++ b/pkg/controller/util/util.go @@ -1,8 +1,12 @@ package util import ( + "context" + "github.com/galal-hussein/k3k/pkg/apis/k3k.io/v1alpha1" + v1 "k8s.io/api/core/v1" "k8s.io/klog" + "sigs.k8s.io/controller-runtime/pkg/client" ) const ( @@ -22,3 +26,36 @@ func WrapErr(errString string, err error) error { klog.Errorf("%s: %v", errString, err) return err } + +// return all the nodes external addresses, if not found then return internal addresses +func Addresses(ctx context.Context, client client.Client) ([]string, error) { + addresses := []string{} + nodeList := v1.NodeList{} + if err := client.List(ctx, &nodeList); err != nil { + return nil, err + } + + for _, node := range nodeList.Items { + addresses = append(addresses, getNodeAddress(&node)) + } + + return addresses, nil +} + +func getNodeAddress(node *v1.Node) string { + externalIP := "" + internalIP := "" + for _, ip := range node.Status.Addresses { + if ip.Type == "ExternalIP" && ip.Address != "" { + externalIP = ip.Address + break + } else if ip.Type == "InternalIP" && ip.Address != "" { + internalIP = ip.Address + } + } + if externalIP != "" { + return externalIP + } + + return internalIP +}