From 51a8fd8a8de537a281703fcac3ee6226920e2f5b Mon Sep 17 00:00:00 2001 From: Enrico Candino Date: Fri, 14 Feb 2025 12:38:42 +0100 Subject: [PATCH] Fix and enhancements to IngressExposeConfig (`annotations`) (#248) * ingress fixes * added annotations to IngressConfig * sync annotations with CR * removed hosts * small doc for ingress --- charts/k3k/crds/k3k.io_clusters.yaml | 8 +- docs/advanced-usage.md | 9 +- docs/crds/crd-docs.md | 2 +- pkg/apis/k3k.io/v1alpha1/types.go | 6 +- pkg/controller/cluster/cluster.go | 71 +++++++++------ pkg/controller/cluster/server/ingress.go | 110 +++++++++++------------ pkg/controller/controller.go | 38 -------- pkg/controller/kubeconfig/kubeconfig.go | 2 +- 8 files changed, 115 insertions(+), 131 deletions(-) diff --git a/charts/k3k/crds/k3k.io_clusters.yaml b/charts/k3k/crds/k3k.io_clusters.yaml index 08aa21a2..8a9e2d95 100644 --- a/charts/k3k/crds/k3k.io_clusters.yaml +++ b/charts/k3k/crds/k3k.io_clusters.yaml @@ -111,8 +111,12 @@ spec: properties: ingress: properties: - enabled: - type: boolean + annotations: + additionalProperties: + type: string + description: Annotations is a key value map that will enrich + the Ingress annotations + type: object ingressClassName: type: string type: object diff --git a/docs/advanced-usage.md b/docs/advanced-usage.md index 753cb5fa..f7e64a82 100644 --- a/docs/advanced-usage.md +++ b/docs/advanced-usage.md @@ -31,12 +31,17 @@ spec: mode: shared version: v1.31.3-k3s1 servers: 3 + tlsSANs: + - my-cluster.example.com nodeSelector: disktype: ssd expose: ingress: - enabled: true ingressClassName: nginx + annotations: + nginx.ingress.kubernetes.io/ssl-passthrough: "true" + nginx.ingress.kubernetes.io/backend-protocol: "true" + nginx.ingress.kubernetes.io/ssl-redirect: "HTTPS" clusterCIDR: 10.42.0.0/16 serviceCIDR: 10.43.0.0/16 clusterDNS: 10.43.0.10 @@ -84,6 +89,8 @@ The `expose` field contains options for exposing the API server of the virtual c You can use the `expose` field to enable exposure via `NodePort`, `LoadBalancer`, or `Ingress`. +In this example we are exposing the Cluster with a Nginx ingress-controller, that has to be configured with the `--enable-ssl-passthrough` flag. + ### `clusterCIDR` diff --git a/docs/crds/crd-docs.md b/docs/crds/crd-docs.md index e967e95a..7e8e428a 100644 --- a/docs/crds/crd-docs.md +++ b/docs/crds/crd-docs.md @@ -163,7 +163,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `enabled` _boolean_ | | | | +| `annotations` _object (keys:string, values:string)_ | Annotations is a key value map that will enrich the Ingress annotations | | | | `ingressClassName` _string_ | | | | diff --git a/pkg/apis/k3k.io/v1alpha1/types.go b/pkg/apis/k3k.io/v1alpha1/types.go index 3eac862f..3562e8b2 100644 --- a/pkg/apis/k3k.io/v1alpha1/types.go +++ b/pkg/apis/k3k.io/v1alpha1/types.go @@ -151,8 +151,10 @@ type ExposeConfig struct { } type IngressConfig struct { - Enabled bool `json:"enabled,omitempty"` - IngressClassName string `json:"ingressClassName,omitempty"` + // Annotations is a key value map that will enrich the Ingress annotations + // +optional + Annotations map[string]string `json:"annotations,omitempty"` + IngressClassName string `json:"ingressClassName,omitempty"` } type LoadBalancerConfig struct { diff --git a/pkg/controller/cluster/cluster.go b/pkg/controller/cluster/cluster.go index 088feab6..dcefbecc 100644 --- a/pkg/controller/cluster/cluster.go +++ b/pkg/controller/cluster/cluster.go @@ -199,19 +199,8 @@ func (c *ClusterReconciler) reconcileCluster(ctx context.Context, cluster *v1alp return err } - if cluster.Spec.Expose != nil { - if cluster.Spec.Expose.Ingress != nil { - serverIngress, err := s.Ingress(ctx, c.Client) - if err != nil { - return err - } - - if err := c.Client.Create(ctx, serverIngress); err != nil { - if !apierrors.IsAlreadyExists(err) { - return err - } - } - } + if err := c.ensureIngress(ctx, cluster); err != nil { + return err } if err := c.ensureBootstrapSecret(ctx, cluster, serviceIP, token); err != nil { @@ -293,32 +282,62 @@ func (c *ClusterReconciler) ensureClusterService(ctx context.Context, cluster *v log := ctrl.LoggerFrom(ctx) log.Info("ensuring cluster service") - service := server.Service(cluster) + expectedService := server.Service(cluster) - createdClusterService := &v1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: service.Name, - Namespace: service.Namespace, - }, - } - result, err := controllerutil.CreateOrUpdate(ctx, c.Client, createdClusterService, func() error { - if err := controllerutil.SetControllerReference(cluster, createdClusterService, c.Scheme); err != nil { + currentService := expectedService.DeepCopy() + result, err := controllerutil.CreateOrUpdate(ctx, c.Client, currentService, func() error { + if err := controllerutil.SetControllerReference(cluster, currentService, c.Scheme); err != nil { return err } - createdClusterService.Spec = service.Spec + currentService.Spec = expectedService.Spec return nil }) if err != nil { return nil, err } - key := client.ObjectKeyFromObject(createdClusterService) + key := client.ObjectKeyFromObject(currentService) if result != controllerutil.OperationResultNone { - log.Info("ensuring cluster service", "key", key, "result", result) + log.Info("cluster service updated", "key", key, "result", result) } - return createdClusterService, nil + return currentService, nil +} + +func (c *ClusterReconciler) ensureIngress(ctx context.Context, cluster *v1alpha1.Cluster) error { + log := ctrl.LoggerFrom(ctx) + log.Info("ensuring cluster ingress") + + expectedServerIngress := server.Ingress(ctx, cluster) + + // delete existing Ingress if Expose or IngressConfig are nil + if cluster.Spec.Expose == nil || cluster.Spec.Expose.Ingress == nil { + err := c.Client.Delete(ctx, &expectedServerIngress) + return client.IgnoreNotFound(err) + } + + currentServerIngress := expectedServerIngress.DeepCopy() + result, err := controllerutil.CreateOrUpdate(ctx, c.Client, currentServerIngress, func() error { + if err := controllerutil.SetControllerReference(cluster, currentServerIngress, c.Scheme); err != nil { + return err + } + + currentServerIngress.Spec = expectedServerIngress.Spec + currentServerIngress.Annotations = expectedServerIngress.Annotations + + return nil + }) + if err != nil { + return err + } + + key := client.ObjectKeyFromObject(currentServerIngress) + if result != controllerutil.OperationResultNone { + log.Info("cluster ingress updated", "key", key, "result", result) + } + + return nil } func (c *ClusterReconciler) server(ctx context.Context, cluster *v1alpha1.Cluster, server *server.Server) error { diff --git a/pkg/controller/cluster/server/ingress.go b/pkg/controller/cluster/server/ingress.go index 39c5dd54..d26303d2 100644 --- a/pkg/controller/cluster/server/ingress.go +++ b/pkg/controller/cluster/server/ingress.go @@ -3,94 +3,84 @@ package server import ( "context" + "github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1" "github.com/rancher/k3k/pkg/controller" networkingv1 "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/controller-runtime/pkg/client" + "k8s.io/utils/ptr" ) const ( - wildcardDNS = ".sslip.io" - - nginxSSLPassthroughAnnotation = "nginx.ingress.kubernetes.io/ssl-passthrough" - nginxBackendProtocolAnnotation = "nginx.ingress.kubernetes.io/backend-protocol" - nginxSSLRedirectAnnotation = "nginx.ingress.kubernetes.io/ssl-redirect" - servicePort = 443 serverPort = 6443 etcdPort = 2379 ) -func (s *Server) Ingress(ctx context.Context, client client.Client) (*networkingv1.Ingress, error) { - addresses, err := controller.Addresses(ctx, client) - if err != nil { - return nil, err - } - ingressRules := s.ingressRules(addresses) - ingress := &networkingv1.Ingress{ +func IngressName(clusterName string) string { + return controller.SafeConcatNameWithPrefix(clusterName, "ingress") +} + +func Ingress(ctx context.Context, cluster *v1alpha1.Cluster) networkingv1.Ingress { + ingress := networkingv1.Ingress{ TypeMeta: metav1.TypeMeta{ Kind: "Ingress", APIVersion: "networking.k8s.io/v1", }, ObjectMeta: metav1.ObjectMeta{ - Name: controller.SafeConcatNameWithPrefix(s.cluster.Name, "ingress"), - Namespace: s.cluster.Namespace, + Name: IngressName(cluster.Name), + Namespace: cluster.Namespace, }, Spec: networkingv1.IngressSpec{ - IngressClassName: &s.cluster.Spec.Expose.Ingress.IngressClassName, - Rules: ingressRules, + Rules: ingressRules(cluster), }, } - configureIngressOptions(ingress, s.cluster.Spec.Expose.Ingress.IngressClassName) + if cluster.Spec.Expose != nil && cluster.Spec.Expose.Ingress != nil { + ingressConfig := cluster.Spec.Expose.Ingress - return ingress, nil + if ingressConfig.IngressClassName != "" { + ingress.Spec.IngressClassName = ptr.To(ingressConfig.IngressClassName) + } + + if ingressConfig.Annotations != nil { + ingress.Annotations = ingressConfig.Annotations + } + } + + return ingress } -func (s *Server) ingressRules(addresses []string) []networkingv1.IngressRule { +func ingressRules(cluster *v1alpha1.Cluster) []networkingv1.IngressRule { var ingressRules []networkingv1.IngressRule - pathTypePrefix := networkingv1.PathTypePrefix - for _, address := range addresses { - rule := networkingv1.IngressRule{ - Host: s.cluster.Name + "." + address + wildcardDNS, - IngressRuleValue: networkingv1.IngressRuleValue{ - HTTP: &networkingv1.HTTPIngressRuleValue{ - Paths: []networkingv1.HTTPIngressPath{ - { - Path: "/", - PathType: &pathTypePrefix, - Backend: networkingv1.IngressBackend{ - Service: &networkingv1.IngressServiceBackend{ - Name: ServiceName(s.cluster.Name), - Port: networkingv1.ServiceBackendPort{ - Number: serverPort, - }, - }, - }, - }, - }, + + if cluster.Spec.Expose == nil || cluster.Spec.Expose.Ingress == nil { + return ingressRules + } + + path := networkingv1.HTTPIngressPath{ + Path: "/", + PathType: ptr.To(networkingv1.PathTypePrefix), + Backend: networkingv1.IngressBackend{ + Service: &networkingv1.IngressServiceBackend{ + Name: ServiceName(cluster.Name), + Port: networkingv1.ServiceBackendPort{ + Number: serverPort, }, }, - } - ingressRules = append(ingressRules, rule) + }, + } + + hosts := cluster.Spec.TLSSANs + for _, host := range hosts { + ingressRules = append(ingressRules, networkingv1.IngressRule{ + Host: host, + IngressRuleValue: networkingv1.IngressRuleValue{ + HTTP: &networkingv1.HTTPIngressRuleValue{ + Paths: []networkingv1.HTTPIngressPath{path}, + }, + }, + }) } return ingressRules } - -// configureIngressOptions will configure the ingress object by -// adding tls passthrough capabilities and TLS needed annotations -// it depends on the ingressclassname to configure each ingress -// TODO: add treafik support through ingresstcproutes -func configureIngressOptions(ingress *networkingv1.Ingress, ingressClassName string) { - // initial support for nginx ingress via annotations - if ingressClassName == "nginx" { - ingress.Annotations = make(map[string]string) - ingress.Annotations[nginxSSLPassthroughAnnotation] = "true" - ingress.Annotations[nginxSSLRedirectAnnotation] = "true" - ingress.Annotations[nginxBackendProtocolAnnotation] = "HTTPS" - } -} -func IngressName(clusterName string) string { - return controller.SafeConcatNameWithPrefix(clusterName, "ingress") -} diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 357028df..5bee65a9 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -1,16 +1,13 @@ package controller import ( - "context" "crypto/sha256" "encoding/hex" "strings" "time" "github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1" - v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/wait" - ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" ) const ( @@ -42,41 +39,6 @@ func K3SImage(cluster *v1alpha1.Cluster) string { return k3SImageName } -func nodeAddress(node *v1.Node) string { - var externalIP string - var internalIP string - - for _, ip := range node.Status.Addresses { - if ip.Type == "ExternalIP" && ip.Address != "" { - externalIP = ip.Address - break - } - if ip.Type == "InternalIP" && ip.Address != "" { - internalIP = ip.Address - } - } - if externalIP != "" { - return externalIP - } - - return internalIP -} - -// return all the nodes external addresses, if not found then return internal addresses -func Addresses(ctx context.Context, client ctrlruntimeclient.Client) ([]string, error) { - var nodeList v1.NodeList - if err := client.List(ctx, &nodeList); err != nil { - return nil, err - } - - addresses := make([]string, len(nodeList.Items)) - for i, node := range nodeList.Items { - addresses[i] = nodeAddress(&node) - } - - return addresses, nil -} - // SafeConcatNameWithPrefix runs the SafeConcatName with extra prefix. func SafeConcatNameWithPrefix(name ...string) string { return SafeConcatName(append([]string{namePrefix}, name...)...) diff --git a/pkg/controller/kubeconfig/kubeconfig.go b/pkg/controller/kubeconfig/kubeconfig.go index 48889ecb..baad8bbe 100644 --- a/pkg/controller/kubeconfig/kubeconfig.go +++ b/pkg/controller/kubeconfig/kubeconfig.go @@ -108,7 +108,7 @@ func getURLFromService(ctx context.Context, client client.Client, cluster *v1alp } expose := cluster.Spec.Expose - if expose != nil && expose.Ingress != nil && expose.Ingress.Enabled { + if expose != nil && expose.Ingress != nil { var k3kIngress networkingv1.Ingress ingressKey := types.NamespacedName{ Name: server.IngressName(cluster.Name),