Fix and enhancements to IngressExposeConfig (annotations) (#248)

* ingress fixes

* added annotations to IngressConfig

* sync annotations with CR

* removed hosts

* small doc for ingress

charts/k3k/crds/k3k.io_clusters.yaml

@@ -111,8 +111,12 @@ spec:
                 properties:
                   ingress:
                     properties:
-                      enabled:
-                        type: boolean
+                      annotations:
+                        additionalProperties:
+                          type: string
+                        description: Annotations is a key value map that will enrich
+                          the Ingress annotations
+                        type: object
                       ingressClassName:
                         type: string
                     type: object

docs/advanced-usage.md

@@ -31,12 +31,17 @@ spec:
   mode: shared
   version: v1.31.3-k3s1
   servers: 3
+  tlsSANs:
+    - my-cluster.example.com
   nodeSelector:
     disktype: ssd
   expose:
     ingress:
-      enabled: true
       ingressClassName: nginx
+      annotations:
+        nginx.ingress.kubernetes.io/ssl-passthrough: "true"
+        nginx.ingress.kubernetes.io/backend-protocol: "true"
+        nginx.ingress.kubernetes.io/ssl-redirect: "HTTPS"
   clusterCIDR: 10.42.0.0/16
   serviceCIDR: 10.43.0.0/16
   clusterDNS: 10.43.0.10
@@ -84,6 +89,8 @@ The `expose` field contains options for exposing the API server of the virtual c
 
 You can use the `expose` field to enable exposure via `NodePort`, `LoadBalancer`, or `Ingress`.
 
+In this example we are exposing the Cluster with a Nginx ingress-controller, that has to be configured with the `--enable-ssl-passthrough` flag.
+
 
 ### `clusterCIDR`
 

docs/crds/crd-docs.md

@@ -163,7 +163,7 @@ _Appears in:_
 
 | Field | Description | Default | Validation |
 | --- | --- | --- | --- |
-| `enabled` _boolean_ |  |  |  |
+| `annotations` _object (keys:string, values:string)_ | Annotations is a key value map that will enrich the Ingress annotations |  |  |
 | `ingressClassName` _string_ |  |  |  |
 
 

pkg/apis/k3k.io/v1alpha1/types.go

@@ -151,8 +151,10 @@ type ExposeConfig struct {
 }
 
 type IngressConfig struct {
-	Enabled          bool   `json:"enabled,omitempty"`
-	IngressClassName string `json:"ingressClassName,omitempty"`
+	// Annotations is a key value map that will enrich the Ingress annotations
+	// +optional
+	Annotations      map[string]string `json:"annotations,omitempty"`
+	IngressClassName string            `json:"ingressClassName,omitempty"`
 }
 
 type LoadBalancerConfig struct {

pkg/controller/cluster/cluster.go

@@ -199,19 +199,8 @@ func (c *ClusterReconciler) reconcileCluster(ctx context.Context, cluster *v1alp
 		return err
 	}
 
-	if cluster.Spec.Expose != nil {
-		if cluster.Spec.Expose.Ingress != nil {
-			serverIngress, err := s.Ingress(ctx, c.Client)
-			if err != nil {
-				return err
-			}
-
-			if err := c.Client.Create(ctx, serverIngress); err != nil {
-				if !apierrors.IsAlreadyExists(err) {
-					return err
-				}
-			}
-		}
+	if err := c.ensureIngress(ctx, cluster); err != nil {
+		return err
 	}
 
 	if err := c.ensureBootstrapSecret(ctx, cluster, serviceIP, token); err != nil {
@@ -293,32 +282,62 @@ func (c *ClusterReconciler) ensureClusterService(ctx context.Context, cluster *v
 	log := ctrl.LoggerFrom(ctx)
 	log.Info("ensuring cluster service")
 
-	service := server.Service(cluster)
+	expectedService := server.Service(cluster)
 
-	createdClusterService := &v1.Service{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      service.Name,
-			Namespace: service.Namespace,
-		},
-	}
-	result, err := controllerutil.CreateOrUpdate(ctx, c.Client, createdClusterService, func() error {
-		if err := controllerutil.SetControllerReference(cluster, createdClusterService, c.Scheme); err != nil {
+	currentService := expectedService.DeepCopy()
+	result, err := controllerutil.CreateOrUpdate(ctx, c.Client, currentService, func() error {
+		if err := controllerutil.SetControllerReference(cluster, currentService, c.Scheme); err != nil {
 			return err
 		}
 
-		createdClusterService.Spec = service.Spec
+		currentService.Spec = expectedService.Spec
 		return nil
 	})
 	if err != nil {
 		return nil, err
 	}
 
-	key := client.ObjectKeyFromObject(createdClusterService)
+	key := client.ObjectKeyFromObject(currentService)
 	if result != controllerutil.OperationResultNone {
-		log.Info("ensuring cluster service", "key", key, "result", result)
+		log.Info("cluster service updated", "key", key, "result", result)
 	}
 
-	return createdClusterService, nil
+	return currentService, nil
+}
+
+func (c *ClusterReconciler) ensureIngress(ctx context.Context, cluster *v1alpha1.Cluster) error {
+	log := ctrl.LoggerFrom(ctx)
+	log.Info("ensuring cluster ingress")
+
+	expectedServerIngress := server.Ingress(ctx, cluster)
+
+	// delete existing Ingress if Expose or IngressConfig are nil
+	if cluster.Spec.Expose == nil || cluster.Spec.Expose.Ingress == nil {
+		err := c.Client.Delete(ctx, &expectedServerIngress)
+		return client.IgnoreNotFound(err)
+	}
+
+	currentServerIngress := expectedServerIngress.DeepCopy()
+	result, err := controllerutil.CreateOrUpdate(ctx, c.Client, currentServerIngress, func() error {
+		if err := controllerutil.SetControllerReference(cluster, currentServerIngress, c.Scheme); err != nil {
+			return err
+		}
+
+		currentServerIngress.Spec = expectedServerIngress.Spec
+		currentServerIngress.Annotations = expectedServerIngress.Annotations
+
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	key := client.ObjectKeyFromObject(currentServerIngress)
+	if result != controllerutil.OperationResultNone {
+		log.Info("cluster ingress updated", "key", key, "result", result)
+	}
+
+	return nil
 }
 
 func (c *ClusterReconciler) server(ctx context.Context, cluster *v1alpha1.Cluster, server *server.Server) error {

pkg/controller/cluster/server/ingress.go

@@ -3,94 +3,84 @@ package server
 import (
 	"context"
 
+	"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
 	"github.com/rancher/k3k/pkg/controller"
 	networkingv1 "k8s.io/api/networking/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"sigs.k8s.io/controller-runtime/pkg/client"
+	"k8s.io/utils/ptr"
 )
 
 const (
-	wildcardDNS = ".sslip.io"
-
-	nginxSSLPassthroughAnnotation  = "nginx.ingress.kubernetes.io/ssl-passthrough"
-	nginxBackendProtocolAnnotation = "nginx.ingress.kubernetes.io/backend-protocol"
-	nginxSSLRedirectAnnotation     = "nginx.ingress.kubernetes.io/ssl-redirect"
-
 	servicePort = 443
 	serverPort  = 6443
 	etcdPort    = 2379
 )
 
-func (s *Server) Ingress(ctx context.Context, client client.Client) (*networkingv1.Ingress, error) {
-	addresses, err := controller.Addresses(ctx, client)
-	if err != nil {
-		return nil, err
-	}
-	ingressRules := s.ingressRules(addresses)
-	ingress := &networkingv1.Ingress{
+func IngressName(clusterName string) string {
+	return controller.SafeConcatNameWithPrefix(clusterName, "ingress")
+}
+
+func Ingress(ctx context.Context, cluster *v1alpha1.Cluster) networkingv1.Ingress {
+	ingress := networkingv1.Ingress{
 		TypeMeta: metav1.TypeMeta{
 			Kind:       "Ingress",
 			APIVersion: "networking.k8s.io/v1",
 		},
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      controller.SafeConcatNameWithPrefix(s.cluster.Name, "ingress"),
-			Namespace: s.cluster.Namespace,
+			Name:      IngressName(cluster.Name),
+			Namespace: cluster.Namespace,
 		},
 		Spec: networkingv1.IngressSpec{
-			IngressClassName: &s.cluster.Spec.Expose.Ingress.IngressClassName,
-			Rules:            ingressRules,
+			Rules: ingressRules(cluster),
 		},
 	}
 
-	configureIngressOptions(ingress, s.cluster.Spec.Expose.Ingress.IngressClassName)
+	if cluster.Spec.Expose != nil && cluster.Spec.Expose.Ingress != nil {
+		ingressConfig := cluster.Spec.Expose.Ingress
 
-	return ingress, nil
+		if ingressConfig.IngressClassName != "" {
+			ingress.Spec.IngressClassName = ptr.To(ingressConfig.IngressClassName)
+		}
+
+		if ingressConfig.Annotations != nil {
+			ingress.Annotations = ingressConfig.Annotations
+		}
+	}
+
+	return ingress
 }
 
-func (s *Server) ingressRules(addresses []string) []networkingv1.IngressRule {
+func ingressRules(cluster *v1alpha1.Cluster) []networkingv1.IngressRule {
 	var ingressRules []networkingv1.IngressRule
-	pathTypePrefix := networkingv1.PathTypePrefix
-	for _, address := range addresses {
-		rule := networkingv1.IngressRule{
-			Host: s.cluster.Name + "." + address + wildcardDNS,
-			IngressRuleValue: networkingv1.IngressRuleValue{
-				HTTP: &networkingv1.HTTPIngressRuleValue{
-					Paths: []networkingv1.HTTPIngressPath{
-						{
-							Path:     "/",
-							PathType: &pathTypePrefix,
-							Backend: networkingv1.IngressBackend{
-								Service: &networkingv1.IngressServiceBackend{
-									Name: ServiceName(s.cluster.Name),
-									Port: networkingv1.ServiceBackendPort{
-										Number: serverPort,
-									},
-								},
-							},
-						},
-					},
+
+	if cluster.Spec.Expose == nil || cluster.Spec.Expose.Ingress == nil {
+		return ingressRules
+	}
+
+	path := networkingv1.HTTPIngressPath{
+		Path:     "/",
+		PathType: ptr.To(networkingv1.PathTypePrefix),
+		Backend: networkingv1.IngressBackend{
+			Service: &networkingv1.IngressServiceBackend{
+				Name: ServiceName(cluster.Name),
+				Port: networkingv1.ServiceBackendPort{
+					Number: serverPort,
 				},
 			},
-		}
-		ingressRules = append(ingressRules, rule)
+		},
+	}
+
+	hosts := cluster.Spec.TLSSANs
+	for _, host := range hosts {
+		ingressRules = append(ingressRules, networkingv1.IngressRule{
+			Host: host,
+			IngressRuleValue: networkingv1.IngressRuleValue{
+				HTTP: &networkingv1.HTTPIngressRuleValue{
+					Paths: []networkingv1.HTTPIngressPath{path},
+				},
+			},
+		})
 	}
 
 	return ingressRules
 }
-
-// configureIngressOptions will configure the ingress object by
-// adding tls passthrough capabilities and TLS needed annotations
-// it depends on the ingressclassname to configure each ingress
-// TODO: add treafik support through ingresstcproutes
-func configureIngressOptions(ingress *networkingv1.Ingress, ingressClassName string) {
-	// initial support for nginx ingress via annotations
-	if ingressClassName == "nginx" {
-		ingress.Annotations = make(map[string]string)
-		ingress.Annotations[nginxSSLPassthroughAnnotation] = "true"
-		ingress.Annotations[nginxSSLRedirectAnnotation] = "true"
-		ingress.Annotations[nginxBackendProtocolAnnotation] = "HTTPS"
-	}
-}
-func IngressName(clusterName string) string {
-	return controller.SafeConcatNameWithPrefix(clusterName, "ingress")
-}

pkg/controller/controller.go

@@ -1,16 +1,13 @@
 package controller
 
 import (
-	"context"
 	"crypto/sha256"
 	"encoding/hex"
 	"strings"
 	"time"
 
 	"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
-	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
-	ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 const (
@@ -42,41 +39,6 @@ func K3SImage(cluster *v1alpha1.Cluster) string {
 	return k3SImageName
 }
 
-func nodeAddress(node *v1.Node) string {
-	var externalIP string
-	var internalIP string
-
-	for _, ip := range node.Status.Addresses {
-		if ip.Type == "ExternalIP" && ip.Address != "" {
-			externalIP = ip.Address
-			break
-		}
-		if ip.Type == "InternalIP" && ip.Address != "" {
-			internalIP = ip.Address
-		}
-	}
-	if externalIP != "" {
-		return externalIP
-	}
-
-	return internalIP
-}
-
-// return all the nodes external addresses, if not found then return internal addresses
-func Addresses(ctx context.Context, client ctrlruntimeclient.Client) ([]string, error) {
-	var nodeList v1.NodeList
-	if err := client.List(ctx, &nodeList); err != nil {
-		return nil, err
-	}
-
-	addresses := make([]string, len(nodeList.Items))
-	for i, node := range nodeList.Items {
-		addresses[i] = nodeAddress(&node)
-	}
-
-	return addresses, nil
-}
-
 // SafeConcatNameWithPrefix runs the SafeConcatName with extra prefix.
 func SafeConcatNameWithPrefix(name ...string) string {
 	return SafeConcatName(append([]string{namePrefix}, name...)...)

pkg/controller/kubeconfig/kubeconfig.go

@@ -108,7 +108,7 @@ func getURLFromService(ctx context.Context, client client.Client, cluster *v1alp
 	}
 
 	expose := cluster.Spec.Expose
-	if expose != nil && expose.Ingress != nil && expose.Ingress.Enabled {
+	if expose != nil && expose.Ingress != nil {
 		var k3kIngress networkingv1.Ingress
 		ingressKey := types.NamespacedName{
 			Name:      server.IngressName(cluster.Name),