From bd9b82afb4d0c9b095f90e5f163cb4b7efee1304 Mon Sep 17 00:00:00 2001 From: Enrico Candino Date: Mon, 17 Aug 2026 10:30:08 +0200 Subject: [PATCH] feat: enhance ingress exposure validation and add tests for tlsSANs requirements (#1147) --- docs/advanced-usage.md | 72 +++++++- pkg/controller/cluster/cluster.go | 34 +++- pkg/controller/cluster/cluster_test.go | 128 ++++++++++++++ pkg/controller/cluster/server/ingress.go | 8 +- pkg/controller/cluster/server/ingress_test.go | 161 ++++++++++++++++++ pkg/controller/controller.go | 9 + pkg/controller/controller_test.go | 41 +++++ tests/integration/cluster/cluster_test.go | 95 +++++++++++ 8 files changed, 534 insertions(+), 14 deletions(-) create mode 100644 pkg/controller/cluster/cluster_test.go create mode 100644 pkg/controller/cluster/server/ingress_test.go diff --git a/docs/advanced-usage.md b/docs/advanced-usage.md index 43d828d3..b4b0ac85 100644 --- a/docs/advanced-usage.md +++ b/docs/advanced-usage.md @@ -89,7 +89,77 @@ The `expose` field contains options for exposing the API server of the virtual c You can use the `expose` field to enable exposure via `NodePort`, `LoadBalancer`, or `Ingress`. -In this example we are exposing the Cluster with a Nginx ingress-controller, that has to be configured with the `--enable-ssl-passthrough` flag. +#### TLS passthrough is required + +The K3s API server authenticates clients with mTLS certificates, so **the TLS connection must reach +the API server untouched**. An ingress controller that terminates TLS will break both `kubectl` and +agent authentication. Any controller used with `expose.ingress` must therefore support TLS +passthrough. + +`expose.ingress` also requires at least one **DNS name** in `spec.tlsSANs`: the SANs are used as the +Ingress hosts, and the Ingress API does not accept IP addresses there. IPs in `tlsSANs` are ignored +when building the Ingress, and a cluster with no DNS SAN stays in the `Pending` phase with a +`ValidationFailed` condition rather than generating an invalid Ingress. + +#### Nginx + +Supported directly through `expose.ingress`. The ingress controller has to be started with the +`--enable-ssl-passthrough` flag, and the annotations enabling passthrough must be set on the +Ingress: + +```yaml +spec: + tlsSANs: + - my-cluster.example.com + expose: + ingress: + ingressClassName: nginx + annotations: + nginx.ingress.kubernetes.io/ssl-passthrough: "true" + nginx.ingress.kubernetes.io/backend-protocol: "HTTPS" + nginx.ingress.kubernetes.io/ssl-redirect: "HTTPS" +``` + +#### Traefik + +Traefik **cannot** perform layer 4 TLS passthrough with a standard `Ingress`, so `expose.ingress` +does not work with it. This matters on K3s and RKE2, where Traefik is the default ingress +controller. + +Use `expose.loadBalancer` or `expose.nodePort` instead, or leave `expose` unset and create a Traefik +`IngressRouteTCP` pointing at the cluster's `ClusterIP` service: + +```yaml +apiVersion: traefik.io/v1alpha1 +kind: IngressRouteTCP +metadata: + name: my-virtual-cluster + namespace: my-namespace +spec: + entryPoints: + - websecure + routes: + - match: HostSNI(`my-cluster.example.com`) + services: + - name: k3k-my-virtual-cluster-service # k3k--service + port: 443 + tls: + passthrough: true +``` + +Add `my-cluster.example.com` to `spec.tlsSANs` so the API server certificate covers it, and generate +the kubeconfig with the matching endpoint: + +```bash +k3kcli kubeconfig generate --namespace my-namespace --name my-virtual-cluster \ + --kubeconfig-server https://my-cluster.example.com +``` + +**Limitation in `hcp` mode:** when the routing resource is managed outside of K3k, K3k does not know +the external endpoint. In `hcp` mode it owns the `default/kubernetes` Endpoints inside the virtual +cluster so that pods on external worker nodes can reach the API server, and without an `expose` +configuration it can only point them at the host cluster's `ClusterIP`, which external nodes cannot +route to. For `hcp` clusters with external workers, use `expose.nodePort` or `expose.loadBalancer`. ### `clusterCIDR` diff --git a/pkg/controller/cluster/cluster.go b/pkg/controller/cluster/cluster.go index f3679806..361f2e08 100644 --- a/pkg/controller/cluster/cluster.go +++ b/pkg/controller/cluster/cluster.go @@ -346,17 +346,20 @@ func (c *ClusterReconciler) reconcile(ctx context.Context, cluster *v1beta1.Clus policyName, found := ns.Labels[policy.PolicyNameLabelKey] cluster.Status.PolicyName = policyName - if found && policyName != "" { - var policy v1beta1.VirtualClusterPolicy - if err := c.Client.Get(ctx, client.ObjectKey{Name: policyName}, &policy); err != nil { - return err - } + // vcp is nil when the namespace is not bound to any VirtualClusterPolicy. + var vcp *v1beta1.VirtualClusterPolicy - if err := c.validate(cluster, policy); err != nil { + if found && policyName != "" { + vcp = &v1beta1.VirtualClusterPolicy{} + if err := c.Client.Get(ctx, client.ObjectKey{Name: policyName}, vcp); err != nil { return err } } + if err := c.validate(cluster, vcp); err != nil { + return err + } + // if the Version is not specified we will try to use the same Kubernetes version of the host. // This version is stored in the Status object, and it will not be updated if already set. if cluster.Status.HostVersion == "" { @@ -730,7 +733,7 @@ func (c *ClusterReconciler) ensureIngress(ctx context.Context, cluster *v1beta1. log := ctrl.LoggerFrom(ctx) log.V(1).Info("Ensuring cluster ingress") - expectedServerIngress := server.Ingress(ctx, cluster) + expectedServerIngress := server.Ingress(cluster) // delete existing Ingress if Expose or IngressConfig are nil if cluster.Spec.Expose == nil || cluster.Spec.Expose.Ingress == nil { @@ -983,12 +986,15 @@ func (c *ClusterReconciler) ensureAgent(ctx context.Context, cluster *v1beta1.Cl return agentEnsurer.EnsureResources(ctx) } -func (c *ClusterReconciler) validate(cluster *v1beta1.Cluster, policy v1beta1.VirtualClusterPolicy) error { +// validate validates a Cluster before reconciling it. The policy is nil when the namespace +// of the Cluster is not bound to any VirtualClusterPolicy: only the checks that depend on it +// are skipped in that case. +func (c *ClusterReconciler) validate(cluster *v1beta1.Cluster, policy *v1beta1.VirtualClusterPolicy) error { if cluster.Name == ClusterInvalidName { return fmt.Errorf("%w: invalid cluster name %q", ErrClusterValidation, cluster.Name) } - if cluster.Spec.Mode != policy.Spec.AllowedMode { + if policy != nil && cluster.Spec.Mode != policy.Spec.AllowedMode { return fmt.Errorf("%w: mode %q is not allowed by the policy %q", ErrClusterValidation, cluster.Spec.Mode, policy.Name) } @@ -998,6 +1004,16 @@ func (c *ClusterReconciler) validate(cluster *v1beta1.Cluster, policy v1beta1.Vi } } + // The Ingress hosts are taken from the tlsSANs, and the IP addresses are not valid. + // Without at least one DNS name the generated Ingress would have no rules, + // and the API server would reject it. + if cluster.Spec.Expose != nil && cluster.Spec.Expose.Ingress != nil { + hosts := controller.FilterDNSNames(cluster.Spec.TLSSANs) + if len(hosts) == 0 { + return fmt.Errorf("%w: expose.ingress requires at least one DNS name in spec.tlsSANs to use as the ingress host", ErrClusterValidation) + } + } + return nil } diff --git a/pkg/controller/cluster/cluster_test.go b/pkg/controller/cluster/cluster_test.go new file mode 100644 index 00000000..fb8e1cce --- /dev/null +++ b/pkg/controller/cluster/cluster_test.go @@ -0,0 +1,128 @@ +package cluster + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/assert" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1" +) + +func Test_validate(t *testing.T) { + tests := []struct { + name string + clusterName string + mode v1beta1.ClusterMode + tlsSANs []string + expose *v1beta1.ExposeConfig + policy *v1beta1.VirtualClusterPolicy + wantErr string + }{ + { + name: "valid cluster without a policy", + }, + { + name: "valid cluster with a policy", + policy: newTestPolicy(v1beta1.SharedClusterMode), + }, + { + // the name check does not depend on the policy, so it also runs + // in namespaces that are not bound to one + name: "invalid cluster name without a policy", + clusterName: ClusterInvalidName, + wantErr: "invalid cluster name", + }, + { + name: "invalid cluster name with a policy", + clusterName: ClusterInvalidName, + policy: newTestPolicy(v1beta1.SharedClusterMode), + wantErr: "invalid cluster name", + }, + { + name: "mode not allowed by the policy", + mode: v1beta1.VirtualClusterMode, + policy: newTestPolicy(v1beta1.SharedClusterMode), + wantErr: "is not allowed by the policy", + }, + { + // without a policy there is no allowed mode to check against + name: "any mode is allowed without a policy", + mode: v1beta1.VirtualClusterMode, + }, + { + name: "expose without ingress", + expose: &v1beta1.ExposeConfig{NodePort: &v1beta1.NodePortConfig{}}, + }, + { + name: "expose ingress without tlsSANs", + expose: &v1beta1.ExposeConfig{Ingress: &v1beta1.IngressConfig{}}, + wantErr: "spec.tlsSANs", + }, + { + name: "expose ingress with only IP tlsSANs", + tlsSANs: []string{"10.0.0.5", "::1"}, + expose: &v1beta1.ExposeConfig{Ingress: &v1beta1.IngressConfig{}}, + wantErr: "spec.tlsSANs", + }, + { + name: "expose ingress with a DNS tlsSAN", + tlsSANs: []string{"10.0.0.5", "my-cluster.example.com"}, + expose: &v1beta1.ExposeConfig{Ingress: &v1beta1.IngressConfig{}}, + }, + { + name: "no ingress, IP-only tlsSANs is fine", + tlsSANs: []string{"10.0.0.5"}, + expose: &v1beta1.ExposeConfig{LoadBalancer: &v1beta1.LoadBalancerConfig{}}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + clusterName := tt.clusterName + if clusterName == "" { + clusterName = "test-cluster" + } + + mode := tt.mode + if mode == "" { + mode = v1beta1.SharedClusterMode + } + + cluster := &v1beta1.Cluster{ + ObjectMeta: metav1.ObjectMeta{Name: clusterName, Namespace: "test-namespace"}, + Spec: v1beta1.ClusterSpec{ + Mode: mode, + TLSSANs: tt.tlsSANs, + Expose: tt.expose, + }, + } + + // the Client is only needed to validate the customCAs secrets, + // which none of these clusters enable + reconciler := &ClusterReconciler{} + + err := reconciler.validate(cluster, tt.policy) + + if tt.wantErr == "" { + assert.NoError(t, err) + return + } + + assert.Error(t, err) + // the status controller relies on this to report Pending/ValidationFailed + // instead of letting the API server reject an invalid resource. + assert.True(t, errors.Is(err, ErrClusterValidation)) + assert.Contains(t, err.Error(), tt.wantErr) + }) + } +} + +func newTestPolicy(allowedMode v1beta1.ClusterMode) *v1beta1.VirtualClusterPolicy { + return &v1beta1.VirtualClusterPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: "test-policy"}, + Spec: v1beta1.VirtualClusterPolicySpec{AllowedMode: allowedMode}, + } +} diff --git a/pkg/controller/cluster/server/ingress.go b/pkg/controller/cluster/server/ingress.go index 36a2d90e..485ef283 100644 --- a/pkg/controller/cluster/server/ingress.go +++ b/pkg/controller/cluster/server/ingress.go @@ -1,8 +1,6 @@ package server import ( - "context" - "k8s.io/utils/ptr" networkingv1 "k8s.io/api/networking/v1" @@ -22,7 +20,7 @@ func IngressName(clusterName string) string { return controller.SafeConcatNameWithPrefix(clusterName, "ingress") } -func Ingress(ctx context.Context, cluster *v1beta1.Cluster) networkingv1.Ingress { +func Ingress(cluster *v1beta1.Cluster) networkingv1.Ingress { ingress := networkingv1.Ingress{ TypeMeta: metav1.TypeMeta{ Kind: "Ingress", @@ -72,7 +70,9 @@ func ingressRules(cluster *v1beta1.Cluster) []networkingv1.IngressRule { }, } - hosts := cluster.Spec.TLSSANs + // the IP addresses of the tlsSANs are skipped: the Ingress API only accepts DNS names + hosts := controller.FilterDNSNames(cluster.Spec.TLSSANs) + for _, host := range hosts { ingressRules = append(ingressRules, networkingv1.IngressRule{ Host: host, diff --git a/pkg/controller/cluster/server/ingress_test.go b/pkg/controller/cluster/server/ingress_test.go new file mode 100644 index 00000000..146a27d6 --- /dev/null +++ b/pkg/controller/cluster/server/ingress_test.go @@ -0,0 +1,161 @@ +package server + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "k8s.io/utils/ptr" + + networkingv1 "k8s.io/api/networking/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1" +) + +func TestIngress(t *testing.T) { + tests := map[string]struct { + clusterOpts []func(*v1beta1.Cluster) + ingressOpts []func(*networkingv1.Ingress) + }{ + "no expose": {}, + "expose ingress without tlsSANs has no rules": { + clusterOpts: []func(*v1beta1.Cluster){ + func(c *v1beta1.Cluster) { + c.Spec.Expose = &v1beta1.ExposeConfig{ + Ingress: &v1beta1.IngressConfig{}, + } + }, + }, + }, + "expose ingress with only IP tlsSANs has no rules": { + clusterOpts: []func(*v1beta1.Cluster){ + func(c *v1beta1.Cluster) { + c.Spec.TLSSANs = []string{"10.0.0.5", "::1"} + c.Spec.Expose = &v1beta1.ExposeConfig{ + Ingress: &v1beta1.IngressConfig{}, + } + }, + }, + }, + "expose ingress with a wildcard tlsSAN": { + clusterOpts: []func(*v1beta1.Cluster){ + func(c *v1beta1.Cluster) { + c.Spec.TLSSANs = []string{"*.example.com"} + c.Spec.Expose = &v1beta1.ExposeConfig{ + Ingress: &v1beta1.IngressConfig{}, + } + }, + }, + ingressOpts: []func(*networkingv1.Ingress){ + func(i *networkingv1.Ingress) { + i.Spec.Rules = []networkingv1.IngressRule{testIngressRule("*.example.com")} + }, + }, + }, + "expose ingress skips IP tlsSANs": { + clusterOpts: []func(*v1beta1.Cluster){ + func(c *v1beta1.Cluster) { + c.Spec.TLSSANs = []string{"10.0.0.5", "my-cluster.example.com"} + c.Spec.Expose = &v1beta1.ExposeConfig{ + Ingress: &v1beta1.IngressConfig{}, + } + }, + }, + ingressOpts: []func(*networkingv1.Ingress){ + func(i *networkingv1.Ingress) { + i.Spec.Rules = []networkingv1.IngressRule{testIngressRule("my-cluster.example.com")} + }, + }, + }, + "expose ingress with multiple hosts": { + clusterOpts: []func(*v1beta1.Cluster){ + func(c *v1beta1.Cluster) { + c.Spec.TLSSANs = []string{"my-cluster.example.com", "other.example.com"} + c.Spec.Expose = &v1beta1.ExposeConfig{ + Ingress: &v1beta1.IngressConfig{}, + } + }, + }, + ingressOpts: []func(*networkingv1.Ingress){ + func(i *networkingv1.Ingress) { + i.Spec.Rules = []networkingv1.IngressRule{ + testIngressRule("my-cluster.example.com"), + testIngressRule("other.example.com"), + } + }, + }, + }, + "expose ingress with class and annotations": { + clusterOpts: []func(*v1beta1.Cluster){ + func(c *v1beta1.Cluster) { + c.Spec.TLSSANs = []string{"my-cluster.example.com"} + c.Spec.Expose = &v1beta1.ExposeConfig{ + Ingress: &v1beta1.IngressConfig{ + IngressClassName: "nginx", + Annotations: map[string]string{ + "nginx.ingress.kubernetes.io/ssl-passthrough": "true", + }, + }, + } + }, + }, + ingressOpts: []func(*networkingv1.Ingress){ + func(i *networkingv1.Ingress) { + i.Annotations = map[string]string{ + "nginx.ingress.kubernetes.io/ssl-passthrough": "true", + } + i.Spec.IngressClassName = ptr.To("nginx") + i.Spec.Rules = []networkingv1.IngressRule{testIngressRule("my-cluster.example.com")} + }, + }, + }, + } + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + cluster := newTestCluster(tt.clusterOpts...) + want := newTestIngress(cluster, tt.ingressOpts...) + assert.Equal(t, *want, Ingress(cluster)) + }) + } +} + +func newTestIngress(cluster *v1beta1.Cluster, opts ...func(*networkingv1.Ingress)) *networkingv1.Ingress { + ingress := &networkingv1.Ingress{ + TypeMeta: metav1.TypeMeta{ + Kind: "Ingress", + APIVersion: "networking.k8s.io/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "k3k-test-cluster-ingress", + Namespace: cluster.Namespace, + }, + } + for _, opt := range opts { + opt(ingress) + } + + return ingress +} + +func testIngressRule(host string) networkingv1.IngressRule { + return networkingv1.IngressRule{ + Host: host, + IngressRuleValue: networkingv1.IngressRuleValue{ + HTTP: &networkingv1.HTTPIngressRuleValue{ + Paths: []networkingv1.HTTPIngressPath{ + { + Path: "/", + PathType: ptr.To(networkingv1.PathTypePrefix), + Backend: networkingv1.IngressBackend{ + Service: &networkingv1.IngressServiceBackend{ + Name: "k3k-test-cluster-service", + Port: networkingv1.ServiceBackendPort{Number: 443}, + }, + }, + }, + }, + }, + }, + } +} diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 0b1da271..cf560e1e 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -3,6 +3,7 @@ package controller import ( "crypto/sha256" "encoding/hex" + "net" "slices" "strings" "time" @@ -45,6 +46,14 @@ func K3SVersion(cluster *v1beta1.Cluster) string { return "latest" } +// FilterDNSNames returns only the DNS names of the given list, dropping the IP addresses. +// It is useful for the fields that cannot hold an IP, like the hosts of an Ingress. +func FilterDNSNames(names []string) []string { + return slices.DeleteFunc(slices.Clone(names), func(name string) bool { + return net.ParseIP(name) != nil + }) +} + // SafeConcatNameWithPrefix runs the SafeConcatName with extra prefix. func SafeConcatNameWithPrefix(name ...string) string { return SafeConcatName(append([]string{namePrefix}, name...)...) diff --git a/pkg/controller/controller_test.go b/pkg/controller/controller_test.go index 736301e5..a4418021 100644 --- a/pkg/controller/controller_test.go +++ b/pkg/controller/controller_test.go @@ -1,6 +1,7 @@ package controller import ( + "slices" "testing" "github.com/stretchr/testify/assert" @@ -10,6 +11,46 @@ import ( "github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1" ) +func Test_FilterDNSNames(t *testing.T) { + tests := map[string]struct { + names []string + expected []string + }{ + "no names": { + names: nil, + expected: nil, + }, + "only IPs": { + names: []string{"10.0.0.5", "192.168.1.1", "::1"}, + expected: []string{}, + }, + "only DNS names": { + names: []string{"my-cluster.example.com", "other.example.com"}, + expected: []string{"my-cluster.example.com", "other.example.com"}, + }, + "mixed IPs and DNS names keeps the order": { + names: []string{"10.0.0.5", "my-cluster.example.com", "fd00::1", "other.example.com"}, + expected: []string{"my-cluster.example.com", "other.example.com"}, + }, + "wildcards are kept": { + names: []string{"*.example.com"}, + expected: []string{"*.example.com"}, + }, + } + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + original := slices.Clone(tt.names) + + filtered := FilterDNSNames(tt.names) + assert.Equal(t, tt.expected, filtered) + + // the given slice should be left untouched + assert.Equal(t, original, tt.names) + }) + } +} + func Test_K3S_Image(t *testing.T) { type args struct { cluster *v1beta1.Cluster diff --git a/tests/integration/cluster/cluster_test.go b/tests/integration/cluster/cluster_test.go index a8eb0222..a1dc2631 100644 --- a/tests/integration/cluster/cluster_test.go +++ b/tests/integration/cluster/cluster_test.go @@ -5,6 +5,7 @@ import ( "strings" "time" + "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" @@ -12,6 +13,7 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/rancher/k3k/k3k-kubelet/translate" @@ -337,6 +339,99 @@ var _ = Describe("Cluster Controller", Label("controller"), Label("Cluster"), fu }) }) + When("exposing the cluster with ingress", func() { + It("will not be provisioned without a DNS name in the tlsSANs", func() { + cluster := &v1beta1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "cluster-", + Namespace: namespace, + }, + Spec: v1beta1.ClusterSpec{ + Expose: &v1beta1.ExposeConfig{ + Ingress: &v1beta1.IngressConfig{}, + }, + }, + } + + Expect(k8sClient.Create(ctx, cluster)).To(Succeed()) + + // the Ingress hosts come from the tlsSANs: without one the generated + // Ingress would have no rules and be rejected by the API server, so + // the cluster should stay Pending with a validation error instead. + Eventually(func(g Gomega) { + err := k8sClient.Get(ctx, client.ObjectKeyFromObject(cluster), cluster) + g.Expect(err).To(Not(HaveOccurred())) + + g.Expect(cluster.Status.Phase).To(Equal(v1beta1.ClusterPending)) + + readyCondition := meta.FindStatusCondition(cluster.Status.Conditions, "Ready") + g.Expect(readyCondition).To(Not(BeNil())) + g.Expect(readyCondition.Status).To(Equal(metav1.ConditionFalse)) + g.Expect(readyCondition.Reason).To(Equal("ValidationFailed")) + g.Expect(readyCondition.Message).To(ContainSubstring("spec.tlsSANs")) + }). + WithTimeout(time.Second * 30). + WithPolling(time.Second). + Should(Succeed()) + + // no invalid Ingress should have been submitted + ingressKey := client.ObjectKey{ + Name: server.IngressName(cluster.Name), + Namespace: cluster.Namespace, + } + + var ingress networkingv1.Ingress + Expect(apierrors.IsNotFound(k8sClient.Get(ctx, ingressKey, &ingress))).To(BeTrue()) + }) + + It("will create an Ingress with only the DNS names from the tlsSANs", func() { + cluster := &v1beta1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "cluster-", + Namespace: namespace, + }, + Spec: v1beta1.ClusterSpec{ + // the IP is not a valid Ingress host and must be skipped + TLSSANs: []string{"10.0.0.5", "my-cluster.example.com"}, + Expose: &v1beta1.ExposeConfig{ + Ingress: &v1beta1.IngressConfig{ + IngressClassName: "nginx", + Annotations: map[string]string{ + "nginx.ingress.kubernetes.io/ssl-passthrough": "true", + }, + }, + }, + }, + } + + Expect(k8sClient.Create(ctx, cluster)).To(Succeed()) + + ingressKey := client.ObjectKey{ + Name: server.IngressName(cluster.Name), + Namespace: cluster.Namespace, + } + + var ingress networkingv1.Ingress + + Eventually(func() error { + return k8sClient.Get(ctx, ingressKey, &ingress) + }). + WithTimeout(time.Second * 30). + WithPolling(time.Second). + Should(Succeed()) + + Expect(ingress.Spec.IngressClassName).To(Equal(ptr.To("nginx"))) + Expect(ingress.Annotations).To(HaveKeyWithValue("nginx.ingress.kubernetes.io/ssl-passthrough", "true")) + + Expect(ingress.Spec.Rules).To(HaveLen(1)) + Expect(ingress.Spec.Rules[0].Host).To(Equal("my-cluster.example.com")) + + backend := ingress.Spec.Rules[0].HTTP.Paths[0].Backend.Service + Expect(backend.Name).To(Equal(server.ServiceName(cluster.Name))) + Expect(backend.Port.Number).To(BeEquivalentTo(443)) + }) + }) + When("exposing the cluster with nodePort and loadbalancer", func() { It("will fail", func() { cluster := &v1beta1.Cluster{