Merge pull request #950 from bigkevmcd/expose-service-annotations

Support annotations on exposed Services

Makefile

@@ -11,6 +11,7 @@ GINKGO_FLAGS ?= -v -r --coverprofile=cover.out --coverpkg=./...
 ENVTEST_VERSION ?= v0.0.0-20250505003155-b6c5897febe5
 ENVTEST_K8S_VERSION := 1.31.0
 CRD_REF_DOCS_VER ?= v0.2.0
+FLAKE_ATTEMPTS ?= 3
 
 GOLANGCI_LINT ?= go run github.com/golangci/golangci-lint/v2/cmd/golangci-lint@$(GOLANGCI_LINT_VERSION)
 GINKGO ?= go run github.com/onsi/ginkgo/v2/ginkgo@$(GINKGO_VERSION)
@@ -81,7 +82,7 @@ test-integration:	## Run the controller tests that use envtest (tests/integratio
 
 .PHONY: test-e2e
 test-e2e:	## Run the e2e tests
-	$(GINKGO) $(GINKGO_FLAGS) --flake-attempts=3 --label-filter="$(E2E_LABEL_FILTER)" tests/e2e
+	$(GINKGO) $(GINKGO_FLAGS) --flake-attempts=$(FLAKE_ATTEMPTS) --label-filter="$(E2E_LABEL_FILTER)" tests/e2e
 
 .PHONY: test-cli
 test-cli:	## Run the cli tests

charts/k3k/templates/crds/k3k.io_clusters.yaml

@@ -1224,6 +1224,11 @@ spec:
                   Expose specifies options for exposing the API server.
                   By default, it's only exposed as a ClusterIP.
                 properties:
+                  annotations:
+                    additionalProperties:
+                      type: string
+                    description: Annotations specifies annotations to add to the generated Service.
+                    type: object
                   ingress:
                     description: Ingress specifies options for exposing the API server through an Ingress.
                     properties:

docs/crds/crds.adoc

@@ -381,6 +381,7 @@ _Appears In:_
 | *`ingress`* __xref:{anchor_prefix}-github-com-rancher-k3k-pkg-apis-k3k-io-v1beta1-ingressconfig[$$IngressConfig$$]__ | Ingress specifies options for exposing the API server through an Ingress. + |  | 
 | *`loadBalancer`* __xref:{anchor_prefix}-github-com-rancher-k3k-pkg-apis-k3k-io-v1beta1-loadbalancerconfig[$$LoadBalancerConfig$$]__ | LoadBalancer specifies options for exposing the API server through a LoadBalancer service. + |  | 
 | *`nodePort`* __xref:{anchor_prefix}-github-com-rancher-k3k-pkg-apis-k3k-io-v1beta1-nodeportconfig[$$NodePortConfig$$]__ | NodePort specifies options for exposing the API server through NodePort. + |  | 
+| *`annotations`* __object (keys:string, values:string)__ | Annotations specifies annotations to add to the generated Service. + |  | 
 |===
 
 

docs/crds/crds.md

@@ -284,6 +284,7 @@ _Appears in:_
 | `ingress` _[IngressConfig](#ingressconfig)_ | Ingress specifies options for exposing the API server through an Ingress. |  |  |
 | `loadBalancer` _[LoadBalancerConfig](#loadbalancerconfig)_ | LoadBalancer specifies options for exposing the API server through a LoadBalancer service. |  |  |
 | `nodePort` _[NodePortConfig](#nodeportconfig)_ | NodePort specifies options for exposing the API server through NodePort. |  |  |
+| `annotations` _object (keys:string, values:string)_ | Annotations specifies annotations to add to the generated Service. |  |  |
 
 
 #### IngressConfig

pkg/apis/k3k.io/v1beta1/types.go

@@ -494,6 +494,11 @@ type ExposeConfig struct {
 	//
 	// +optional
 	NodePort *NodePortConfig `json:"nodePort,omitempty"`
+
+	// Annotations specifies annotations to add to the generated Service.
+	//
+	// +optional
+	Annotations map[string]string `json:"annotations,omitempty"`
 }
 
 // IngressConfig specifies options for exposing the API server through an Ingress.

pkg/apis/k3k.io/v1beta1/zz_generated.deepcopy.go

@@ -408,6 +408,13 @@ func (in *ExposeConfig) DeepCopyInto(out *ExposeConfig) {
 		*out = new(NodePortConfig)
 		(*in).DeepCopyInto(*out)
 	}
+	if in.Annotations != nil {
+		in, out := &in.Annotations, &out.Annotations
+		*out = make(map[string]string, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExposeConfig.

pkg/controller/cluster/cluster.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"maps"
 	"net"
 	"slices"
 	"strings"
@@ -662,6 +663,13 @@ func (c *ClusterReconciler) ensureClusterService(ctx context.Context, cluster *v
 			return err
 		}
 
+		// Retain existing annotations but overwrite with our generated
+		// annotations.
+		if currentService.Annotations == nil {
+			currentService.Annotations = expectedService.Annotations
+		} else {
+			maps.Copy(currentService.Annotations, expectedService.Annotations)
+		}
 		currentService.Spec = expectedService.Spec
 
 		return nil

pkg/controller/cluster/server/service.go

@@ -1,6 +1,8 @@
 package server
 
 import (
+	"maps"
+
 	"k8s.io/apimachinery/pkg/util/intstr"
 
 	corev1 "k8s.io/api/core/v1"
@@ -10,6 +12,10 @@ import (
 	"github.com/rancher/k3k/pkg/controller"
 )
 
+// Service creates a Kubernetes Service for the given cluster.
+//
+// It sets the service type based on the cluster's expose configuration and adds the
+// appropriate ports for k3s server and etcd.
 func Service(cluster *v1beta1.Cluster) *corev1.Service {
 	service := &corev1.Service{
 		TypeMeta: metav1.TypeMeta{
@@ -50,6 +56,13 @@ func Service(cluster *v1beta1.Cluster) *corev1.Service {
 	// If expose is specified, set the type to the appropriate type
 	if cluster.Spec.Expose != nil {
 		expose := cluster.Spec.Expose
+		if expose.Annotations != nil {
+			if service.Annotations == nil {
+				service.Annotations = map[string]string{}
+			}
+
+			maps.Copy(service.Annotations, expose.Annotations)
+		}
 
 		switch {
 		case expose.LoadBalancer != nil:

pkg/controller/cluster/server/service_test.go

@@ -0,0 +1,178 @@
+package server
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/utils/ptr"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
+)
+
+func TestService(t *testing.T) {
+	tests := map[string]struct {
+		clusterOpts []func(*v1beta1.Cluster)
+		serviceOpts []func(*corev1.Service)
+	}{
+		"no expose": {
+			serviceOpts: []func(*corev1.Service){
+				func(s *corev1.Service) {
+					s.Spec.Type = corev1.ServiceTypeClusterIP
+					s.Spec.Ports = []corev1.ServicePort{
+						{
+							Name:       "k3s-server-port",
+							Protocol:   corev1.ProtocolTCP,
+							Port:       int32(443),
+							TargetPort: intstr.FromInt(6443),
+						},
+						{
+							Name:     "k3s-etcd-port",
+							Protocol: corev1.ProtocolTCP,
+							Port:     int32(2379),
+						},
+					}
+				},
+			},
+		},
+		"expose load balancer": {
+			clusterOpts: []func(*v1beta1.Cluster){
+				func(c *v1beta1.Cluster) {
+					c.Spec = v1beta1.ClusterSpec{
+						Expose: &v1beta1.ExposeConfig{
+							LoadBalancer: &v1beta1.LoadBalancerConfig{
+								ServerPort: ptr.To[int32](9443),
+							},
+						},
+					}
+				},
+			},
+			serviceOpts: []func(*corev1.Service){
+				func(s *corev1.Service) {
+					s.Spec.Type = corev1.ServiceTypeLoadBalancer
+					s.Spec.Ports = []corev1.ServicePort{
+						{
+							Name:       "k3s-server-port",
+							Protocol:   corev1.ProtocolTCP,
+							Port:       int32(9443),
+							TargetPort: intstr.FromInt(6443),
+						},
+						{
+							Name:     "k3s-etcd-port",
+							Protocol: corev1.ProtocolTCP,
+							Port:     int32(2379),
+						},
+					}
+				},
+			},
+		},
+		"expose load balancer with annotations": {
+			clusterOpts: []func(*v1beta1.Cluster){
+				func(c *v1beta1.Cluster) {
+					c.Spec.Expose = &v1beta1.ExposeConfig{
+						Annotations: map[string]string{
+							"example.com/testing": "test-annotation",
+						},
+						LoadBalancer: &v1beta1.LoadBalancerConfig{
+							ServerPort: ptr.To[int32](9443),
+						},
+					}
+				},
+			},
+			serviceOpts: []func(*corev1.Service){
+				func(s *corev1.Service) {
+					s.Annotations = map[string]string{
+						"example.com/testing": "test-annotation",
+					}
+					s.Spec.Type = corev1.ServiceTypeLoadBalancer
+					s.Spec.Ports = []corev1.ServicePort{
+						{
+							Name:       "k3s-server-port",
+							Protocol:   corev1.ProtocolTCP,
+							Port:       int32(9443),
+							TargetPort: intstr.FromInt(6443),
+						},
+						{
+							Name:     "k3s-etcd-port",
+							Protocol: corev1.ProtocolTCP,
+							Port:     int32(2379),
+						},
+					}
+				},
+			},
+		},
+		"expose node port": {
+			clusterOpts: []func(*v1beta1.Cluster){
+				func(c *v1beta1.Cluster) {
+					c.Spec.Expose = &v1beta1.ExposeConfig{
+						NodePort: &v1beta1.NodePortConfig{
+							ServerPort: ptr.To[int32](7443),
+						},
+					}
+				},
+			},
+			serviceOpts: []func(*corev1.Service){
+				func(s *corev1.Service) {
+					s.Spec.Type = corev1.ServiceTypeNodePort
+					s.Spec.Ports = []corev1.ServicePort{
+						{
+							Name:     "k3s-etcd-port",
+							Protocol: corev1.ProtocolTCP,
+							Port:     int32(2379),
+						},
+					}
+				},
+			},
+		},
+	}
+
+	for name, tt := range tests {
+		t.Run(name, func(t *testing.T) {
+			cluster := newTestCluster(tt.clusterOpts...)
+			want := newTestService(cluster, tt.serviceOpts...)
+			assert.Equal(t, want, Service(cluster))
+		})
+	}
+}
+
+func newTestService(cluster *v1beta1.Cluster, opts ...func(*corev1.Service)) *corev1.Service {
+	svc := &corev1.Service{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Service",
+			APIVersion: "v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "k3k-test-cluster-service",
+			Namespace: cluster.Namespace,
+		},
+		Spec: corev1.ServiceSpec{
+			Selector: map[string]string{
+				"cluster": "test-cluster",
+				"role":    "server",
+			},
+		},
+	}
+	for _, opt := range opts {
+		opt(svc)
+	}
+
+	return svc
+}
+
+func newTestCluster(opts ...func(*v1beta1.Cluster)) *v1beta1.Cluster {
+	cluster := &v1beta1.Cluster{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test-cluster",
+			Namespace: "test-namespace",
+		},
+		Spec: v1beta1.ClusterSpec{},
+	}
+	for _, opt := range opts {
+		opt(cluster)
+	}
+
+	return cluster
+}

tests/e2e/cluster_create_test.go

@@ -7,10 +7,12 @@ import (
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
+	"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
 	fwk3k "github.com/rancher/k3k/tests/framework/k3k"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	. "github.com/onsi/gomega/gstruct"
 )
 
 var _ = When("creating a shared mode cluster", Label(e2eTestLabel), Label(slowTestsLabel), func() {
@@ -23,7 +25,11 @@ var _ = When("creating a shared mode cluster", Label(e2eTestLabel), Label(slowTe
 			fwk3k.DeleteNamespaces(k8s, namespace.Name)
 		})
 
-		cluster := NewCluster(namespace.Name)
+		cluster := NewCluster(namespace.Name, func(c *v1beta1.Cluster) {
+			c.Spec.Expose.Annotations = map[string]string{
+				"example.com/test": "testing",
+			}
+		})
 		CreateCluster(cluster)
 		client, restConfig := NewVirtualK8sClientAndConfig(cluster)
 
@@ -48,6 +54,75 @@ var _ = When("creating a shared mode cluster", Label(e2eTestLabel), Label(slowTe
 			Should(Succeed())
 	})
 
+	It("creates services with annotations", func() {
+		Eventually(func(g Gomega) {
+			ctx := GinkgoT().Context()
+
+			cluster := virtualCluster.Cluster
+			service, err := k8s.CoreV1().Services(cluster.Namespace).Get(
+				ctx, "k3k-"+cluster.GetName()+"-service", metav1.GetOptions{})
+			g.Expect(err).To(Not(HaveOccurred()))
+
+			g.Expect(service.GetAnnotations()).To(MatchAllKeys(Keys{
+				"example.com/test": Equal("testing"),
+			}))
+		}).
+			WithTimeout(time.Minute).
+			WithPolling(time.Second).
+			Should(Succeed())
+	})
+
+	It("updates the annotations when the cluster is updated", func() {
+		// Wait for Service to be created.
+		ctx := GinkgoT().Context()
+		cluster := virtualCluster.Cluster
+
+		Eventually(func(g Gomega) {
+			service, err := k8s.CoreV1().Services(cluster.Namespace).Get(
+				ctx, "k3k-"+cluster.GetName()+"-service", metav1.GetOptions{})
+			g.Expect(err).To(Not(HaveOccurred()))
+
+			g.Expect(service.GetAnnotations()).To(MatchAllKeys(Keys{
+				"example.com/test": Equal("testing"),
+			}))
+		}).
+			WithTimeout(time.Minute).
+			WithPolling(time.Second).
+			Should(Succeed())
+
+		service, err := k8s.CoreV1().Services(cluster.Namespace).Get(
+			ctx, "k3k-"+cluster.GetName()+"-service", metav1.GetOptions{})
+		Expect(err).To(Not(HaveOccurred()))
+
+		service.Annotations["example.com/other-annotation"] = "retain-this"
+		_, err = k8s.CoreV1().Services(cluster.Namespace).Update(ctx, service, metav1.UpdateOptions{})
+		Expect(err).To(Not(HaveOccurred()))
+
+		// Reload cluster
+		key := client.ObjectKeyFromObject(cluster)
+		Expect(k8sClient.Get(ctx, key, cluster)).To(Succeed())
+
+		// Update annotations
+		cluster.Spec.Expose.Annotations = map[string]string{
+			"example.com/test": "updated",
+		}
+		Expect(k8sClient.Update(ctx, cluster)).To(Succeed())
+
+		Eventually(func(g Gomega) {
+			service, err := k8s.CoreV1().Services(cluster.Namespace).Get(
+				ctx, "k3k-"+cluster.GetName()+"-service", metav1.GetOptions{})
+			g.Expect(err).To(Not(HaveOccurred()))
+
+			g.Expect(service.GetAnnotations()).To(MatchAllKeys(Keys{
+				"example.com/test":             Equal("updated"),
+				"example.com/other-annotation": Equal("retain-this"),
+			}))
+		}).
+			WithTimeout(time.Minute).
+			WithPolling(time.Second).
+			Should(Succeed())
+	})
+
 	It("has the provider.cattle.io label set to k3k", func() {
 		Eventually(func(g Gomega) {
 			ctx := GinkgoT().Context()

tests/e2e/cluster_sync_test.go

@@ -32,7 +32,7 @@ var _ = When("a shared mode cluster is created", Ordered, Label(e2eTestLabel), f
 
 	When("a ConfigMap is created in the virtual cluster", func() {
 		BeforeAll(func() {
-			ctx := context.Background()
+			ctx := GinkgoT().Context()
 
 			virtualConfigMap = &corev1.ConfigMap{
 				ObjectMeta: metav1.ObjectMeta{
@@ -48,7 +48,7 @@ var _ = When("a shared mode cluster is created", Ordered, Label(e2eTestLabel), f
 		})
 
 		It("is replicated in the host cluster", func() {
-			ctx := context.Background()
+			ctx := GinkgoT().Context()
 
 			hostTranslator := translate.NewHostTranslator(virtualCluster.Cluster)
 			namespacedName := hostTranslator.NamespacedName(virtualConfigMap)
@@ -66,7 +66,7 @@ var _ = When("a shared mode cluster is created", Ordered, Label(e2eTestLabel), f
 
 	When("a Service is created in the virtual cluster", func() {
 		BeforeAll(func() {
-			ctx := context.Background()
+			ctx := GinkgoT().Context()
 
 			virtualService = &corev1.Service{
 				ObjectMeta: metav1.ObjectMeta{
@@ -86,7 +86,7 @@ var _ = When("a shared mode cluster is created", Ordered, Label(e2eTestLabel), f
 		})
 
 		It("is replicated in the host cluster", func() {
-			ctx := context.Background()
+			ctx := GinkgoT().Context()
 
 			hostTranslator := translate.NewHostTranslator(virtualCluster.Cluster)
 			namespacedName := hostTranslator.NamespacedName(virtualService)

tests/e2e/common_test.go

@@ -87,8 +87,8 @@ func NewVirtualClusters(n int) []*VirtualCluster {
 	return clusters
 }
 
-func NewCluster(namespace string) *v1beta1.Cluster {
-	return &v1beta1.Cluster{
+func NewCluster(namespace string, opts ...func(*v1beta1.Cluster)) *v1beta1.Cluster {
+	c := &v1beta1.Cluster{
 		ObjectMeta: metav1.ObjectMeta{
 			GenerateName: "cluster-",
 			Namespace:    namespace,
@@ -103,6 +103,12 @@ func NewCluster(namespace string) *v1beta1.Cluster {
 			},
 		},
 	}
+
+	for _, optFn := range opts {
+		optFn(c)
+	}
+
+	return c
 }
 
 func CreateCluster(cluster *v1beta1.Cluster) {