diff --git a/charts/k3k/crds/k3k.io_clusters.yaml b/charts/k3k/crds/k3k.io_clusters.yaml
index 9185d05e..08aa21a2 100644
--- a/charts/k3k/crds/k3k.io_clusters.yaml
+++ b/charts/k3k/crds/k3k.io_clusters.yaml
@@ -125,10 +125,24 @@ spec:
type: object
nodePort:
properties:
- enabled:
- type: boolean
- required:
- - enabled
+ etcdPort:
+ description: |-
+ ETCDPort is the port on each node on which the ETCD service is exposed when type is NodePort.
+ If not specified, a port will be allocated (default: 30000-32767)
+ format: int32
+ type: integer
+ serverPort:
+ description: |-
+ ServerPort is the port on each node on which the K3s server service is exposed when type is NodePort.
+ If not specified, a port will be allocated (default: 30000-32767)
+ format: int32
+ type: integer
+ servicePort:
+ description: |-
+ ServicePort is the port on each node on which the K3s service is exposed when type is NodePort.
+ If not specified, a port will be allocated (default: 30000-32767)
+ format: int32
+ type: integer
type: object
type: object
mode:
diff --git a/cli/cmds/cluster/create.go b/cli/cmds/cluster/create.go
index 87957057..5425303e 100644
--- a/cli/cmds/cluster/create.go
+++ b/cli/cmds/cluster/create.go
@@ -100,9 +100,7 @@ func createAction(config *CreateConfig) cli.ActionFunc {
cluster := newCluster(name, cmds.Namespace(), config)
cluster.Spec.Expose = &v1alpha1.ExposeConfig{
- NodePort: &v1alpha1.NodePortConfig{
- Enabled: true,
- },
+ NodePort: &v1alpha1.NodePortConfig{},
}
// add Host IP address as an extra TLS-SAN to expose the k3k cluster
diff --git a/docs/crds/crd-docs.md b/docs/crds/crd-docs.md
index 435be85e..e967e95a 100644
--- a/docs/crds/crd-docs.md
+++ b/docs/crds/crd-docs.md
@@ -196,7 +196,9 @@ _Appears in:_
| Field | Description | Default | Validation |
| --- | --- | --- | --- |
-| `enabled` _boolean_ | | | |
+| `serverPort` _integer_ | ServerPort is the port on each node on which the K3s server service is exposed when type is NodePort.
If not specified, a port will be allocated (default: 30000-32767) | | |
+| `servicePort` _integer_ | ServicePort is the port on each node on which the K3s service is exposed when type is NodePort.
If not specified, a port will be allocated (default: 30000-32767) | | |
+| `etcdPort` _integer_ | ETCDPort is the port on each node on which the ETCD service is exposed when type is NodePort.
If not specified, a port will be allocated (default: 30000-32767) | | |
#### PersistenceConfig
diff --git a/pkg/apis/k3k.io/v1alpha1/types.go b/pkg/apis/k3k.io/v1alpha1/types.go
index 3279cb3f..3eac862f 100644
--- a/pkg/apis/k3k.io/v1alpha1/types.go
+++ b/pkg/apis/k3k.io/v1alpha1/types.go
@@ -160,7 +160,18 @@ type LoadBalancerConfig struct {
}
type NodePortConfig struct {
- Enabled bool `json:"enabled"`
+ // ServerPort is the port on each node on which the K3s server service is exposed when type is NodePort.
+ // If not specified, a port will be allocated (default: 30000-32767)
+ // +optional
+ ServerPort *int32 `json:"serverPort,omitempty"`
+ // ServicePort is the port on each node on which the K3s service is exposed when type is NodePort.
+ // If not specified, a port will be allocated (default: 30000-32767)
+ // +optional
+ ServicePort *int32 `json:"servicePort,omitempty"`
+ // ETCDPort is the port on each node on which the ETCD service is exposed when type is NodePort.
+ // If not specified, a port will be allocated (default: 30000-32767)
+ // +optional
+ ETCDPort *int32 `json:"etcdPort,omitempty"`
}
type ClusterStatus struct {
diff --git a/pkg/controller/cluster/cluster.go b/pkg/controller/cluster/cluster.go
index 04e50443..088feab6 100644
--- a/pkg/controller/cluster/cluster.go
+++ b/pkg/controller/cluster/cluster.go
@@ -180,12 +180,11 @@ func (c *ClusterReconciler) reconcileCluster(ctx context.Context, cluster *v1alp
cluster.Status.ServiceCIDR = defaultClusterServiceCIDR
}
- log.Info("creating cluster service")
-
- serviceIP, err := c.createClusterService(ctx, cluster, s)
+ service, err := c.ensureClusterService(ctx, cluster)
if err != nil {
return err
}
+ serviceIP := service.Spec.ClusterIP
if err := c.createClusterConfigs(ctx, cluster, s, serviceIP); err != nil {
return err
@@ -290,30 +289,36 @@ func (c *ClusterReconciler) createClusterConfigs(ctx context.Context, cluster *v
return nil
}
-func (c *ClusterReconciler) createClusterService(ctx context.Context, cluster *v1alpha1.Cluster, s *server.Server) (string, error) {
- // create cluster service
- clusterService := s.Service(cluster)
+func (c *ClusterReconciler) ensureClusterService(ctx context.Context, cluster *v1alpha1.Cluster) (*v1.Service, error) {
+ log := ctrl.LoggerFrom(ctx)
+ log.Info("ensuring cluster service")
- if err := controllerutil.SetControllerReference(cluster, clusterService, c.Scheme); err != nil {
- return "", err
+ service := server.Service(cluster)
+
+ createdClusterService := &v1.Service{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: service.Name,
+ Namespace: service.Namespace,
+ },
}
- if err := c.Client.Create(ctx, clusterService); err != nil {
- if !apierrors.IsAlreadyExists(err) {
- return "", err
+ result, err := controllerutil.CreateOrUpdate(ctx, c.Client, createdClusterService, func() error {
+ if err := controllerutil.SetControllerReference(cluster, createdClusterService, c.Scheme); err != nil {
+ return err
}
+
+ createdClusterService.Spec = service.Spec
+ return nil
+ })
+ if err != nil {
+ return nil, err
}
- var service v1.Service
-
- objKey := ctrlruntimeclient.ObjectKey{
- Namespace: cluster.Namespace,
- Name: server.ServiceName(cluster.Name),
- }
- if err := c.Client.Get(ctx, objKey, &service); err != nil {
- return "", err
+ key := client.ObjectKeyFromObject(createdClusterService)
+ if result != controllerutil.OperationResultNone {
+ log.Info("ensuring cluster service", "key", key, "result", result)
}
- return service.Spec.ClusterIP, nil
+ return createdClusterService, nil
}
func (c *ClusterReconciler) server(ctx context.Context, cluster *v1alpha1.Cluster, server *server.Server) error {
diff --git a/pkg/controller/cluster/cluster_test.go b/pkg/controller/cluster/cluster_test.go
index 762d4d4e..133c9e34 100644
--- a/pkg/controller/cluster/cluster_test.go
+++ b/pkg/controller/cluster/cluster_test.go
@@ -6,10 +6,12 @@ import (
"time"
"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
+ "github.com/rancher/k3k/pkg/controller/cluster/server"
"sigs.k8s.io/controller-runtime/pkg/client"
corev1 "k8s.io/api/core/v1"
- v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ v1 "k8s.io/api/core/v1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/utils/ptr"
. "github.com/onsi/ginkgo/v2"
@@ -25,17 +27,19 @@ var _ = Describe("Cluster Controller", func() {
)
BeforeEach(func() {
- createdNS := &corev1.Namespace{ObjectMeta: v1.ObjectMeta{GenerateName: "ns-"}}
+ createdNS := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{GenerateName: "ns-"}}
err := k8sClient.Create(context.Background(), createdNS)
Expect(err).To(Not(HaveOccurred()))
namespace = createdNS.Name
})
- When("created with a default spec", func() {
+ When("creating a Cluster", func() {
- It("should have been created with some defaults", func() {
- cluster := &v1alpha1.Cluster{
- ObjectMeta: v1.ObjectMeta{
+ var cluster *v1alpha1.Cluster
+
+ BeforeEach(func() {
+ cluster = &v1alpha1.Cluster{
+ ObjectMeta: metav1.ObjectMeta{
GenerateName: "clusterset-",
Namespace: namespace,
},
@@ -43,7 +47,9 @@ var _ = Describe("Cluster Controller", func() {
err := k8sClient.Create(ctx, cluster)
Expect(err).To(Not(HaveOccurred()))
+ })
+ It("will be created with some defaults", func() {
Expect(cluster.Spec.Mode).To(Equal(v1alpha1.SharedClusterMode))
Expect(cluster.Spec.Agents).To(Equal(ptr.To[int32](0)))
Expect(cluster.Spec.Servers).To(Equal(ptr.To[int32](1)))
@@ -63,6 +69,63 @@ var _ = Describe("Cluster Controller", func() {
WithPolling(time.Second).
Should(Equal(expectedHostVersion))
})
+
+ When("exposing the cluster with nodePort and custom posrts", func() {
+ It("will have a NodePort service with the specified port exposed", func() {
+ cluster.Spec.Expose = &v1alpha1.ExposeConfig{
+ NodePort: &v1alpha1.NodePortConfig{
+ ServerPort: ptr.To[int32](30010),
+ ServicePort: ptr.To[int32](30011),
+ ETCDPort: ptr.To[int32](30012),
+ },
+ }
+
+ err := k8sClient.Update(ctx, cluster)
+ Expect(err).To(Not(HaveOccurred()))
+
+ var service v1.Service
+
+ Eventually(func() v1.ServiceType {
+ serviceKey := client.ObjectKey{
+ Name: server.ServiceName(cluster.Name),
+ Namespace: cluster.Namespace,
+ }
+
+ err := k8sClient.Get(ctx, serviceKey, &service)
+ Expect(client.IgnoreNotFound(err)).To(Not(HaveOccurred()))
+ return service.Spec.Type
+ }).
+ WithTimeout(time.Second * 30).
+ WithPolling(time.Second).
+ Should(Equal(v1.ServiceTypeNodePort))
+
+ servicePorts := service.Spec.Ports
+ Expect(servicePorts).NotTo(BeEmpty())
+ Expect(servicePorts).To(HaveLen(3))
+
+ Expect(servicePorts).To(ContainElement(
+ And(
+ HaveField("Name", "k3s-server-port"),
+ HaveField("Port", BeEquivalentTo(6443)),
+ HaveField("NodePort", BeEquivalentTo(30010)),
+ ),
+ ))
+ Expect(servicePorts).To(ContainElement(
+ And(
+ HaveField("Name", "k3s-service-port"),
+ HaveField("Port", BeEquivalentTo(443)),
+ HaveField("NodePort", BeEquivalentTo(30011)),
+ ),
+ ))
+ Expect(servicePorts).To(ContainElement(
+ And(
+ HaveField("Name", "k3s-etcd-port"),
+ HaveField("Port", BeEquivalentTo(2379)),
+ HaveField("NodePort", BeEquivalentTo(30012)),
+ ),
+ ))
+ })
+ })
})
})
})
diff --git a/pkg/controller/cluster/server/service.go b/pkg/controller/cluster/server/service.go
index a25b371d..d9bd307a 100644
--- a/pkg/controller/cluster/server/service.go
+++ b/pkg/controller/cluster/server/service.go
@@ -8,51 +8,69 @@ import (
"k8s.io/apimachinery/pkg/util/intstr"
)
-func (s *Server) Service(cluster *v1alpha1.Cluster) *v1.Service {
- serviceType := v1.ServiceTypeClusterIP
- if cluster.Spec.Expose != nil {
- if cluster.Spec.Expose.NodePort != nil {
- if cluster.Spec.Expose.NodePort.Enabled {
- serviceType = v1.ServiceTypeNodePort
- }
- }
- }
-
- return &v1.Service{
+func Service(cluster *v1alpha1.Cluster) *v1.Service {
+ service := &v1.Service{
TypeMeta: metav1.TypeMeta{
Kind: "Service",
APIVersion: "v1",
},
ObjectMeta: metav1.ObjectMeta{
- Name: ServiceName(s.cluster.Name),
+ Name: ServiceName(cluster.Name),
Namespace: cluster.Namespace,
},
Spec: v1.ServiceSpec{
- Type: serviceType,
+ Type: v1.ServiceTypeClusterIP,
Selector: map[string]string{
"cluster": cluster.Name,
"role": "server",
},
- Ports: []v1.ServicePort{
- {
- Name: "k3s-server-port",
- Protocol: v1.ProtocolTCP,
- Port: serverPort,
- },
- {
- Name: "k3s-service-port",
- Protocol: v1.ProtocolTCP,
- Port: servicePort,
- TargetPort: intstr.FromInt(serverPort),
- },
- {
- Name: "k3s-etcd-port",
- Protocol: v1.ProtocolTCP,
- Port: etcdPort,
- },
- },
},
}
+
+ k3sServerPort := v1.ServicePort{
+ Name: "k3s-server-port",
+ Protocol: v1.ProtocolTCP,
+ Port: serverPort,
+ }
+
+ k3sServicePort := v1.ServicePort{
+ Name: "k3s-service-port",
+ Protocol: v1.ProtocolTCP,
+ Port: servicePort,
+ TargetPort: intstr.FromInt(serverPort),
+ }
+
+ etcdPort := v1.ServicePort{
+ Name: "k3s-etcd-port",
+ Protocol: v1.ProtocolTCP,
+ Port: etcdPort,
+ }
+
+ if cluster.Spec.Expose != nil {
+ nodePortConfig := cluster.Spec.Expose.NodePort
+ if nodePortConfig != nil {
+ service.Spec.Type = v1.ServiceTypeNodePort
+
+ if nodePortConfig.ServerPort != nil {
+ k3sServerPort.NodePort = *nodePortConfig.ServerPort
+ }
+ if nodePortConfig.ServicePort != nil {
+ k3sServicePort.NodePort = *nodePortConfig.ServicePort
+ }
+ if nodePortConfig.ETCDPort != nil {
+ etcdPort.NodePort = *nodePortConfig.ETCDPort
+ }
+ }
+ }
+
+ service.Spec.Ports = append(
+ service.Spec.Ports,
+ k3sServicePort,
+ etcdPort,
+ k3sServerPort,
+ )
+
+ return service
}
func (s *Server) StatefulServerService() *v1.Service {
diff --git a/tests/cluster_test.go b/tests/cluster_test.go
index e3be92fb..75cfa527 100644
--- a/tests/cluster_test.go
+++ b/tests/cluster_test.go
@@ -64,9 +64,7 @@ var _ = When("a cluster is installed", func() {
Spec: v1alpha1.ClusterSpec{
TLSSANs: []string{hostIP},
Expose: &v1alpha1.ExposeConfig{
- NodePort: &v1alpha1.NodePortConfig{
- Enabled: true,
- },
+ NodePort: &v1alpha1.NodePortConfig{},
},
},
}
@@ -130,9 +128,7 @@ var _ = When("a cluster is installed", func() {
Spec: v1alpha1.ClusterSpec{
TLSSANs: []string{hostIP},
Expose: &v1alpha1.ExposeConfig{
- NodePort: &v1alpha1.NodePortConfig{
- Enabled: true,
- },
+ NodePort: &v1alpha1.NodePortConfig{},
},
},
}
@@ -154,7 +150,6 @@ var _ = When("a cluster is installed", func() {
serverPod := serverPods.Items[0]
fmt.Fprintf(GinkgoWriter, "deleting pod %s/%s\n", serverPod.Namespace, serverPod.Name)
- // GracePeriodSeconds: ptr.To[int64](0)
err = k8s.CoreV1().Pods(namespace).Delete(ctx, serverPod.Name, v1.DeleteOptions{})
Expect(err).To(Not(HaveOccurred()))