diff --git a/charts/k3k/crds/k3k.io_clusters.yaml b/charts/k3k/crds/k3k.io_clusters.yaml
index 3e9283a..9e4acd6 100644
--- a/charts/k3k/crds/k3k.io_clusters.yaml
+++ b/charts/k3k/crds/k3k.io_clusters.yaml
@@ -230,6 +230,21 @@ spec:
loadbalancer:
description: LoadBalancer specifies options for exposing the API
server through a LoadBalancer service.
+ properties:
+ etcdPort:
+ description: |-
+ ETCDPort is the port on which the ETCD service is exposed when type is LoadBalancer.
+ If not specified, the default etcd 2379 port will be allocated.
+ If 0 or negative, the port will not be exposed.
+ format: int32
+ type: integer
+ serverPort:
+ description: |-
+ ServerPort is the port on which the K3s server is exposed when type is LoadBalancer.
+ If not specified, the default https 443 port will be allocated.
+ If 0 or negative, the port will not be exposed.
+ format: int32
+ type: integer
type: object
nodePort:
description: NodePort specifies options for exposing the API server
@@ -238,19 +253,15 @@ spec:
etcdPort:
description: |-
ETCDPort is the port on each node on which the ETCD service is exposed when type is NodePort.
- If not specified, a port will be allocated (default: 30000-32767).
+ If not specified, a random port between 30000-32767 will be allocated.
+ If out of range, the port will not be exposed.
format: int32
type: integer
serverPort:
description: |-
- ServerPort is the port on each node on which the K3s server service is exposed when type is NodePort.
- If not specified, a port will be allocated (default: 30000-32767).
- format: int32
- type: integer
- servicePort:
- description: |-
- ServicePort is the port on each node on which the K3s service is exposed when type is NodePort.
- If not specified, a port will be allocated (default: 30000-32767).
+ ServerPort is the port on each node on which the K3s server is exposed when type is NodePort.
+ If not specified, a random port between 30000-32767 will be allocated.
+ If out of range, the port will not be exposed.
format: int32
type: integer
type: object
diff --git a/docs/crds/crd-docs.md b/docs/crds/crd-docs.md
index fc0fdbb..0bc3d59 100644
--- a/docs/crds/crd-docs.md
+++ b/docs/crds/crd-docs.md
@@ -166,6 +166,10 @@ LoadBalancerConfig specifies options for exposing the API server through a LoadB
_Appears in:_
- [ExposeConfig](#exposeconfig)
+| Field | Description | Default | Validation |
+| --- | --- | --- | --- |
+| `serverPort` _integer_ | ServerPort is the port on which the K3s server is exposed when type is LoadBalancer.
If not specified, the default https 443 port will be allocated.
If 0 or negative, the port will not be exposed. | | |
+| `etcdPort` _integer_ | ETCDPort is the port on which the ETCD service is exposed when type is LoadBalancer.
If not specified, the default etcd 2379 port will be allocated.
If 0 or negative, the port will not be exposed. | | |
#### NodePortConfig
@@ -181,9 +185,8 @@ _Appears in:_
| Field | Description | Default | Validation |
| --- | --- | --- | --- |
-| `serverPort` _integer_ | ServerPort is the port on each node on which the K3s server service is exposed when type is NodePort.
If not specified, a port will be allocated (default: 30000-32767). | | |
-| `servicePort` _integer_ | ServicePort is the port on each node on which the K3s service is exposed when type is NodePort.
If not specified, a port will be allocated (default: 30000-32767). | | |
-| `etcdPort` _integer_ | ETCDPort is the port on each node on which the ETCD service is exposed when type is NodePort.
If not specified, a port will be allocated (default: 30000-32767). | | |
+| `serverPort` _integer_ | ServerPort is the port on each node on which the K3s server is exposed when type is NodePort.
If not specified, a random port between 30000-32767 will be allocated.
If out of range, the port will not be exposed. | | |
+| `etcdPort` _integer_ | ETCDPort is the port on each node on which the ETCD service is exposed when type is NodePort.
If not specified, a random port between 30000-32767 will be allocated.
If out of range, the port will not be exposed. | | |
#### PersistenceConfig
diff --git a/k3k-kubelet/kubelet.go b/k3k-kubelet/kubelet.go
index 29ea45a..b88d9a8 100644
--- a/k3k-kubelet/kubelet.go
+++ b/k3k-kubelet/kubelet.go
@@ -336,7 +336,7 @@ func virtRestConfig(ctx context.Context, virtualConfigPath string, hostClient ct
return nil, err
}
- url := fmt.Sprintf("https://%s:%d", server.ServiceName(cluster.Name), server.ServerPort)
+ url := "https://" + server.ServiceName(cluster.Name)
kubeconfigData, err := kubeconfigBytes(url, []byte(b.ServerCA.Content), adminCert, adminKey)
if err != nil {
diff --git a/k3k-kubelet/provider/provider.go b/k3k-kubelet/provider/provider.go
index 083eeac..c8d759d 100644
--- a/k3k-kubelet/provider/provider.go
+++ b/k3k-kubelet/provider/provider.go
@@ -826,13 +826,10 @@ func configureNetworking(pod *corev1.Pod, podName, podNamespace, serverIP, dnsIP
}
updatedEnvVars := []corev1.EnvVar{
- {Name: "KUBERNETES_PORT", Value: "tcp://" + serverIP + ":6443"},
{Name: "KUBERNETES_SERVICE_HOST", Value: serverIP},
- {Name: "KUBERNETES_SERVICE_PORT", Value: "6443"},
- {Name: "KUBERNETES_SERVICE_PORT_HTTPS", Value: "6443"},
- {Name: "KUBERNETES_PORT_443_TCP", Value: "tcp://" + serverIP + ":6443"},
+ {Name: "KUBERNETES_PORT", Value: "tcp://" + serverIP + ":443"},
+ {Name: "KUBERNETES_PORT_443_TCP", Value: "tcp://" + serverIP + ":443"},
{Name: "KUBERNETES_PORT_443_TCP_ADDR", Value: serverIP},
- {Name: "KUBERNETES_PORT_443_TCP_PORT", Value: "6443"},
}
// inject networking information to the pod's environment variables
diff --git a/pkg/apis/k3k.io/v1alpha1/types.go b/pkg/apis/k3k.io/v1alpha1/types.go
index df040f3..d7f0a7d 100644
--- a/pkg/apis/k3k.io/v1alpha1/types.go
+++ b/pkg/apis/k3k.io/v1alpha1/types.go
@@ -251,24 +251,34 @@ type IngressConfig struct {
}
// LoadBalancerConfig specifies options for exposing the API server through a LoadBalancer service.
-type LoadBalancerConfig struct{}
-
-// NodePortConfig specifies options for exposing the API server through NodePort.
-type NodePortConfig struct {
- // ServerPort is the port on each node on which the K3s server service is exposed when type is NodePort.
- // If not specified, a port will be allocated (default: 30000-32767).
+type LoadBalancerConfig struct {
+ // ServerPort is the port on which the K3s server is exposed when type is LoadBalancer.
+ // If not specified, the default https 443 port will be allocated.
+ // If 0 or negative, the port will not be exposed.
//
// +optional
ServerPort *int32 `json:"serverPort,omitempty"`
- // ServicePort is the port on each node on which the K3s service is exposed when type is NodePort.
- // If not specified, a port will be allocated (default: 30000-32767).
+ // ETCDPort is the port on which the ETCD service is exposed when type is LoadBalancer.
+ // If not specified, the default etcd 2379 port will be allocated.
+ // If 0 or negative, the port will not be exposed.
//
// +optional
- ServicePort *int32 `json:"servicePort,omitempty"`
+ ETCDPort *int32 `json:"etcdPort,omitempty"`
+}
+
+// NodePortConfig specifies options for exposing the API server through NodePort.
+type NodePortConfig struct {
+ // ServerPort is the port on each node on which the K3s server is exposed when type is NodePort.
+ // If not specified, a random port between 30000-32767 will be allocated.
+ // If out of range, the port will not be exposed.
+ //
+ // +optional
+ ServerPort *int32 `json:"serverPort,omitempty"`
// ETCDPort is the port on each node on which the ETCD service is exposed when type is NodePort.
- // If not specified, a port will be allocated (default: 30000-32767).
+ // If not specified, a random port between 30000-32767 will be allocated.
+ // If out of range, the port will not be exposed.
//
// +optional
ETCDPort *int32 `json:"etcdPort,omitempty"`
diff --git a/pkg/apis/k3k.io/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/k3k.io/v1alpha1/zz_generated.deepcopy.go
index 29bee4b..e5f0f28 100644
--- a/pkg/apis/k3k.io/v1alpha1/zz_generated.deepcopy.go
+++ b/pkg/apis/k3k.io/v1alpha1/zz_generated.deepcopy.go
@@ -340,7 +340,7 @@ func (in *ExposeConfig) DeepCopyInto(out *ExposeConfig) {
if in.LoadBalancer != nil {
in, out := &in.LoadBalancer, &out.LoadBalancer
*out = new(LoadBalancerConfig)
- **out = **in
+ (*in).DeepCopyInto(*out)
}
if in.NodePort != nil {
in, out := &in.NodePort, &out.NodePort
@@ -386,6 +386,16 @@ func (in *IngressConfig) DeepCopy() *IngressConfig {
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *LoadBalancerConfig) DeepCopyInto(out *LoadBalancerConfig) {
*out = *in
+ if in.ServerPort != nil {
+ in, out := &in.ServerPort, &out.ServerPort
+ *out = new(int32)
+ **out = **in
+ }
+ if in.ETCDPort != nil {
+ in, out := &in.ETCDPort, &out.ETCDPort
+ *out = new(int32)
+ **out = **in
+ }
return
}
@@ -407,11 +417,6 @@ func (in *NodePortConfig) DeepCopyInto(out *NodePortConfig) {
*out = new(int32)
**out = **in
}
- if in.ServicePort != nil {
- in, out := &in.ServicePort, &out.ServicePort
- *out = new(int32)
- **out = **in
- }
if in.ETCDPort != nil {
in, out := &in.ETCDPort, &out.ETCDPort
*out = new(int32)
diff --git a/pkg/controller/cluster/agent/virtual.go b/pkg/controller/cluster/agent/virtual.go
index fe91d81..1bdfbb9 100644
--- a/pkg/controller/cluster/agent/virtual.go
+++ b/pkg/controller/cluster/agent/virtual.go
@@ -72,7 +72,7 @@ func (v *VirtualAgent) config(ctx context.Context) error {
}
func virtualAgentData(serviceIP, token string) string {
- return fmt.Sprintf(`server: https://%s:6443
+ return fmt.Sprintf(`server: https://%s
token: %s
with-node-id: true`, serviceIP, token)
}
diff --git a/pkg/controller/cluster/agent/virtual_test.go b/pkg/controller/cluster/agent/virtual_test.go
index 886349d..7052e7c 100644
--- a/pkg/controller/cluster/agent/virtual_test.go
+++ b/pkg/controller/cluster/agent/virtual_test.go
@@ -25,7 +25,7 @@ func Test_virtualAgentData(t *testing.T) {
token: "dnjklsdjnksd892389238",
},
expectedData: map[string]string{
- "server": "https://10.0.0.21:6443",
+ "server": "https://10.0.0.21",
"token": "dnjklsdjnksd892389238",
"with-node-id": "true",
},
diff --git a/pkg/controller/cluster/cluster.go b/pkg/controller/cluster/cluster.go
index a523564..4b658ca 100644
--- a/pkg/controller/cluster/cluster.go
+++ b/pkg/controller/cluster/cluster.go
@@ -84,6 +84,7 @@ func Add(ctx context.Context, mgr manager.Manager, sharedAgentImage, sharedAgent
MaxConcurrentReconciles: maxConcurrentReconciles,
}).
Owns(&apps.StatefulSet{}).
+ Owns(&v1.Service{}).
Complete(&reconciler)
}
diff --git a/pkg/controller/cluster/cluster_test.go b/pkg/controller/cluster/cluster_test.go
index d2f9a37..1b4ecab 100644
--- a/pkg/controller/cluster/cluster_test.go
+++ b/pkg/controller/cluster/cluster_test.go
@@ -92,13 +92,37 @@ var _ = Describe("Cluster Controller", Label("controller"), Label("Cluster"), fu
Expect(spec.Ingress).To(Equal([]networkingv1.NetworkPolicyIngressRule{{}}))
})
- When("exposing the cluster with nodePort and custom ports", func() {
- It("will have a NodePort service with the specified port exposed", func() {
+ When("exposing the cluster with nodePort", func() {
+ It("will have a NodePort service", func() {
+ cluster.Spec.Expose = &v1alpha1.ExposeConfig{
+ NodePort: &v1alpha1.NodePortConfig{},
+ }
+
+ err := k8sClient.Update(ctx, cluster)
+ Expect(err).To(Not(HaveOccurred()))
+
+ var service v1.Service
+
+ Eventually(func() v1.ServiceType {
+ serviceKey := client.ObjectKey{
+ Name: server.ServiceName(cluster.Name),
+ Namespace: cluster.Namespace,
+ }
+
+ err := k8sClient.Get(ctx, serviceKey, &service)
+ Expect(client.IgnoreNotFound(err)).To(Not(HaveOccurred()))
+ return service.Spec.Type
+ }).
+ WithTimeout(time.Second * 30).
+ WithPolling(time.Second).
+ Should(Equal(v1.ServiceTypeNodePort))
+ })
+
+ It("will have the specified ports exposed when specified", func() {
cluster.Spec.Expose = &v1alpha1.ExposeConfig{
NodePort: &v1alpha1.NodePortConfig{
- ServerPort: ptr.To[int32](30010),
- ServicePort: ptr.To[int32](30011),
- ETCDPort: ptr.To[int32](30012),
+ ServerPort: ptr.To[int32](30010),
+ ETCDPort: ptr.To[int32](30011),
},
}
@@ -123,29 +147,95 @@ var _ = Describe("Cluster Controller", Label("controller"), Label("Cluster"), fu
servicePorts := service.Spec.Ports
Expect(servicePorts).NotTo(BeEmpty())
- Expect(servicePorts).To(HaveLen(3))
+ Expect(servicePorts).To(HaveLen(2))
- Expect(servicePorts).To(ContainElement(
- And(
- HaveField("Name", "k3s-server-port"),
- HaveField("Port", BeEquivalentTo(6443)),
- HaveField("NodePort", BeEquivalentTo(30010)),
- ),
- ))
- Expect(servicePorts).To(ContainElement(
- And(
- HaveField("Name", "k3s-service-port"),
- HaveField("Port", BeEquivalentTo(443)),
- HaveField("NodePort", BeEquivalentTo(30011)),
- ),
- ))
- Expect(servicePorts).To(ContainElement(
- And(
- HaveField("Name", "k3s-etcd-port"),
- HaveField("Port", BeEquivalentTo(2379)),
- HaveField("NodePort", BeEquivalentTo(30012)),
- ),
- ))
+ serverPort := servicePorts[0]
+ Expect(serverPort.Name).To(Equal("k3s-server-port"))
+ Expect(serverPort.Port).To(BeEquivalentTo(443))
+ Expect(serverPort.NodePort).To(BeEquivalentTo(30010))
+
+ etcdPort := servicePorts[1]
+ Expect(etcdPort.Name).To(Equal("k3s-etcd-port"))
+ Expect(etcdPort.Port).To(BeEquivalentTo(2379))
+ Expect(etcdPort.NodePort).To(BeEquivalentTo(30011))
+ })
+
+ It("will not expose the port when out of range", func() {
+ cluster.Spec.Expose = &v1alpha1.ExposeConfig{
+ NodePort: &v1alpha1.NodePortConfig{
+ ETCDPort: ptr.To[int32](2222),
+ },
+ }
+
+ err := k8sClient.Update(ctx, cluster)
+ Expect(err).To(Not(HaveOccurred()))
+
+ var service v1.Service
+
+ Eventually(func() v1.ServiceType {
+ serviceKey := client.ObjectKey{
+ Name: server.ServiceName(cluster.Name),
+ Namespace: cluster.Namespace,
+ }
+
+ err := k8sClient.Get(ctx, serviceKey, &service)
+ Expect(client.IgnoreNotFound(err)).To(Not(HaveOccurred()))
+ return service.Spec.Type
+ }).
+ WithTimeout(time.Second * 30).
+ WithPolling(time.Second).
+ Should(Equal(v1.ServiceTypeNodePort))
+
+ servicePorts := service.Spec.Ports
+ Expect(servicePorts).NotTo(BeEmpty())
+ Expect(servicePorts).To(HaveLen(1))
+
+ serverPort := servicePorts[0]
+ Expect(serverPort.Name).To(Equal("k3s-server-port"))
+ Expect(serverPort.Port).To(BeEquivalentTo(443))
+ Expect(serverPort.TargetPort.IntValue()).To(BeEquivalentTo(6443))
+ })
+
+ })
+
+ When("exposing the cluster with loadbalancer", func() {
+ It("will have a LoadBalancer service with the default ports exposed", func() {
+ cluster.Spec.Expose = &v1alpha1.ExposeConfig{
+ LoadBalancer: &v1alpha1.LoadBalancerConfig{},
+ }
+
+ err := k8sClient.Update(ctx, cluster)
+ Expect(err).To(Not(HaveOccurred()))
+
+ var service v1.Service
+
+ Eventually(func() v1.ServiceType {
+ serviceKey := client.ObjectKey{
+ Name: server.ServiceName(cluster.Name),
+ Namespace: cluster.Namespace,
+ }
+
+ err := k8sClient.Get(ctx, serviceKey, &service)
+ Expect(client.IgnoreNotFound(err)).To(Not(HaveOccurred()))
+ return service.Spec.Type
+ }).
+ WithTimeout(time.Second * 30).
+ WithPolling(time.Second).
+ Should(Equal(v1.ServiceTypeLoadBalancer))
+
+ servicePorts := service.Spec.Ports
+ Expect(servicePorts).NotTo(BeEmpty())
+ Expect(servicePorts).To(HaveLen(2))
+
+ serverPort := servicePorts[0]
+ Expect(serverPort.Name).To(Equal("k3s-server-port"))
+ Expect(serverPort.Port).To(BeEquivalentTo(443))
+ Expect(serverPort.TargetPort.IntValue()).To(BeEquivalentTo(6443))
+
+ etcdPort := servicePorts[1]
+ Expect(etcdPort.Name).To(Equal("k3s-etcd-port"))
+ Expect(etcdPort.Port).To(BeEquivalentTo(2379))
+ Expect(etcdPort.TargetPort.IntValue()).To(BeEquivalentTo(2379))
})
})
})
diff --git a/pkg/controller/cluster/server/bootstrap/bootstrap.go b/pkg/controller/cluster/server/bootstrap/bootstrap.go
index 82fe9d8..cbc2c14 100644
--- a/pkg/controller/cluster/server/bootstrap/bootstrap.go
+++ b/pkg/controller/cluster/server/bootstrap/bootstrap.go
@@ -51,7 +51,7 @@ func GenerateBootstrapData(ctx context.Context, cluster *v1alpha1.Cluster, ip, t
}
func requestBootstrap(token, serverIP string) (*ControlRuntimeBootstrap, error) {
- url := "https://" + serverIP + ":6443/v1-k3s/server-bootstrap"
+ url := "https://" + serverIP + "/v1-k3s/server-bootstrap"
client := http.Client{
Transport: &http.Transport{
diff --git a/pkg/controller/cluster/server/config.go b/pkg/controller/cluster/server/config.go
index 190ead7..0417d4b 100644
--- a/pkg/controller/cluster/server/config.go
+++ b/pkg/controller/cluster/server/config.go
@@ -39,7 +39,7 @@ func (s *Server) Config(init bool, serviceIP string) (*v1.Secret, error) {
}
func serverConfigData(serviceIP string, cluster *v1alpha1.Cluster, token string) string {
- return "cluster-init: true\nserver: https://" + serviceIP + ":6443\n" + serverOptions(cluster, token)
+ return "cluster-init: true\nserver: https://" + serviceIP + "\n" + serverOptions(cluster, token)
}
func initConfigData(cluster *v1alpha1.Cluster, token string) string {
diff --git a/pkg/controller/cluster/server/ingress.go b/pkg/controller/cluster/server/ingress.go
index d26303d..0d04101 100644
--- a/pkg/controller/cluster/server/ingress.go
+++ b/pkg/controller/cluster/server/ingress.go
@@ -11,9 +11,9 @@ import (
)
const (
- servicePort = 443
- serverPort = 6443
- etcdPort = 2379
+ httpsPort = 443
+ k3sServerPort = 6443
+ etcdPort = 2379
)
func IngressName(clusterName string) string {
@@ -64,7 +64,7 @@ func ingressRules(cluster *v1alpha1.Cluster) []networkingv1.IngressRule {
Service: &networkingv1.IngressServiceBackend{
Name: ServiceName(cluster.Name),
Port: networkingv1.ServiceBackendPort{
- Number: serverPort,
+ Number: httpsPort,
},
},
},
diff --git a/pkg/controller/cluster/server/server.go b/pkg/controller/cluster/server/server.go
index f91b5e6..75e67e0 100644
--- a/pkg/controller/cluster/server/server.go
+++ b/pkg/controller/cluster/server/server.go
@@ -24,8 +24,6 @@ const (
serverName = "server"
configName = "server-config"
initConfigName = "init-server-config"
-
- ServerPort = 6443
)
// Server
diff --git a/pkg/controller/cluster/server/service.go b/pkg/controller/cluster/server/service.go
index 6effdb6..cceed13 100644
--- a/pkg/controller/cluster/server/service.go
+++ b/pkg/controller/cluster/server/service.go
@@ -19,7 +19,6 @@ func Service(cluster *v1alpha1.Cluster) *v1.Service {
Namespace: cluster.Namespace,
},
Spec: v1.ServiceSpec{
- Type: v1.ServiceTypeClusterIP,
Selector: map[string]string{
"cluster": cluster.Name,
"role": "server",
@@ -28,16 +27,10 @@ func Service(cluster *v1alpha1.Cluster) *v1.Service {
}
k3sServerPort := v1.ServicePort{
- Name: "k3s-server-port",
- Protocol: v1.ProtocolTCP,
- Port: serverPort,
- }
-
- k3sServicePort := v1.ServicePort{
- Name: "k3s-service-port",
+ Name: "k3s-server-port",
Protocol: v1.ProtocolTCP,
- Port: servicePort,
- TargetPort: intstr.FromInt(serverPort),
+ Port: httpsPort,
+ TargetPort: intstr.FromInt(k3sServerPort),
}
etcdPort := v1.ServicePort{
@@ -46,35 +39,90 @@ func Service(cluster *v1alpha1.Cluster) *v1.Service {
Port: etcdPort,
}
+ // If no expose is specified, default to ClusterIP
+ if cluster.Spec.Expose == nil {
+ service.Spec.Type = v1.ServiceTypeClusterIP
+ service.Spec.Ports = append(service.Spec.Ports, k3sServerPort, etcdPort)
+ }
+
+ // If expose is specified, set the type to the appropriate type
if cluster.Spec.Expose != nil {
- nodePortConfig := cluster.Spec.Expose.NodePort
- if nodePortConfig != nil {
+ expose := cluster.Spec.Expose
+
+ // ingress
+ if expose.Ingress != nil {
+ service.Spec.Type = v1.ServiceTypeClusterIP
+ service.Spec.Ports = append(service.Spec.Ports, k3sServerPort, etcdPort)
+ }
+
+ // loadbalancer
+ if expose.LoadBalancer != nil {
+ service.Spec.Type = v1.ServiceTypeLoadBalancer
+ addLoadBalancerPorts(service, *expose.LoadBalancer, k3sServerPort, etcdPort)
+ }
+
+ // nodeport
+ if expose.NodePort != nil {
service.Spec.Type = v1.ServiceTypeNodePort
-
- if nodePortConfig.ServerPort != nil {
- k3sServerPort.NodePort = *nodePortConfig.ServerPort
- }
-
- if nodePortConfig.ServicePort != nil {
- k3sServicePort.NodePort = *nodePortConfig.ServicePort
- }
-
- if nodePortConfig.ETCDPort != nil {
- etcdPort.NodePort = *nodePortConfig.ETCDPort
- }
+ addNodePortPorts(service, *expose.NodePort, k3sServerPort, etcdPort)
}
}
- service.Spec.Ports = append(
- service.Spec.Ports,
- k3sServicePort,
- etcdPort,
- k3sServerPort,
- )
-
return service
}
+// addLoadBalancerPorts adds the load balancer ports to the service
+func addLoadBalancerPorts(service *v1.Service, loadbalancerConfig v1alpha1.LoadBalancerConfig, k3sServerPort, etcdPort v1.ServicePort) {
+ // If the server port is not specified, use the default port
+ if loadbalancerConfig.ServerPort == nil {
+ service.Spec.Ports = append(service.Spec.Ports, k3sServerPort)
+ } else if *loadbalancerConfig.ServerPort > 0 {
+ // If the server port is specified, set the port, otherwise the service will not be exposed
+ k3sServerPort.Port = *loadbalancerConfig.ServerPort
+ service.Spec.Ports = append(service.Spec.Ports, k3sServerPort)
+ }
+
+ // If the etcd port is not specified, use the default port
+ if loadbalancerConfig.ETCDPort == nil {
+ service.Spec.Ports = append(service.Spec.Ports, etcdPort)
+ } else if *loadbalancerConfig.ETCDPort > 0 {
+ // If the etcd port is specified, set the port, otherwise the service will not be exposed
+ etcdPort.Port = *loadbalancerConfig.ETCDPort
+ service.Spec.Ports = append(service.Spec.Ports, etcdPort)
+ }
+}
+
+// addNodePortPorts adds the node port ports to the service
+func addNodePortPorts(service *v1.Service, nodePortConfig v1alpha1.NodePortConfig, k3sServerPort, etcdPort v1.ServicePort) {
+ // If the server port is not specified Kubernetes will set the node port to a random port between 30000-32767
+ if nodePortConfig.ServerPort == nil {
+ service.Spec.Ports = append(service.Spec.Ports, k3sServerPort)
+ } else {
+ serverNodePort := *nodePortConfig.ServerPort
+
+ // If the server port is in the range of 30000-32767, set the node port
+ // otherwise the service will not be exposed
+ if serverNodePort >= 30000 && serverNodePort <= 32767 {
+ k3sServerPort.NodePort = serverNodePort
+ service.Spec.Ports = append(service.Spec.Ports, k3sServerPort)
+ }
+ }
+
+ // If the etcd port is not specified Kubernetes will set the node port to a random port between 30000-32767
+ if nodePortConfig.ETCDPort == nil {
+ service.Spec.Ports = append(service.Spec.Ports, etcdPort)
+ } else {
+ etcdNodePort := *nodePortConfig.ETCDPort
+
+ // If the etcd port is in the range of 30000-32767, set the node port
+ // otherwise the service will not be exposed
+ if etcdNodePort >= 30000 && etcdNodePort <= 32767 {
+ etcdPort.NodePort = etcdNodePort
+ service.Spec.Ports = append(service.Spec.Ports, etcdPort)
+ }
+ }
+}
+
func (s *Server) StatefulServerService() *v1.Service {
return &v1.Service{
TypeMeta: metav1.TypeMeta{
@@ -94,15 +142,10 @@ func (s *Server) StatefulServerService() *v1.Service {
},
Ports: []v1.ServicePort{
{
- Name: "k3s-server-port",
- Protocol: v1.ProtocolTCP,
- Port: serverPort,
- },
- {
- Name: "k3s-service-port",
+ Name: "k3s-server-port",
Protocol: v1.ProtocolTCP,
- Port: servicePort,
- TargetPort: intstr.FromInt(serverPort),
+ Port: httpsPort,
+ TargetPort: intstr.FromInt(k3sServerPort),
},
{
Name: "k3s-etcd-port",
diff --git a/pkg/controller/kubeconfig/kubeconfig.go b/pkg/controller/kubeconfig/kubeconfig.go
index aa977c3..eebfe1b 100644
--- a/pkg/controller/kubeconfig/kubeconfig.go
+++ b/pkg/controller/kubeconfig/kubeconfig.go
@@ -4,6 +4,7 @@ import (
"context"
"crypto/x509"
"fmt"
+ "slices"
"time"
certutil "github.com/rancher/dynamiclistener/cert"
@@ -12,6 +13,7 @@ import (
"github.com/rancher/k3k/pkg/controller/certs"
"github.com/rancher/k3k/pkg/controller/cluster/server"
"github.com/rancher/k3k/pkg/controller/cluster/server/bootstrap"
+ "github.com/sirupsen/logrus"
v1 "k8s.io/api/core/v1"
networkingv1 "k8s.io/api/networking/v1"
"k8s.io/apimachinery/pkg/types"
@@ -101,15 +103,41 @@ func getURLFromService(ctx context.Context, client client.Client, cluster *v1alp
return "", err
}
- url := fmt.Sprintf("https://%s:%d", k3kService.Spec.ClusterIP, server.ServerPort)
+ ip := k3kService.Spec.ClusterIP
+ port := int32(443)
- if k3kService.Spec.Type == v1.ServiceTypeNodePort {
- nodePort := k3kService.Spec.Ports[0].NodePort
- url = fmt.Sprintf("https://%s:%d", hostServerIP, nodePort)
+ switch k3kService.Spec.Type {
+ case v1.ServiceTypeNodePort:
+ ip = hostServerIP
+ port = k3kService.Spec.Ports[0].NodePort
+ case v1.ServiceTypeLoadBalancer:
+ ip = k3kService.Status.LoadBalancer.Ingress[0].IP
+ port = k3kService.Spec.Ports[0].Port
}
- expose := cluster.Spec.Expose
- if expose != nil && expose.Ingress != nil {
+ if !slices.Contains(cluster.Status.TLSSANs, ip) {
+ logrus.Warnf("ip %s not in tlsSANs", ip)
+
+ if len(cluster.Spec.TLSSANs) > 0 {
+ logrus.Warnf("Using the first TLS SAN in the spec as a fallback: %s", cluster.Spec.TLSSANs[0])
+
+ ip = cluster.Spec.TLSSANs[0]
+ } else if len(cluster.Status.TLSSANs) > 0 {
+ logrus.Warnf("No explicit tlsSANs specified. Trying to use the first TLS SAN in the status: %s", cluster.Status.TLSSANs[0])
+
+ ip = cluster.Status.TLSSANs[0]
+ } else {
+ logrus.Warn("ip not found in tlsSANs. This could cause issue with the certificate validation.")
+ }
+ }
+
+ url := "https://" + ip
+ if port != 443 {
+ url = fmt.Sprintf("%s:%d", url, port)
+ }
+
+ // if ingress is specified, use the ingress host
+ if cluster.Spec.Expose != nil && cluster.Spec.Expose.Ingress != nil {
var k3kIngress networkingv1.Ingress
ingressKey := types.NamespacedName{