Add HCP (Hosted Control Plane) support

Introduce hosted control plane mode for k3k virtual clusters, including
API types, controller logic, server endpoint handling, CLI flags,
CRD updates, kubeconfig generation, and examples.

Co-Authored-By: RuFlo <ruv@ruv.net>
This commit is contained in:
jpgouin
2026-06-12 17:36:48 +02:00
committed by Enrico Candino
co-authored by RuFlo
parent 651da42ef0
commit 67d5f4dbfc
20 changed files with 1311 additions and 419 deletions
+26 -3
View File
@@ -400,9 +400,10 @@ func (c *ClusterReconciler) reconcile(ctx context.Context, cluster *v1beta1.Clus
}
}
// in virtual mode assign a default serviceCIDR
if cluster.Spec.Mode == v1beta1.VirtualClusterMode {
log.V(1).Info("assign default service CIDR for virtual mode")
// virtual and hcp modes both run a self-contained K3s control plane and
// need their own pod/service CIDR independent of the host cluster.
if cluster.Spec.Mode == v1beta1.VirtualClusterMode || cluster.Spec.Mode == v1beta1.HCPClusterMode {
log.V(1).Info("assign default service CIDR", "mode", cluster.Spec.Mode)
cluster.Status.ServiceCIDR = defaultVirtualServiceCIDR
}
@@ -447,6 +448,21 @@ func (c *ClusterReconciler) reconcile(ctx context.Context, cluster *v1beta1.Clus
return err
}
// In hcp mode, derive the K3s installer command end-users run on their
// external nodes and surface it on the Cluster status. We also own the
// default/kubernetes Endpoints inside the virtual cluster (the apiserver
// reconciler is disabled for HCP) so external-node pods can reach the
// in-cluster apiserver ClusterIP.
if cluster.Spec.Mode == v1beta1.HCPClusterMode {
if err := c.ensureHCPRegistration(ctx, cluster, token); err != nil {
return err
}
if err := c.ensureHCPKubernetesEndpoints(ctx, cluster); err != nil {
return err
}
}
// Important: if you need to call the Server API of the Virtual Cluster
// this needs to be done AFTER he kubeconfig has been generated
@@ -896,6 +912,13 @@ func (c *ClusterReconciler) bindClusterRoles(ctx context.Context, cluster *v1bet
}
func (c *ClusterReconciler) ensureAgent(ctx context.Context, cluster *v1beta1.Cluster, serviceIP, token string) error {
// hcp mode is BYO-node by design: external (out-of-host-cluster) nodes join
// using the standard K3s installer command surfaced via Status.HCPRegistration.
// k3k therefore does not provision any agent pods on the host cluster.
if cluster.Spec.Mode == v1beta1.HCPClusterMode {
return nil
}
config := agent.NewConfig(cluster, c.Client, c.Scheme)
var agentEnsurer agent.ResourceEnsurer
+216
View File
@@ -0,0 +1,216 @@
package cluster
import (
"context"
"fmt"
"net"
"net/url"
"strconv"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
"github.com/rancher/k3k/pkg/controller/cluster/server"
)
// endpointSliceSkipMirrorLabel is the upstream label that opts an Endpoints
// object out of the kube-controller-manager EndpointSlice mirroring controller.
// The kube-apiserver normally sets it on default/kubernetes (because it
// manages EndpointSlices itself); in HCP mode we want the mirror controller
// to handle slices, so we strip the label.
const endpointSliceSkipMirrorLabel = "endpointslice.kubernetes.io/skip-mirror"
// ensureHCPRegistration computes the K3s installer command external nodes can
// run to join an HCP-mode cluster and stores it on cluster.Status.HCPRegistration.
//
// When the cluster's Service is not externally reachable (no NodePort,
// LoadBalancer or Ingress configured) the command cannot be built; in that
// case the Ready condition is set to False with reason HCPNoExternalEndpoint
// so the operator surfaces the problem without failing the reconciliation.
func (c *ClusterReconciler) ensureHCPRegistration(ctx context.Context, cluster *v1beta1.Cluster, token string) error {
log := ctrl.LoggerFrom(ctx)
url, external, err := server.ServerURL(ctx, c.Client, cluster, "", 0)
if err != nil {
return err
}
if !external {
log.Info("HCP cluster has no externally-routable endpoint; skipping registration command",
"cluster", cluster.Name, "namespace", cluster.Namespace)
meta.SetStatusCondition(&cluster.Status.Conditions, metav1.Condition{
Type: ConditionReady,
Status: metav1.ConditionFalse,
Reason: ReasonHCPNoExternalEndpoint,
Message: "HCP cluster has no external endpoint; set spec.expose.nodePort, spec.expose.loadBalancer or spec.expose.ingress so external nodes can reach the API server",
})
cluster.Status.HCPRegistration = ""
return nil
}
version := cluster.Spec.Version
if version == "" {
version = cluster.Status.HostVersion
}
cluster.Status.HCPRegistration = hcpRegistrationCommand(version, url, token)
return nil
}
// hcpRegistrationCommand returns the standard K3s installer one-liner an
// end-user can copy onto an external host to join an HCP cluster.
func hcpRegistrationCommand(version, serverURL, token string) string {
if version == "" {
return fmt.Sprintf("curl -sfL https://get.k3s.io | K3S_URL=%s K3S_TOKEN=%s sh -", serverURL, token)
}
return fmt.Sprintf("curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=%s K3S_URL=%s K3S_TOKEN=%s sh -",
version, serverURL, token)
}
// ensureHCPKubernetesEndpoints maintains the default/kubernetes Service
// Endpoints inside the virtual cluster, pointing them at the externally
// reachable host:port (NodePort / LoadBalancer / Ingress) so that pods
// scheduled on external worker nodes can reach the in-cluster apiserver
// ClusterIP.
//
// Background: the kube-apiserver normally reconciles default/kubernetes
// Endpoints to its own --advertise-address:--secure-port (the host-cluster
// pod IP and 6443). External worker nodes have no route to the host-cluster
// pod CIDR, so kube-proxy DNAT to that endpoint fails. We disable the
// apiserver reconciler in HCP mode (see serverOptions) and own this
// Endpoints object instead.
func (c *ClusterReconciler) ensureHCPKubernetesEndpoints(ctx context.Context, cluster *v1beta1.Cluster) error {
log := ctrl.LoggerFrom(ctx)
rawURL, external, err := server.ServerURL(ctx, c.Client, cluster, "", 0)
if err != nil {
return err
}
if !external {
// ensureHCPRegistration already surfaces this via Ready=False;
// nothing for us to do here.
return nil
}
host, port, err := parseHCPHostPort(rawURL)
if err != nil {
return fmt.Errorf("parsing HCP server URL %q: %w", rawURL, err)
}
addr, err := hcpEndpointAddress(host)
if err != nil {
return err
}
virtClient, err := newVirtualClient(ctx, c.Client, cluster.Name, cluster.Namespace)
if err != nil {
return fmt.Errorf("creating virtual cluster client: %w", err)
}
endpoints := &corev1.Endpoints{
ObjectMeta: metav1.ObjectMeta{
Name: "kubernetes",
Namespace: metav1.NamespaceDefault,
},
}
_, err = controllerutil.CreateOrUpdate(ctx, virtClient, endpoints, func() error {
// Allow EndpointSlice mirroring; the apiserver may have set
// skip-mirror=true before we disabled its endpoint reconciler.
if endpoints.Labels != nil {
delete(endpoints.Labels, endpointSliceSkipMirrorLabel)
}
endpoints.Subsets = []corev1.EndpointSubset{
{
Addresses: []corev1.EndpointAddress{addr},
Ports: []corev1.EndpointPort{
{
Name: "https",
Port: port,
Protocol: corev1.ProtocolTCP,
},
},
},
}
return nil
})
if err != nil {
return fmt.Errorf("upserting default/kubernetes endpoints in virtual cluster: %w", err)
}
log.V(1).Info("HCP kubernetes endpoints reconciled",
"address", addr.IP, "hostname", addr.Hostname, "port", port)
return nil
}
// parseHCPHostPort extracts the host and port from a server URL produced by
// server.ServerURL. The port defaults to 443 when omitted.
func parseHCPHostPort(rawURL string) (string, int32, error) {
u, err := url.Parse(rawURL)
if err != nil {
return "", 0, err
}
host := u.Hostname()
if host == "" {
return "", 0, fmt.Errorf("missing host in URL %q", rawURL)
}
portStr := u.Port()
var port int32 = 443
if portStr != "" {
p, err := strconv.Atoi(portStr)
if err != nil {
return "", 0, fmt.Errorf("invalid port in URL %q: %w", rawURL, err)
}
if p <= 0 || p > 65535 {
return "", 0, fmt.Errorf("port %d out of range in URL %q", p, rawURL)
}
port = int32(p)
}
return host, port, nil
}
// hcpEndpointAddress builds a corev1.EndpointAddress from the externally
// reachable host. Endpoints require an IP; if the host is a DNS name we
// resolve it and keep the original name as Hostname so logs/events remain
// human-readable.
func hcpEndpointAddress(host string) (corev1.EndpointAddress, error) {
if ip := net.ParseIP(host); ip != nil {
return corev1.EndpointAddress{IP: host}, nil
}
ips, err := net.LookupIP(host)
if err != nil {
return corev1.EndpointAddress{}, fmt.Errorf("HCP endpoint host %q is not an IP and does not resolve: %w", host, err)
}
for _, ip := range ips {
if v4 := ip.To4(); v4 != nil {
return corev1.EndpointAddress{IP: v4.String(), Hostname: host}, nil
}
}
if len(ips) == 0 {
return corev1.EndpointAddress{}, fmt.Errorf("HCP endpoint host %q resolved to no IPs", host)
}
return corev1.EndpointAddress{IP: ips[0].String(), Hostname: host}, nil
}
+200
View File
@@ -0,0 +1,200 @@
package cluster
import (
"context"
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apimachinery/pkg/runtime"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
"github.com/rancher/k3k/pkg/controller"
"github.com/rancher/k3k/pkg/controller/cluster/server"
)
func Test_hcpRegistrationCommand(t *testing.T) {
tests := []struct {
name string
version string
serverURL string
token string
want string
}{
{
name: "with version",
version: "v1.33.1-k3s1",
serverURL: "https://1.2.3.4:30443",
token: "abcd1234",
want: "curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=v1.33.1-k3s1 K3S_URL=https://1.2.3.4:30443 K3S_TOKEN=abcd1234 sh -",
},
{
name: "without version",
version: "",
serverURL: "https://hcp.example.com",
token: "tok",
want: "curl -sfL https://get.k3s.io | K3S_URL=https://hcp.example.com K3S_TOKEN=tok sh -",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := hcpRegistrationCommand(tt.version, tt.serverURL, tt.token)
assert.Equal(t, tt.want, got)
})
}
}
func Test_ensureHCPRegistration(t *testing.T) {
scheme := runtime.NewScheme()
require.NoError(t, corev1.AddToScheme(scheme))
require.NoError(t, v1beta1.AddToScheme(scheme))
cluster := &v1beta1.Cluster{
ObjectMeta: metav1.ObjectMeta{Name: "demo", Namespace: "team-a"},
Spec: v1beta1.ClusterSpec{
Mode: v1beta1.HCPClusterMode,
Version: "v1.33.1-k3s1",
TLSSANs: []string{"hcp.example.com"},
},
Status: v1beta1.ClusterStatus{
TLSSANs: []string{"hcp.example.com"},
},
}
t.Run("nodeport service produces ready-to-copy command", func(t *testing.T) {
svc := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: server.ServiceName(cluster.Name),
Namespace: cluster.Namespace,
},
Spec: corev1.ServiceSpec{
Type: corev1.ServiceTypeNodePort,
ClusterIP: "10.43.0.50",
Ports: []corev1.ServicePort{
{Name: "k3s-server-port", Port: 443, NodePort: 31001},
},
},
}
fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(svc).Build()
r := &ClusterReconciler{Client: fakeClient}
c := cluster.DeepCopy()
require.NoError(t, r.ensureHCPRegistration(context.Background(), c, "join-token-xyz"))
assert.Contains(t, c.Status.HCPRegistration, "K3S_URL=https://hcp.example.com:31001")
assert.Contains(t, c.Status.HCPRegistration, "K3S_TOKEN=join-token-xyz")
assert.Contains(t, c.Status.HCPRegistration, "INSTALL_K3S_VERSION=v1.33.1-k3s1")
assert.True(t, strings.HasPrefix(c.Status.HCPRegistration, "curl -sfL https://get.k3s.io"))
})
t.Run("clusterip-only service sets degraded condition and clears registration", func(t *testing.T) {
svc := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: server.ServiceName(cluster.Name),
Namespace: cluster.Namespace,
},
Spec: corev1.ServiceSpec{
Type: corev1.ServiceTypeClusterIP,
ClusterIP: "10.43.0.50",
Ports: []corev1.ServicePort{
{Name: "k3s-server-port", Port: 443},
},
},
}
fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(svc).Build()
r := &ClusterReconciler{Client: fakeClient}
c := cluster.DeepCopy()
c.Status.HCPRegistration = "stale-value"
require.NoError(t, r.ensureHCPRegistration(context.Background(), c, "ignored"))
assert.Empty(t, c.Status.HCPRegistration)
cond := meta.FindStatusCondition(c.Status.Conditions, ConditionReady)
require.NotNil(t, cond)
assert.Equal(t, metav1.ConditionFalse, cond.Status)
assert.Equal(t, ReasonHCPNoExternalEndpoint, cond.Reason)
})
}
func Test_parseHCPHostPort(t *testing.T) {
tests := []struct {
name string
url string
wantHost string
wantPort int32
wantErr bool
}{
{
name: "ip with explicit port",
url: "https://10.144.101.195:30337",
wantHost: "10.144.101.195",
wantPort: 30337,
},
{
name: "hostname without port defaults to 443",
url: "https://hcp.example.com",
wantHost: "hcp.example.com",
wantPort: 443,
},
{
name: "hostname with explicit port",
url: "https://hcp.example.com:6443",
wantHost: "hcp.example.com",
wantPort: 6443,
},
{
name: "missing host",
url: "https://",
wantErr: true,
},
{
name: "non-numeric port",
url: "https://host:abc",
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
host, port, err := parseHCPHostPort(tt.url)
if tt.wantErr {
require.Error(t, err)
return
}
require.NoError(t, err)
assert.Equal(t, tt.wantHost, host)
assert.Equal(t, tt.wantPort, port)
})
}
}
func Test_hcpEndpointAddress(t *testing.T) {
t.Run("ipv4 literal is passed through", func(t *testing.T) {
got, err := hcpEndpointAddress("10.144.101.195")
require.NoError(t, err)
assert.Equal(t, "10.144.101.195", got.IP)
assert.Empty(t, got.Hostname)
})
t.Run("unresolvable hostname errors", func(t *testing.T) {
_, err := hcpEndpointAddress("definitely-not-a-real-host.invalid")
require.Error(t, err)
})
}
// Compile-time assertion: every reused exported name from the controller
// package below this test file must remain stable. If `controller.K3SImage`
// disappears (refactor), this guards the dependency.
var _ = controller.K3SImage
+45 -5
View File
@@ -11,7 +11,6 @@ import (
"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
"github.com/rancher/k3k/pkg/controller"
"github.com/rancher/k3k/pkg/controller/cluster/agent"
)
// serverConfig are few options from k3s server options that will
@@ -77,10 +76,51 @@ func buildServerConfig(cluster *v1beta1.Cluster, initServer bool, serviceIP, tok
serverConfig.Server = "https://" + serviceIP
}
if cluster.Spec.Mode != agent.VirtualNodeMode {
serverConfig.DisableAgent = true
serverConfig.EgressSelectorMode = "disabled"
serverConfig.Disable = []string{"servicelb", "traefik", "metrics-server", "local-storage"}
// shared and hcp modes both run K3s with --disable-agent (agentless server).
// hcp additionally relies on this to satisfy the PRD requirement that the
// control plane never runs a kubelet and is not enumerated as a node.
if cluster.Spec.Mode != v1beta1.VirtualClusterMode {
opts = opts + "disable-agent: true\ndisable:\n- servicelb\n- traefik\n- metrics-server\n- local-storage\n"
}
// In shared mode workloads run on the host cluster, so the apiserver pod
// can reach them directly via the host pod network and the egress
// selector is unnecessary.
//
// In hcp mode the apiserver pod has NO route to the virtual cluster's
// pod CIDR (which only exists on joined external worker nodes), and the
// kube-apiserver bypasses kube-proxy when calling webhooks / proxying
// to pods: it resolves Service -> Endpoints itself and dials the Pod IP
// directly. We therefore tunnel apiserver egress through the WebSocket
// each k3s-agent maintains back to the server.
//
// We pick "cluster" rather than "pod" or "agent" because the agent-side
// authorizer differs by mode (k3s pkg/agent/tunnel/tunnel.go):
// - agent: only kubelet calls are tunneled; pod-IP dials go direct
// and fail in HCP (no route to virtual pod CIDR).
// - pod: authorizer only allows pod IPs the agent has *already
// watched*. A newly-created pod's IP is rejected with
// "connect not allowed", which terminates the entire
// remotedialer session and 502s in-flight kubelet streams
// -> kubectl logs / exec / webhooks become flaky.
// - cluster: authorizer pre-populates the cluster CIDR + node IPs as
// non-hostNet entries, so every pod IP and every node port
// is permitted. No race, no per-port allowlist. This is
// what we want for a managed control plane.
switch cluster.Spec.Mode {
case v1beta1.SharedClusterMode:
opts = opts + "egress-selector-mode: disabled\n"
case v1beta1.HCPClusterMode:
opts = opts + "egress-selector-mode: cluster\n"
}
// In hcp mode the apiserver pod IP is unreachable from external worker
// nodes, so the kube-apiserver's default lease-based endpoint reconciler
// would publish a broken default/kubernetes Endpoints (advertise-address +
// secure-port). Disable it so K3k can own that Endpoints object and point
// it at the externally-reachable host:port (NodePort / LB / Ingress).
if cluster.Spec.Mode == v1beta1.HCPClusterMode {
opts = opts + "kube-apiserver-arg:\n- endpoint-reconciler-type=none\n"
}
return serverConfig
+111
View File
@@ -0,0 +1,111 @@
package server
import (
"context"
"fmt"
"slices"
"github.com/sirupsen/logrus"
corev1 "k8s.io/api/core/v1"
networkingv1 "k8s.io/api/networking/v1"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/client"
"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
)
// ServerURL returns the URL at which the K3s API server of a virtual cluster
// is reachable. The second return value reports whether that URL is routable
// from outside the host cluster (true for NodePort/LoadBalancer/Ingress, false
// for plain ClusterIP exposition).
//
// hostServerIP is used as the address when the underlying Service is a
// NodePort. serverPort, when non-zero, overrides the port discovered from the
// Service.
func ServerURL(ctx context.Context, c client.Client, cluster *v1beta1.Cluster, hostServerIP string, serverPort int) (string, bool, error) {
key := types.NamespacedName{
Name: ServiceName(cluster.Name),
Namespace: cluster.Namespace,
}
var k3kService corev1.Service
if err := c.Get(ctx, key, &k3kService); err != nil {
return "", false, err
}
ip := k3kService.Spec.ClusterIP
port := int32(httpsPort)
external := false
if len(k3kService.Spec.Ports) == 0 {
logrus.Warn("No ports exposed by the cluster service.")
}
switch k3kService.Spec.Type {
case corev1.ServiceTypeNodePort:
ip = hostServerIP
external = true
if len(k3kService.Spec.Ports) > 0 {
port = k3kService.Spec.Ports[0].NodePort
}
case corev1.ServiceTypeLoadBalancer:
external = true
if len(k3kService.Status.LoadBalancer.Ingress) > 0 {
ip = k3kService.Status.LoadBalancer.Ingress[0].IP
} else {
logrus.Warn("No ingress found in LoadBalancer service.")
}
if len(k3kService.Spec.Ports) > 0 {
port = k3kService.Spec.Ports[0].Port
}
}
if serverPort != 0 {
port = int32(serverPort)
}
if !slices.Contains(cluster.Status.TLSSANs, ip) {
logrus.Warnf("IP %s not in tlsSANs.", ip)
if len(cluster.Spec.TLSSANs) > 0 {
logrus.Warnf("Using the first TLS SAN in the spec as a fallback: %s", cluster.Spec.TLSSANs[0])
ip = cluster.Spec.TLSSANs[0]
} else if len(cluster.Status.TLSSANs) > 0 {
logrus.Warnf("No explicit tlsSANs specified. Trying to use the first TLS SAN in the status: %s", cluster.Status.TLSSANs[0])
ip = cluster.Status.TLSSANs[0]
} else {
logrus.Warn("IP not found in tlsSANs. This could cause issue with the certificate validation.")
}
}
url := "https://" + ip
if port != httpsPort {
url = fmt.Sprintf("%s:%d", url, port)
}
// if ingress is specified, use the ingress host
if cluster.Spec.Expose != nil && cluster.Spec.Expose.Ingress != nil {
var k3kIngress networkingv1.Ingress
ingressKey := types.NamespacedName{
Name: IngressName(cluster.Name),
Namespace: cluster.Namespace,
}
if err := c.Get(ctx, ingressKey, &k3kIngress); err != nil {
return "", external, err
}
if len(k3kIngress.Spec.Rules) > 0 {
url = fmt.Sprintf("https://%s", k3kIngress.Spec.Rules[0].Host)
external = true
}
}
return url, external, nil
}
+4 -3
View File
@@ -22,7 +22,6 @@ import (
"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
"github.com/rancher/k3k/pkg/controller"
"github.com/rancher/k3k/pkg/controller/cluster/agent"
"github.com/rancher/k3k/pkg/controller/cluster/mounts"
)
@@ -250,8 +249,10 @@ func (s *Server) podSpec(ctx context.Context, image, name string, persistent boo
},
},
}
// start the pod unprivileged in shared mode
if s.mode == agent.VirtualNodeMode {
// virtual mode runs an embedded kubelet inside the server pod and therefore
// requires Privileged. shared and hcp modes are agentless (no kubelet) and
// run unprivileged.
if s.mode == string(v1beta1.VirtualClusterMode) {
podSpec.Containers[0].SecurityContext = &corev1.SecurityContext{
Privileged: ptr.To(true),
}
+3 -2
View File
@@ -19,7 +19,7 @@ safe_mode() {
CURRENT_IP=$(cat /var/lib/rancher/k3s/k3k-node-ip)
fi
if [ -z "$CURRENT_IP" ] || [ "$CURRENT_IP" = "$POD_IP" ] || [ {{.K3K_MODE}} != "virtual" ]; then
if [ -z "$CURRENT_IP" ] || [ "$CURRENT_IP" = "$POD_IP" ] || [ "{{.K3K_MODE}}" = "shared" ] || [ "{{.K3K_MODE}}" = "hcp" ]; then
return
fi
@@ -116,7 +116,8 @@ configure_cgroups() {
fi
# only configure the cgroups if the runtime used is the default and the mode is virtual
if [ -n "$runtime_class" ] || [ "{{.K3K_MODE}}" != "virtual" ]; then
# shared and hcp run agentless (no kubelet) and don't need cgroup overrides.
if [ -n "$runtime_class" ] || [ "{{.K3K_MODE}}" != "virtual" ]; then
return
fi
+6 -5
View File
@@ -19,11 +19,12 @@ const (
ConditionReady = "Ready"
// Condition Reasons
ReasonValidationFailed = "ValidationFailed"
ReasonProvisioning = "Provisioning"
ReasonProvisioned = "Provisioned"
ReasonProvisioningFailed = "ProvisioningFailed"
ReasonTerminating = "Terminating"
ReasonValidationFailed = "ValidationFailed"
ReasonProvisioning = "Provisioning"
ReasonProvisioned = "Provisioned"
ReasonProvisioningFailed = "ProvisioningFailed"
ReasonTerminating = "Terminating"
ReasonHCPNoExternalEndpoint = "HCPNoExternalEndpoint"
)
func (c *ClusterReconciler) updateStatus(ctx context.Context, cluster *v1beta1.Cluster, reconcileErr error) {
+1 -88
View File
@@ -3,18 +3,12 @@ package kubeconfig
import (
"context"
"crypto/x509"
"fmt"
"slices"
"time"
"github.com/sirupsen/logrus"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apiserver/pkg/authentication/user"
"sigs.k8s.io/controller-runtime/pkg/client"
certutil "github.com/rancher/dynamiclistener/cert"
corev1 "k8s.io/api/core/v1"
networkingv1 "k8s.io/api/networking/v1"
clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
@@ -60,7 +54,7 @@ func (k *KubeConfig) Generate(ctx context.Context, client client.Client, cluster
return nil, err
}
url, err := getURLFromService(ctx, client, cluster, hostServerIP, port)
url, _, err := server.ServerURL(ctx, client, cluster, hostServerIP, port)
if err != nil {
return nil, err
}
@@ -93,84 +87,3 @@ func NewConfig(url string, serverCA, clientCert, clientKey []byte) *clientcmdapi
return config
}
func getURLFromService(ctx context.Context, client client.Client, cluster *v1beta1.Cluster, hostServerIP string, serverPort int) (string, error) {
// get the server service to extract the right IP
key := types.NamespacedName{
Name: server.ServiceName(cluster.Name),
Namespace: cluster.Namespace,
}
var k3kService corev1.Service
if err := client.Get(ctx, key, &k3kService); err != nil {
return "", err
}
ip := k3kService.Spec.ClusterIP
port := int32(443)
if len(k3kService.Spec.Ports) == 0 {
logrus.Warn("No ports exposed by the cluster service.")
}
switch k3kService.Spec.Type {
case corev1.ServiceTypeNodePort:
ip = hostServerIP
if len(k3kService.Spec.Ports) > 0 {
port = k3kService.Spec.Ports[0].NodePort
}
case corev1.ServiceTypeLoadBalancer:
if len(k3kService.Status.LoadBalancer.Ingress) > 0 {
ip = k3kService.Status.LoadBalancer.Ingress[0].IP
} else {
logrus.Warn("No ingress found in LoadBalancer service.")
}
if len(k3kService.Spec.Ports) > 0 {
port = k3kService.Spec.Ports[0].Port
}
}
if serverPort != 0 {
port = int32(serverPort)
}
if !slices.Contains(cluster.Status.TLSSANs, ip) {
logrus.Warnf("IP %s not in tlsSANs.", ip)
if len(cluster.Spec.TLSSANs) > 0 {
logrus.Warnf("Using the first TLS SAN in the spec as a fallback: %s", cluster.Spec.TLSSANs[0])
ip = cluster.Spec.TLSSANs[0]
} else if len(cluster.Status.TLSSANs) > 0 {
logrus.Warnf("No explicit tlsSANs specified. Trying to use the first TLS SAN in the status: %s", cluster.Status.TLSSANs[0])
ip = cluster.Status.TLSSANs[0]
} else {
logrus.Warn("IP not found in tlsSANs. This could cause issue with the certificate validation.")
}
}
url := "https://" + ip
if port != 443 {
url = fmt.Sprintf("%s:%d", url, port)
}
// if ingress is specified, use the ingress host
if cluster.Spec.Expose != nil && cluster.Spec.Expose.Ingress != nil {
var k3kIngress networkingv1.Ingress
ingressKey := types.NamespacedName{
Name: server.IngressName(cluster.Name),
Namespace: cluster.Namespace,
}
if err := client.Get(ctx, ingressKey, &k3kIngress); err != nil {
return "", err
}
url = fmt.Sprintf("https://%s", k3kIngress.Spec.Rules[0].Host)
}
return url, nil
}