Add HCP (Hosted Control Plane) support

Introduce hosted control plane mode for k3k virtual clusters, including
API types, controller logic, server endpoint handling, CLI flags,
CRD updates, kubeconfig generation, and examples.

Co-Authored-By: RuFlo <ruv@ruv.net>

pkg/controller/cluster/server/config.go

@@ -11,7 +11,6 @@ import (
 
 	"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
 	"github.com/rancher/k3k/pkg/controller"
-	"github.com/rancher/k3k/pkg/controller/cluster/agent"
 )
 
 // serverConfig are few options from k3s server options that will
@@ -77,10 +76,51 @@ func buildServerConfig(cluster *v1beta1.Cluster, initServer bool, serviceIP, tok
 		serverConfig.Server = "https://" + serviceIP
 	}
 
-	if cluster.Spec.Mode != agent.VirtualNodeMode {
-		serverConfig.DisableAgent = true
-		serverConfig.EgressSelectorMode = "disabled"
-		serverConfig.Disable = []string{"servicelb", "traefik", "metrics-server", "local-storage"}
+	// shared and hcp modes both run K3s with --disable-agent (agentless server).
+	// hcp additionally relies on this to satisfy the PRD requirement that the
+	// control plane never runs a kubelet and is not enumerated as a node.
+	if cluster.Spec.Mode != v1beta1.VirtualClusterMode {
+		opts = opts + "disable-agent: true\ndisable:\n- servicelb\n- traefik\n- metrics-server\n- local-storage\n"
+	}
+
+	// In shared mode workloads run on the host cluster, so the apiserver pod
+	// can reach them directly via the host pod network and the egress
+	// selector is unnecessary.
+	//
+	// In hcp mode the apiserver pod has NO route to the virtual cluster's
+	// pod CIDR (which only exists on joined external worker nodes), and the
+	// kube-apiserver bypasses kube-proxy when calling webhooks / proxying
+	// to pods: it resolves Service -> Endpoints itself and dials the Pod IP
+	// directly. We therefore tunnel apiserver egress through the WebSocket
+	// each k3s-agent maintains back to the server.
+	//
+	// We pick "cluster" rather than "pod" or "agent" because the agent-side
+	// authorizer differs by mode (k3s pkg/agent/tunnel/tunnel.go):
+	//   - agent:   only kubelet calls are tunneled; pod-IP dials go direct
+	//              and fail in HCP (no route to virtual pod CIDR).
+	//   - pod:     authorizer only allows pod IPs the agent has *already
+	//              watched*. A newly-created pod's IP is rejected with
+	//              "connect not allowed", which terminates the entire
+	//              remotedialer session and 502s in-flight kubelet streams
+	//              -> kubectl logs / exec / webhooks become flaky.
+	//   - cluster: authorizer pre-populates the cluster CIDR + node IPs as
+	//              non-hostNet entries, so every pod IP and every node port
+	//              is permitted. No race, no per-port allowlist. This is
+	//              what we want for a managed control plane.
+	switch cluster.Spec.Mode {
+	case v1beta1.SharedClusterMode:
+		opts = opts + "egress-selector-mode: disabled\n"
+	case v1beta1.HCPClusterMode:
+		opts = opts + "egress-selector-mode: cluster\n"
+	}
+
+	// In hcp mode the apiserver pod IP is unreachable from external worker
+	// nodes, so the kube-apiserver's default lease-based endpoint reconciler
+	// would publish a broken default/kubernetes Endpoints (advertise-address +
+	// secure-port). Disable it so K3k can own that Endpoints object and point
+	// it at the externally-reachable host:port (NodePort / LB / Ingress).
+	if cluster.Spec.Mode == v1beta1.HCPClusterMode {
+		opts = opts + "kube-apiserver-arg:\n- endpoint-reconciler-type=none\n"
 	}
 
 	return serverConfig

pkg/controller/cluster/server/endpoint.go

@@ -0,0 +1,111 @@
+package server
+
+import (
+	"context"
+	"fmt"
+	"slices"
+
+	"github.com/sirupsen/logrus"
+	corev1 "k8s.io/api/core/v1"
+	networkingv1 "k8s.io/api/networking/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
+)
+
+// ServerURL returns the URL at which the K3s API server of a virtual cluster
+// is reachable. The second return value reports whether that URL is routable
+// from outside the host cluster (true for NodePort/LoadBalancer/Ingress, false
+// for plain ClusterIP exposition).
+//
+// hostServerIP is used as the address when the underlying Service is a
+// NodePort. serverPort, when non-zero, overrides the port discovered from the
+// Service.
+func ServerURL(ctx context.Context, c client.Client, cluster *v1beta1.Cluster, hostServerIP string, serverPort int) (string, bool, error) {
+	key := types.NamespacedName{
+		Name:      ServiceName(cluster.Name),
+		Namespace: cluster.Namespace,
+	}
+
+	var k3kService corev1.Service
+	if err := c.Get(ctx, key, &k3kService); err != nil {
+		return "", false, err
+	}
+
+	ip := k3kService.Spec.ClusterIP
+	port := int32(httpsPort)
+	external := false
+
+	if len(k3kService.Spec.Ports) == 0 {
+		logrus.Warn("No ports exposed by the cluster service.")
+	}
+
+	switch k3kService.Spec.Type {
+	case corev1.ServiceTypeNodePort:
+		ip = hostServerIP
+		external = true
+
+		if len(k3kService.Spec.Ports) > 0 {
+			port = k3kService.Spec.Ports[0].NodePort
+		}
+	case corev1.ServiceTypeLoadBalancer:
+		external = true
+
+		if len(k3kService.Status.LoadBalancer.Ingress) > 0 {
+			ip = k3kService.Status.LoadBalancer.Ingress[0].IP
+		} else {
+			logrus.Warn("No ingress found in LoadBalancer service.")
+		}
+
+		if len(k3kService.Spec.Ports) > 0 {
+			port = k3kService.Spec.Ports[0].Port
+		}
+	}
+
+	if serverPort != 0 {
+		port = int32(serverPort)
+	}
+
+	if !slices.Contains(cluster.Status.TLSSANs, ip) {
+		logrus.Warnf("IP %s not in tlsSANs.", ip)
+
+		if len(cluster.Spec.TLSSANs) > 0 {
+			logrus.Warnf("Using the first TLS SAN in the spec as a fallback: %s", cluster.Spec.TLSSANs[0])
+
+			ip = cluster.Spec.TLSSANs[0]
+		} else if len(cluster.Status.TLSSANs) > 0 {
+			logrus.Warnf("No explicit tlsSANs specified. Trying to use the first TLS SAN in the status: %s", cluster.Status.TLSSANs[0])
+
+			ip = cluster.Status.TLSSANs[0]
+		} else {
+			logrus.Warn("IP not found in tlsSANs. This could cause issue with the certificate validation.")
+		}
+	}
+
+	url := "https://" + ip
+	if port != httpsPort {
+		url = fmt.Sprintf("%s:%d", url, port)
+	}
+
+	// if ingress is specified, use the ingress host
+	if cluster.Spec.Expose != nil && cluster.Spec.Expose.Ingress != nil {
+		var k3kIngress networkingv1.Ingress
+
+		ingressKey := types.NamespacedName{
+			Name:      IngressName(cluster.Name),
+			Namespace: cluster.Namespace,
+		}
+
+		if err := c.Get(ctx, ingressKey, &k3kIngress); err != nil {
+			return "", external, err
+		}
+
+		if len(k3kIngress.Spec.Rules) > 0 {
+			url = fmt.Sprintf("https://%s", k3kIngress.Spec.Rules[0].Host)
+			external = true
+		}
+	}
+
+	return url, external, nil
+}

pkg/controller/cluster/server/server.go

@@ -22,7 +22,6 @@ import (
 
 	"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
 	"github.com/rancher/k3k/pkg/controller"
-	"github.com/rancher/k3k/pkg/controller/cluster/agent"
 	"github.com/rancher/k3k/pkg/controller/cluster/mounts"
 )
 
@@ -250,8 +249,10 @@ func (s *Server) podSpec(ctx context.Context, image, name string, persistent boo
 			},
 		},
 	}
-	// start the pod unprivileged in shared mode
-	if s.mode == agent.VirtualNodeMode {
+	// virtual mode runs an embedded kubelet inside the server pod and therefore
+	// requires Privileged. shared and hcp modes are agentless (no kubelet) and
+	// run unprivileged.
+	if s.mode == string(v1beta1.VirtualClusterMode) {
 		podSpec.Containers[0].SecurityContext = &corev1.SecurityContext{
 			Privileged: ptr.To(true),
 		}

pkg/controller/cluster/server/template.go

@@ -19,7 +19,7 @@ safe_mode() {
         CURRENT_IP=$(cat /var/lib/rancher/k3s/k3k-node-ip)
     fi
 
-    if [ -z "$CURRENT_IP" ] || [ "$CURRENT_IP" = "$POD_IP" ] || [ {{.K3K_MODE}} != "virtual" ]; then
+    if [ -z "$CURRENT_IP" ] || [ "$CURRENT_IP" = "$POD_IP" ] || [ "{{.K3K_MODE}}" = "shared" ] || [ "{{.K3K_MODE}}" = "hcp" ]; then
         return
     fi
 
@@ -116,7 +116,8 @@ configure_cgroups() {
 	fi
 
 	# only configure the cgroups if the runtime used is the default and the mode is virtual
-	if [ -n "$runtime_class" ] || [ "{{.K3K_MODE}}" != "virtual" ]; then
+	# shared and hcp run agentless (no kubelet) and don't need cgroup overrides.
+	if [ -n "$runtime_class" ] || [ "{{.K3K_MODE}}" != "virtual" ]; then	
 		return
 	fi