From cc61345ddabaddd9d2e57b87a33b47b92dc75735 Mon Sep 17 00:00:00 2001 From: Enrico Candino Date: Mon, 15 Jun 2026 15:11:11 +0200 Subject: [PATCH] fixed template and comments --- pkg/controller/cluster/server/config.go | 38 +++++++---------------- pkg/controller/cluster/server/template.go | 2 +- 2 files changed, 12 insertions(+), 28 deletions(-) diff --git a/pkg/controller/cluster/server/config.go b/pkg/controller/cluster/server/config.go index 31a30549..c24279d1 100644 --- a/pkg/controller/cluster/server/config.go +++ b/pkg/controller/cluster/server/config.go @@ -85,39 +85,23 @@ func buildServerConfig(cluster *v1beta1.Cluster, initServer bool, serviceIP, tok serverConfig.Disable = []string{"servicelb", "traefik", "metrics-server", "local-storage"} case v1beta1.HCPClusterMode: serverConfig.DisableAgent = true + // Tunnel apiserver egress through the k3s-agent WebSocket: the + // apiserver has no route to the virtual cluster's pod CIDR and + // bypasses kube-proxy when dialing pod IPs (webhooks, log/exec). + // "cluster" is the only safe mode — "agent" lets pod dials go + // direct (no route, fails); "pod" only permits pod IPs the agent + // has already watched, so a newly-created pod's IP is rejected + // and tears down the remotedialer session, making kubelet streams + // flaky. See k3s pkg/agent/tunnel/tunnel.go. serverConfig.EgressSelectorMode = "cluster" - // Disable it so K3k can own that Endpoints object and point - // it at the externally-reachable host:port (NodePort / LB / Ingress). + // Disable the apiserver's built-in endpoint reconciler so K3k can + // own default/kubernetes Endpoints and point it at the externally + // reachable host:port (NodePort / LB / Ingress). serverConfig.KubeApiServerArg = append(serverConfig.KubeApiServerArg, "endpoint-reconciler-type=none") case v1beta1.VirtualClusterMode: // no extra config for virtual mode } - // In shared mode workloads run on the host cluster, so the apiserver pod - // can reach them directly via the host pod network and the egress - // selector is unnecessary. - // - // In hcp mode the apiserver pod has NO route to the virtual cluster's - // pod CIDR (which only exists on joined external worker nodes), and the - // kube-apiserver bypasses kube-proxy when calling webhooks / proxying - // to pods: it resolves Service -> Endpoints itself and dials the Pod IP - // directly. We therefore tunnel apiserver egress through the WebSocket - // each k3s-agent maintains back to the server. - // - // We pick "cluster" rather than "pod" or "agent" because the agent-side - // authorizer differs by mode (k3s pkg/agent/tunnel/tunnel.go): - // - agent: only kubelet calls are tunneled; pod-IP dials go direct - // and fail in HCP (no route to virtual pod CIDR). - // - pod: authorizer only allows pod IPs the agent has *already - // watched*. A newly-created pod's IP is rejected with - // "connect not allowed", which terminates the entire - // remotedialer session and 502s in-flight kubelet streams - // -> kubectl logs / exec / webhooks become flaky. - // - cluster: authorizer pre-populates the cluster CIDR + node IPs as - // non-hostNet entries, so every pod IP and every node port - // is permitted. No race, no per-port allowlist. This is - // what we want for a managed control plane. - return serverConfig } diff --git a/pkg/controller/cluster/server/template.go b/pkg/controller/cluster/server/template.go index 774fb985..0fd48d4e 100644 --- a/pkg/controller/cluster/server/template.go +++ b/pkg/controller/cluster/server/template.go @@ -19,7 +19,7 @@ safe_mode() { CURRENT_IP=$(cat /var/lib/rancher/k3s/k3k-node-ip) fi - if [ -z "$CURRENT_IP" ] || [ "$CURRENT_IP" = "$POD_IP" ] || [ "{{.K3K_MODE}}" = "shared" ] || [ "{{.K3K_MODE}}" = "hcp" ]; then + if [ -z "$CURRENT_IP" ] || [ "$CURRENT_IP" = "$POD_IP" ] || [ "{{.K3K_MODE}}" != "virtual" ]; then return fi