Added HCP (Hosted Control Plane) mode (#876)

* Add HCP (Hosted Control Plane) support

Introduce hosted control plane mode for k3k virtual clusters, including
API types, controller logic, server endpoint handling, CLI flags,
CRD updates, kubeconfig generation, and examples.

Co-Authored-By: RuFlo <ruv@ruv.net>

* removed hcpRegitration command

added HCP conformance tests

warning for hcp

fix multi-VM HCP conformance test networking

  Both QEMU workers booted with `-net user` and ended up registering the
  same InternalIP (10.0.2.15) because each VM gets its own isolated NAT
  slirp. Flannel propagated this to `public-ip` on both nodes, so VXLAN
  could not tunnel between workers and any cross-node pod traffic broke
  (89 failed / 335 passed of 424 conformance specs).

  Replace user-mode networking with a Linux bridge (k3kbr0,
  192.168.100.0/24) and one TAP device per VM, so the two workers share
  an L2 segment with unique routable IPs. NAT outbound from the bridge
  keeps internet access working for image pulls.

  Also set unique hostnames via cloud-init (worker-1/worker-2) and drop
  the `--node-name` flag from INSTALL_K3S_EXEC, since k3s now picks the
  correct node name from the OS hostname on its own.

  Bump hydrophone back to `--parallel 4` to match the single-VM job
  (parallelism was reduced earlier when the failure was thought to be
  resource-related).

added HCP print command

updated crds

adding e2e tests

Refactor selectNonLoopbackSAN function to accept SANs directly and update related logic in ensureHCPRegistration

* Update agent flag validation and enhance ingress host check with a warning log

Refactor descriptions for cluster provisioning mode and role in CRDs and documentation

Refactor logging in ServerURL function to use controller-runtime logger

Rename selectNonLoopbackSAN to findNonLoopbackSAN for clarity and update references

Refactor ServerURL function and related code to remove unused parameters and improve clarity

Remove unused imports from kubeconfig.go to improve code clarity

* suggested changes

* fix comment

* fix test

---------

Co-authored-by: jpgouin <jeanphilippe.gouin@suse.com>
Co-authored-by: RuFlo <ruv@ruv.net>

cli/cmds/cluster_create.go

@@ -87,10 +87,14 @@ func createAction(appCtx *AppContext, config *CreateConfig) func(cmd *cobra.Comm
 			return errors.New("invalid cluster name")
 		}
 
-		if config.mode == string(v1beta1.SharedClusterMode) && config.agents != 0 {
+		if config.agents != 0 && config.mode != string(v1beta1.VirtualClusterMode) {
 			return errors.New("invalid flag, --agents flag is only allowed in virtual mode")
 		}
 
+		if config.mode == string(v1beta1.HCPClusterMode) {
+			logrus.Warn("HCP (Hosted Control Plane) mode is experimental.")
+		}
+
 		namespace := appCtx.Namespace(name)
 
 		if err := createNamespace(ctx, client, namespace, config.policy); err != nil {
@@ -194,10 +198,34 @@ func createAction(appCtx *AppContext, config *CreateConfig) func(cmd *cobra.Comm
 			return err
 		}
 
-		return writeKubeconfigFile(cluster, kubeconfig, "")
+		if err := writeKubeconfigFile(cluster, kubeconfig, ""); err != nil {
+			return err
+		}
+
+		if cluster.Spec.Mode == v1beta1.HCPClusterMode {
+			printHCPJoinInstructions(cluster, kubeconfig)
+		}
+
+		return nil
 	}
 }
 
+func printHCPJoinInstructions(cluster *v1beta1.Cluster, kc *clientcmdapi.Config) {
+	tokenSecretName := k3kcluster.TokenSecretName(cluster.Name)
+	serverURL := kc.Clusters["default"].Server
+
+	logrus.Infof(`To join an external worker node to this HCP cluster:
+
+	1. On this machine, fetch the cluster token:
+
+		kubectl get secret -n %s %s -o jsonpath='{.data.token}' | base64 -d
+
+	2. On the worker node, run (replace <TOKEN> with the value from step 1):
+
+		curl -sfL https://get.k3s.io | K3S_URL=%s K3S_TOKEN=<TOKEN> sh -
+`, cluster.Namespace, tokenSecretName, serverURL)
+}
+
 func newCluster(name, namespace string, config *CreateConfig) (*v1beta1.Cluster, error) {
 	var storageRequestSize *resource.Quantity
 	if config.storageRequestSize != "" {

cli/cmds/cluster_create_flags.go

@@ -29,7 +29,7 @@ func createFlags(cmd *cobra.Command, cfg *CreateConfig) {
 	cmd.Flags().StringArrayVar(&cfg.labels, "labels", []string{}, "Labels to add to the cluster object (e.g. key=value)")
 	cmd.Flags().StringArrayVar(&cfg.annotations, "annotations", []string{}, "Annotations to add to the cluster object (e.g. key=value)")
 	cmd.Flags().StringVar(&cfg.version, "version", "", "k3s version")
-	cmd.Flags().StringVar(&cfg.mode, "mode", "shared", "k3k mode type (shared, virtual)")
+	cmd.Flags().StringVar(&cfg.mode, "mode", "shared", "k3k mode type (shared, virtual, hcp)")
 	cmd.Flags().StringVar(&cfg.kubeconfigServerHost, "kubeconfig-server", "", "override the kubeconfig server host")
 	cmd.Flags().StringVar(&cfg.policy, "policy", "", "The policy to create the cluster in")
 	cmd.Flags().StringVar(&cfg.customCertsPath, "custom-certs", "", "The path for custom certificate directory")
@@ -43,30 +43,39 @@ func createFlags(cmd *cobra.Command, cfg *CreateConfig) {
 	}
 }
 
+var validPersistenceModes = map[v1beta1.PersistenceMode]struct{}{
+	v1beta1.EphemeralPersistenceMode: {},
+	v1beta1.DynamicPersistenceMode:   {},
+}
+
+var validClusterModes = map[v1beta1.ClusterMode]struct{}{
+	v1beta1.VirtualClusterMode: {},
+	v1beta1.SharedClusterMode:  {},
+	v1beta1.HCPClusterMode:     {},
+}
+
 func validateCreateConfig(cfg *CreateConfig) error {
 	if cfg.servers <= 0 {
-		return errors.New("invalid number of servers")
+		return errors.New("invalid number of servers: must be 1 or more")
 	}
 
 	if cfg.persistenceType != "" {
-		switch v1beta1.PersistenceMode(cfg.persistenceType) {
-		case v1beta1.EphemeralPersistenceMode, v1beta1.DynamicPersistenceMode:
-			return nil
-		default:
+		persistenceMode := v1beta1.PersistenceMode(cfg.persistenceType)
+		if _, found := validPersistenceModes[persistenceMode]; !found {
 			return errors.New(`persistence-type should be one of "dynamic" or "ephemeral"`)
 		}
 	}
 
-	if _, err := resource.ParseQuantity(cfg.storageRequestSize); err != nil {
-		return errors.New(`invalid storage size, should be a valid resource quantity e.g "10Gi"`)
+	if cfg.storageRequestSize != "" {
+		if _, err := resource.ParseQuantity(cfg.storageRequestSize); err != nil {
+			return errors.New(`invalid storage size, should be a valid resource quantity e.g "10Gi"`)
+		}
 	}
 
 	if cfg.mode != "" {
-		switch cfg.mode {
-		case string(v1beta1.VirtualClusterMode), string(v1beta1.SharedClusterMode):
-			return nil
-		default:
-			return errors.New(`mode should be one of "shared" or "virtual"`)
+		clusterMode := v1beta1.ClusterMode(cfg.mode)
+		if _, found := validClusterModes[clusterMode]; !found {
+			return errors.New(`mode should be one of "shared", "virtual" or "hcp"`)
 		}
 	}
 

cli/cmds/cluster_create_flags_test.go

@@ -0,0 +1,106 @@
+package cmds
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_validateCreateConfig(t *testing.T) {
+	tests := []struct {
+		name    string
+		cfg     CreateConfig
+		wantErr string
+	}{
+		{
+			name: "valid full config",
+			cfg: CreateConfig{
+				servers:            1,
+				persistenceType:    "dynamic",
+				storageRequestSize: "10Gi",
+				mode:               "shared",
+			},
+		},
+		{
+			name: "empty storage size",
+			cfg: CreateConfig{
+				servers:            1,
+				storageRequestSize: "",
+			},
+		},
+		{
+			name: "zero servers",
+			cfg: CreateConfig{
+				servers: 0,
+			},
+			wantErr: "invalid number of servers: must be 1 or more",
+		},
+		{
+			name: "negative servers",
+			cfg: CreateConfig{
+				servers: -1,
+			},
+			wantErr: "invalid number of servers: must be 1 or more",
+		},
+		{
+			name: "empty persistence type",
+			cfg: CreateConfig{
+				servers:            1,
+				persistenceType:    "",
+				storageRequestSize: "10Gi",
+			},
+		},
+		{
+			name: "invalid persistence type",
+			cfg: CreateConfig{
+				servers:         1,
+				persistenceType: "foo",
+			},
+			wantErr: `persistence-type should be one of "dynamic" or "ephemeral"`,
+		},
+		{
+			name: "invalid storage size",
+			cfg: CreateConfig{
+				servers:            1,
+				storageRequestSize: "abc",
+			},
+			wantErr: `invalid storage size, should be a valid resource quantity e.g "10Gi"`,
+		},
+		{
+			name: "empty mode",
+			cfg: CreateConfig{
+				servers:            1,
+				mode:               "",
+				storageRequestSize: "10Gi",
+			},
+		},
+		{
+			name: "invalid mode",
+			cfg: CreateConfig{
+				servers:            1,
+				mode:               "foo",
+				storageRequestSize: "10Gi",
+			},
+			wantErr: `mode should be one of "shared", "virtual" or "hcp"`,
+		},
+		{
+			name: "valid hcp mode",
+			cfg: CreateConfig{
+				servers:            1,
+				mode:               "hcp",
+				storageRequestSize: "10Gi",
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := validateCreateConfig(&tt.cfg)
+			if tt.wantErr != "" {
+				assert.EqualError(t, err, tt.wantErr)
+			} else {
+				assert.NoError(t, err)
+			}
+		})
+	}
+}

cli/cmds/kubeconfig.go

@@ -5,7 +5,6 @@ import (
 	"net/url"
 	"os"
 	"path/filepath"
-	"strings"
 	"time"
 
 	"github.com/sirupsen/logrus"
@@ -89,14 +88,12 @@ func generate(appCtx *AppContext, cfg *GenerateKubeconfigConfig) func(cmd *cobra
 			return err
 		}
 
-		url, err := url.Parse(appCtx.RestConfig.Host)
+		host, err := resolveServerHost(appCtx.RestConfig.Host, cfg.kubeconfigServerHost)
 		if err != nil {
 			return err
 		}
 
-		host := strings.Split(url.Host, ":")
 		if cfg.kubeconfigServerHost != "" {
-			host = []string{cfg.kubeconfigServerHost}
 			cfg.altNames = append(cfg.altNames, cfg.kubeconfigServerHost)
 		}
 
@@ -118,16 +115,40 @@ func generate(appCtx *AppContext, cfg *GenerateKubeconfigConfig) func(cmd *cobra
 		var kubeconfig *clientcmdapi.Config
 
 		if err := retry.OnError(controller.Backoff, apierrors.IsNotFound, func() error {
-			kubeconfig, err = kubeCfg.Generate(ctx, client, &cluster, host[0])
+			kubeconfig, err = kubeCfg.Generate(ctx, client, &cluster, host)
 			return err
 		}); err != nil {
 			return err
 		}
 
-		return writeKubeconfigFile(&cluster, kubeconfig, cfg.configName)
+		if err := writeKubeconfigFile(&cluster, kubeconfig, cfg.configName); err != nil {
+			return err
+		}
+
+		if cluster.Spec.Mode == v1beta1.HCPClusterMode {
+			printHCPJoinInstructions(&cluster, kubeconfig)
+		}
+
+		return nil
 	}
 }
 
+// resolveServerHost returns the host that should be embedded in the kubeconfig
+// server URL and used as the TLS-SAN. If override is set it takes precedence;
+// otherwise the host is extracted from restConfigHost.
+func resolveServerHost(restConfigHost, override string) (string, error) {
+	if override != "" {
+		return override, nil
+	}
+
+	u, err := url.Parse(restConfigHost)
+	if err != nil {
+		return "", err
+	}
+
+	return u.Hostname(), nil
+}
+
 func writeKubeconfigFile(cluster *v1beta1.Cluster, kubeconfig *clientcmdapi.Config, configName string) error {
 	if configName == "" {
 		configName = cluster.Namespace + "-" + cluster.Name + "-kubeconfig.yaml"

cli/cmds/policy_create.go

@@ -34,10 +34,10 @@ func NewPolicyCreateCmd(appCtx *AppContext) *cobra.Command {
 		Example: "k3kcli policy create [command options] NAME",
 		PreRunE: func(cmd *cobra.Command, args []string) error {
 			switch config.mode {
-			case string(v1beta1.VirtualClusterMode), string(v1beta1.SharedClusterMode):
+			case string(v1beta1.VirtualClusterMode), string(v1beta1.SharedClusterMode), string(v1beta1.HCPClusterMode):
 				return nil
 			default:
-				return errors.New(`mode should be one of "shared" or "virtual"`)
+				return errors.New(`mode must be one of "shared", "virtual" or "hcp"`)
 			}
 		},
 		RunE: policyCreateAction(appCtx, config),