diff --git a/.github/workflows/test-conformance-hcp.yaml b/.github/workflows/test-conformance-hcp.yaml new file mode 100644 index 00000000..ec303517 --- /dev/null +++ b/.github/workflows/test-conformance-hcp.yaml @@ -0,0 +1,407 @@ +name: Conformance Tests - HCP Mode + +on: + schedule: + - cron: "0 1 * * *" + workflow_dispatch: + inputs: + k3k_version: + description: 'K3k version to test (e.g. v1.1.0). Leave empty to build from source.' + required: false + type: string + k8s_version: + description: 'Kubernetes version to test' + required: false + type: choice + options: + - "" + - "v1.34.6" + - "v1.35.3" + +permissions: + contents: read + +env: + K8S_VERSIONS: "v1.34.6,v1.35.3" + HELM_VERSION: v4.1.3 + HELM_CHECKSUM_AMD64: 02ce9722d541238f81459938b84cf47df2fdf1187493b4bfb2346754d82a4700 + +jobs: + setup: + runs-on: ubuntu-latest + outputs: + k8s_versions: ${{ steps.set-matrix.outputs.k8s_versions }} + steps: + - id: set-matrix + run: | + if [[ -z "${{ inputs.k8s_version }}" ]]; then + JSON_ARRAY=$(jq -nc '"${{ env.K8S_VERSIONS }}" | split(",")') + echo "k8s_versions=${JSON_ARRAY}" >> "$GITHUB_OUTPUT" + else + echo "k8s_versions=[\"${{ inputs.k8s_version }}\"]" >> "$GITHUB_OUTPUT" + fi + + conformance: + needs: setup + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + k8s_version: ${{ fromJSON(needs.setup.outputs.k8s_versions) }} + + env: + KUBERNETES_VERSION: ${{ matrix.k8s_version }} + + steps: + - name: Checkout code + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + with: + fetch-depth: 0 + fetch-tags: true + + - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6 + with: + go-version-file: go.mod + + - name: Install helm + env: + FILENAME: helm.tar.gz + run: | + curl -sSfL -o ${{ env.FILENAME }} https://get.helm.sh/helm-${{ env.HELM_VERSION }}-linux-amd64.tar.gz + echo "${{ env.HELM_CHECKSUM_AMD64 }} ${{ env.FILENAME }}" | sha256sum --check + tar -xvzf ${{ env.FILENAME }} linux-amd64/helm + sudo install -m 755 linux-amd64/helm /usr/local/bin/helm + + rm -fr "${{ env.FILENAME }}" linux-amd64/helm + + - name: Install hydrophone + run: go install sigs.k8s.io/hydrophone@3de3e886a2f6f09635d8b981c195490af1584d97 #v0.7.0 + + - name: Install k3s + env: + KUBECONFIG: /etc/rancher/k3s/k3s.yaml + K3S_HOST_VERSION: ${{ env.KUBERNETES_VERSION }}+k3s1 + run: | + curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=${K3S_HOST_VERSION} INSTALL_K3S_EXEC="--write-kubeconfig-mode=777" sh -s - + + kubectl cluster-info + kubectl get nodes + + echo "KUBECONFIG=${KUBECONFIG}" >> $GITHUB_ENV + + - name: Setup K3k (from source) + if: inputs.k3k_version == '' + run: | + export REPO=ttl.sh/$(uuidgen) + export VERSION=1h + + make build + make package + make push + make install + + # add k3kcli to $PATH + echo "${{ github.workspace }}/bin" >> $GITHUB_PATH + + - name: Setup K3k (from release) + if: inputs.k3k_version != '' + run: | + K3K_VERSION="${{ inputs.k3k_version }}" + CHART_VERSION="${K3K_VERSION#v}" + + helm repo add k3k https://rancher.github.io/k3k + helm repo update + helm install --namespace k3k-system --create-namespace --version "${CHART_VERSION}" k3k k3k/k3k + + wget -qO k3kcli "https://github.com/rancher/k3k/releases/download/${{ inputs.k3k_version }}/k3kcli-linux-amd64" + sudo mv k3kcli /usr/local/bin/k3kcli + sudo chmod +x /usr/local/bin/k3kcli + + - name: Wait for K3k controller + run: | + echo "Wait for K3k controller deployment to be available" + kubectl wait -n k3k-system deployment -l "app.kubernetes.io/name=k3k" --for=condition=Available --timeout=5m + + - name: Check k3kcli + run: k3kcli -v + + - name: Create virtual cluster + run: | + kubectl create namespace k3k-mycluster + + cat < user-data-${i} + #cloud-config + hostname: worker-${i} + preserve_hostname: false + manage_etc_hosts: true + # Stop cloud-init from generating its own DHCP netplan that would + # conflict with the static one we write in write_files below. + network: + config: disabled + users: + - name: ubuntu + ssh_authorized_keys: + - ${PUBKEY} + sudo: ['ALL=(ALL) NOPASSWD:ALL'] + shell: /bin/bash + write_files: + - path: /etc/netplan/50-static.yaml + permissions: '0600' + content: | + network: + version: 2 + ethernets: + ens3: + dhcp4: false + addresses: [${IP}/24] + routes: + - to: default + via: 192.168.100.1 + nameservers: + addresses: [8.8.8.8, 1.1.1.1] + runcmd: + - netplan apply + EOF + cloud-localds seed-${i}.img user-data-${i} + done + + - name: Create Worker Disks + run: | + qemu-img create -f qcow2 -b ubuntu-cloudimg.img -F qcow2 worker-1.qcow2 20G + qemu-img create -f qcow2 -b ubuntu-cloudimg.img -F qcow2 worker-2.qcow2 20G + + - name: Launch Worker VMs + run: | + # Launch Worker 1 — attached to tap-w1 on k3kbr0. + sudo qemu-system-x86_64 \ + -m 2048 -smp 2 -cpu host -enable-kvm -nographic \ + -drive file=worker-1.qcow2,if=virtio \ + -drive file=seed-1.img,format=raw,if=virtio \ + -netdev tap,id=net0,ifname=tap-w1,script=no,downscript=no \ + -device virtio-net-pci,netdev=net0,mac=52:54:00:12:34:56 \ + & + + # Wait a moment before launching the second VM + sleep 5 + + # Launch Worker 2 — attached to tap-w2 on k3kbr0. + sudo qemu-system-x86_64 \ + -m 2048 -smp 2 -cpu host -enable-kvm -nographic \ + -drive file=worker-2.qcow2,if=virtio \ + -drive file=seed-2.img,format=raw,if=virtio \ + -netdev tap,id=net0,ifname=tap-w2,script=no,downscript=no \ + -device virtio-net-pci,netdev=net0,mac=52:54:00:12:34:57 \ + & + + - name: Wait for SSH Availability + run: | + echo "Waiting for Worker 1 (192.168.100.11) to respond..." + timeout 180s bash -c ' + until ssh -i ./id_rsa -o StrictHostKeyChecking=no -o ConnectTimeout=2 ubuntu@192.168.100.11 true 2>/dev/null; do sleep 3; done + ' + + echo "Waiting for Worker 2 (192.168.100.12) to respond..." + timeout 180s bash -c ' + until ssh -i ./id_rsa -o StrictHostKeyChecking=no -o ConnectTimeout=2 ubuntu@192.168.100.12 true 2>/dev/null; do sleep 3; done + ' + + echo "Both VMs are up and running!" + + echo "Testing connectivity from VMs to K3k API server..." + ssh -i ./id_rsa -o StrictHostKeyChecking=no ubuntu@192.168.100.11 \ + "curl -kv --max-time 10 https://192.168.100.1:30001/readyz || echo 'Worker 1 connectivity test failed'" + + ssh -i ./id_rsa -o StrictHostKeyChecking=no ubuntu@192.168.100.12 \ + "curl -kv --max-time 10 https://192.168.100.1:30001/readyz || echo 'Worker 2 connectivity test failed'" + + - name: Verify Worker VM Configuration + run: | + for IP in 192.168.100.11 192.168.100.12; do + echo "=== Worker at ${IP} ===" + ssh -i ./id_rsa -o StrictHostKeyChecking=no ubuntu@${IP} \ + "echo 'Hostname:' \$(hostname) && \ + echo 'IP Address:' \$(ip -4 addr show ens3 | grep -oP '(?<=inet\s)\d+(\.\d+){3}') && \ + echo 'Gateway:' \$(ip route | grep default)" + echo "" + done + + - name: Join Workers to K3k Control Plane + env: + K3S_WORKER_VERSION: ${{ env.KUBERNETES_VERSION }}+k3s1 + run: | + K3S_TOKEN=$(kubectl get secret -n k3k-mycluster k3k-mycluster-token -o jsonpath='{.data.token}' | base64 -d) + + echo "Registering Worker 1 (k3s ${K3S_WORKER_VERSION})..." + ssh -i ./id_rsa -o StrictHostKeyChecking=no ubuntu@192.168.100.11 \ + "curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=${K3S_WORKER_VERSION} K3S_URL=https://192.168.100.1:30001 K3S_TOKEN=${K3S_TOKEN} sh -" + + echo "Registering Worker 2 (k3s ${K3S_WORKER_VERSION})..." + ssh -i ./id_rsa -o StrictHostKeyChecking=no ubuntu@192.168.100.12 \ + "curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=${K3S_WORKER_VERSION} K3S_URL=https://192.168.100.1:30001 K3S_TOKEN=${K3S_TOKEN} sh -" + + - name: Verify Cluster Nodes + env: + KUBECONFIG: ${{ github.workspace }}/k3k-mycluster-mycluster-kubeconfig.yaml + run: | + echo "Monitoring K3k Virtual Cluster for Node registration..." + + kubectl get pod -A + kubectl get nodes + + timeout 180s bash -c ' + until [ $(kubectl get nodes --no-headers 2>/dev/null | grep -c "Ready") -eq 2 ]; do + echo "Waiting for both worker nodes to show Ready..." + kubectl get nodes || true + sleep 5 + done + ' + + - name: Run conformance tests + run: | + hydrophone --conformance --parallel 4 \ + --kubeconfig ${{ github.workspace }}/k3k-mycluster-mycluster-kubeconfig.yaml \ + --output-dir /tmp + + - name: Collect logs + if: always() + env: + KUBECONFIG: /etc/rancher/k3s/k3s.yaml + run: | + journalctl -u k3s -o cat --no-pager > /tmp/k3s.log + kubectl logs -n k3k-system -l "app.kubernetes.io/name=k3k" --tail=-1 > /tmp/k3k.log + + - name: Archive K3s logs + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + if: always() + with: + name: k3s-${{ matrix.k8s_version }}-logs + path: /tmp/k3s.log + + - name: Archive K3k logs + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + if: always() + with: + name: k3k-${{ matrix.k8s_version }}-logs + path: /tmp/k3k.log + + - name: Archive conformance logs + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + if: always() + with: + name: conformance-${{ matrix.k8s_version }}-logs + path: /tmp/e2e.log + + - name: Archive results + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + if: always() + with: + name: conformance-${{ matrix.k8s_version }}-results + path: /tmp/junit_01.xml + + - name: Job Summary + if: always() + run: | + echo '## 📊 Conformance Tests Results (${{ matrix.k8s_version }})' >> $GITHUB_STEP_SUMMARY + echo '| Passed | Failed | Pending | Skipped |' >> $GITHUB_STEP_SUMMARY + echo '|---|---|---|---|' >> $GITHUB_STEP_SUMMARY + + RESULTS=$(tail -10 /tmp/e2e.log | grep -E "Passed .* Failed .* Pending .* Skipped" | cut -d '-' -f 3) + RESULTS=$(echo $RESULTS | grep -oE '[0-9]+' | xargs | sed 's/ / | /g') + echo "| $RESULTS |" >> $GITHUB_STEP_SUMMARY + + # only include failed tests section if there are any + if grep -q '\[FAIL\]' /tmp/e2e.log; then + echo '' >> $GITHUB_STEP_SUMMARY + echo '### Failed Tests' >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + grep '\[FAIL\]' /tmp/e2e.log >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + fi diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 80b691c3..e8592ed5 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -84,6 +84,9 @@ jobs: - name: Run cli tests env: K3S_HOST_VERSION: "${{ env.K3S_HOST_VERSION }}" + KUBECONFIG: /etc/rancher/k3s/k3s.yaml + REPO: ${{ env.REPO }} + VERSION: ${{ env.VERSION }} run: make test-cli - name: Convert coverage data diff --git a/charts/k3k/templates/crds/k3k.io_clusters.yaml b/charts/k3k/templates/crds/k3k.io_clusters.yaml index 97738532..0c530371 100644 --- a/charts/k3k/templates/crds/k3k.io_clusters.yaml +++ b/charts/k3k/templates/crds/k3k.io_clusters.yaml @@ -1294,17 +1294,12 @@ spec: are mirrored into the virtual cluster. type: boolean mode: - allOf: - - enum: - - shared - - virtual - - enum: - - shared - - virtual default: shared - description: |- - Mode specifies the cluster provisioning mode: "shared" or "virtual". - Defaults to "shared". This field is immutable. + description: Mode specifies the cluster provisioning mode. This field is immutable. + enum: + - shared + - virtual + - hcp type: string x-kubernetes-validations: - message: mode is immutable @@ -1431,9 +1426,7 @@ spec: description: optional field specify whether the Secret or its keys must be defined type: boolean role: - description: |- - Role is the type of the k3k pod that will be used to mount the secret. - This can be 'server', 'agent', or 'all' (for both). + description: Role is the type of the k3k pod that will be used to mount the secret. enum: - server - agent diff --git a/charts/k3k/templates/crds/k3k.io_virtualclusterpolicies.yaml b/charts/k3k/templates/crds/k3k.io_virtualclusterpolicies.yaml index a056411b..33f78044 100644 --- a/charts/k3k/templates/crds/k3k.io_virtualclusterpolicies.yaml +++ b/charts/k3k/templates/crds/k3k.io_virtualclusterpolicies.yaml @@ -51,10 +51,11 @@ spec: properties: allowedMode: default: shared - description: AllowedMode specifies the allowed cluster provisioning mode. Defaults to "shared". + description: AllowedMode specifies the allowed cluster provisioning mode. enum: - shared - virtual + - hcp type: string x-kubernetes-validations: - message: mode is immutable diff --git a/cli/cmds/cluster_create.go b/cli/cmds/cluster_create.go index 32e7654f..f1135b07 100644 --- a/cli/cmds/cluster_create.go +++ b/cli/cmds/cluster_create.go @@ -87,10 +87,14 @@ func createAction(appCtx *AppContext, config *CreateConfig) func(cmd *cobra.Comm return errors.New("invalid cluster name") } - if config.mode == string(v1beta1.SharedClusterMode) && config.agents != 0 { + if config.agents != 0 && config.mode != string(v1beta1.VirtualClusterMode) { return errors.New("invalid flag, --agents flag is only allowed in virtual mode") } + if config.mode == string(v1beta1.HCPClusterMode) { + logrus.Warn("HCP (Hosted Control Plane) mode is experimental.") + } + namespace := appCtx.Namespace(name) if err := createNamespace(ctx, client, namespace, config.policy); err != nil { @@ -194,10 +198,34 @@ func createAction(appCtx *AppContext, config *CreateConfig) func(cmd *cobra.Comm return err } - return writeKubeconfigFile(cluster, kubeconfig, "") + if err := writeKubeconfigFile(cluster, kubeconfig, ""); err != nil { + return err + } + + if cluster.Spec.Mode == v1beta1.HCPClusterMode { + printHCPJoinInstructions(cluster, kubeconfig) + } + + return nil } } +func printHCPJoinInstructions(cluster *v1beta1.Cluster, kc *clientcmdapi.Config) { + tokenSecretName := k3kcluster.TokenSecretName(cluster.Name) + serverURL := kc.Clusters["default"].Server + + logrus.Infof(`To join an external worker node to this HCP cluster: + + 1. On this machine, fetch the cluster token: + + kubectl get secret -n %s %s -o jsonpath='{.data.token}' | base64 -d + + 2. On the worker node, run (replace with the value from step 1): + + curl -sfL https://get.k3s.io | K3S_URL=%s K3S_TOKEN= sh - +`, cluster.Namespace, tokenSecretName, serverURL) +} + func newCluster(name, namespace string, config *CreateConfig) (*v1beta1.Cluster, error) { var storageRequestSize *resource.Quantity if config.storageRequestSize != "" { diff --git a/cli/cmds/cluster_create_flags.go b/cli/cmds/cluster_create_flags.go index c728f9ee..1cf71882 100644 --- a/cli/cmds/cluster_create_flags.go +++ b/cli/cmds/cluster_create_flags.go @@ -29,7 +29,7 @@ func createFlags(cmd *cobra.Command, cfg *CreateConfig) { cmd.Flags().StringArrayVar(&cfg.labels, "labels", []string{}, "Labels to add to the cluster object (e.g. key=value)") cmd.Flags().StringArrayVar(&cfg.annotations, "annotations", []string{}, "Annotations to add to the cluster object (e.g. key=value)") cmd.Flags().StringVar(&cfg.version, "version", "", "k3s version") - cmd.Flags().StringVar(&cfg.mode, "mode", "shared", "k3k mode type (shared, virtual)") + cmd.Flags().StringVar(&cfg.mode, "mode", "shared", "k3k mode type (shared, virtual, hcp)") cmd.Flags().StringVar(&cfg.kubeconfigServerHost, "kubeconfig-server", "", "override the kubeconfig server host") cmd.Flags().StringVar(&cfg.policy, "policy", "", "The policy to create the cluster in") cmd.Flags().StringVar(&cfg.customCertsPath, "custom-certs", "", "The path for custom certificate directory") @@ -43,30 +43,39 @@ func createFlags(cmd *cobra.Command, cfg *CreateConfig) { } } +var validPersistenceModes = map[v1beta1.PersistenceMode]struct{}{ + v1beta1.EphemeralPersistenceMode: {}, + v1beta1.DynamicPersistenceMode: {}, +} + +var validClusterModes = map[v1beta1.ClusterMode]struct{}{ + v1beta1.VirtualClusterMode: {}, + v1beta1.SharedClusterMode: {}, + v1beta1.HCPClusterMode: {}, +} + func validateCreateConfig(cfg *CreateConfig) error { if cfg.servers <= 0 { - return errors.New("invalid number of servers") + return errors.New("invalid number of servers: must be 1 or more") } if cfg.persistenceType != "" { - switch v1beta1.PersistenceMode(cfg.persistenceType) { - case v1beta1.EphemeralPersistenceMode, v1beta1.DynamicPersistenceMode: - return nil - default: + persistenceMode := v1beta1.PersistenceMode(cfg.persistenceType) + if _, found := validPersistenceModes[persistenceMode]; !found { return errors.New(`persistence-type should be one of "dynamic" or "ephemeral"`) } } - if _, err := resource.ParseQuantity(cfg.storageRequestSize); err != nil { - return errors.New(`invalid storage size, should be a valid resource quantity e.g "10Gi"`) + if cfg.storageRequestSize != "" { + if _, err := resource.ParseQuantity(cfg.storageRequestSize); err != nil { + return errors.New(`invalid storage size, should be a valid resource quantity e.g "10Gi"`) + } } if cfg.mode != "" { - switch cfg.mode { - case string(v1beta1.VirtualClusterMode), string(v1beta1.SharedClusterMode): - return nil - default: - return errors.New(`mode should be one of "shared" or "virtual"`) + clusterMode := v1beta1.ClusterMode(cfg.mode) + if _, found := validClusterModes[clusterMode]; !found { + return errors.New(`mode should be one of "shared", "virtual" or "hcp"`) } } diff --git a/cli/cmds/cluster_create_flags_test.go b/cli/cmds/cluster_create_flags_test.go new file mode 100644 index 00000000..370a9a27 --- /dev/null +++ b/cli/cmds/cluster_create_flags_test.go @@ -0,0 +1,106 @@ +package cmds + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_validateCreateConfig(t *testing.T) { + tests := []struct { + name string + cfg CreateConfig + wantErr string + }{ + { + name: "valid full config", + cfg: CreateConfig{ + servers: 1, + persistenceType: "dynamic", + storageRequestSize: "10Gi", + mode: "shared", + }, + }, + { + name: "empty storage size", + cfg: CreateConfig{ + servers: 1, + storageRequestSize: "", + }, + }, + { + name: "zero servers", + cfg: CreateConfig{ + servers: 0, + }, + wantErr: "invalid number of servers: must be 1 or more", + }, + { + name: "negative servers", + cfg: CreateConfig{ + servers: -1, + }, + wantErr: "invalid number of servers: must be 1 or more", + }, + { + name: "empty persistence type", + cfg: CreateConfig{ + servers: 1, + persistenceType: "", + storageRequestSize: "10Gi", + }, + }, + { + name: "invalid persistence type", + cfg: CreateConfig{ + servers: 1, + persistenceType: "foo", + }, + wantErr: `persistence-type should be one of "dynamic" or "ephemeral"`, + }, + { + name: "invalid storage size", + cfg: CreateConfig{ + servers: 1, + storageRequestSize: "abc", + }, + wantErr: `invalid storage size, should be a valid resource quantity e.g "10Gi"`, + }, + { + name: "empty mode", + cfg: CreateConfig{ + servers: 1, + mode: "", + storageRequestSize: "10Gi", + }, + }, + { + name: "invalid mode", + cfg: CreateConfig{ + servers: 1, + mode: "foo", + storageRequestSize: "10Gi", + }, + wantErr: `mode should be one of "shared", "virtual" or "hcp"`, + }, + { + name: "valid hcp mode", + cfg: CreateConfig{ + servers: 1, + mode: "hcp", + storageRequestSize: "10Gi", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := validateCreateConfig(&tt.cfg) + if tt.wantErr != "" { + assert.EqualError(t, err, tt.wantErr) + } else { + assert.NoError(t, err) + } + }) + } +} diff --git a/cli/cmds/kubeconfig.go b/cli/cmds/kubeconfig.go index 249472d5..f06f088d 100644 --- a/cli/cmds/kubeconfig.go +++ b/cli/cmds/kubeconfig.go @@ -5,7 +5,6 @@ import ( "net/url" "os" "path/filepath" - "strings" "time" "github.com/sirupsen/logrus" @@ -89,14 +88,12 @@ func generate(appCtx *AppContext, cfg *GenerateKubeconfigConfig) func(cmd *cobra return err } - url, err := url.Parse(appCtx.RestConfig.Host) + host, err := resolveServerHost(appCtx.RestConfig.Host, cfg.kubeconfigServerHost) if err != nil { return err } - host := strings.Split(url.Host, ":") if cfg.kubeconfigServerHost != "" { - host = []string{cfg.kubeconfigServerHost} cfg.altNames = append(cfg.altNames, cfg.kubeconfigServerHost) } @@ -118,16 +115,40 @@ func generate(appCtx *AppContext, cfg *GenerateKubeconfigConfig) func(cmd *cobra var kubeconfig *clientcmdapi.Config if err := retry.OnError(controller.Backoff, apierrors.IsNotFound, func() error { - kubeconfig, err = kubeCfg.Generate(ctx, client, &cluster, host[0]) + kubeconfig, err = kubeCfg.Generate(ctx, client, &cluster, host) return err }); err != nil { return err } - return writeKubeconfigFile(&cluster, kubeconfig, cfg.configName) + if err := writeKubeconfigFile(&cluster, kubeconfig, cfg.configName); err != nil { + return err + } + + if cluster.Spec.Mode == v1beta1.HCPClusterMode { + printHCPJoinInstructions(&cluster, kubeconfig) + } + + return nil } } +// resolveServerHost returns the host that should be embedded in the kubeconfig +// server URL and used as the TLS-SAN. If override is set it takes precedence; +// otherwise the host is extracted from restConfigHost. +func resolveServerHost(restConfigHost, override string) (string, error) { + if override != "" { + return override, nil + } + + u, err := url.Parse(restConfigHost) + if err != nil { + return "", err + } + + return u.Hostname(), nil +} + func writeKubeconfigFile(cluster *v1beta1.Cluster, kubeconfig *clientcmdapi.Config, configName string) error { if configName == "" { configName = cluster.Namespace + "-" + cluster.Name + "-kubeconfig.yaml" diff --git a/cli/cmds/policy_create.go b/cli/cmds/policy_create.go index 4bdbe9aa..dd133832 100644 --- a/cli/cmds/policy_create.go +++ b/cli/cmds/policy_create.go @@ -34,10 +34,10 @@ func NewPolicyCreateCmd(appCtx *AppContext) *cobra.Command { Example: "k3kcli policy create [command options] NAME", PreRunE: func(cmd *cobra.Command, args []string) error { switch config.mode { - case string(v1beta1.VirtualClusterMode), string(v1beta1.SharedClusterMode): + case string(v1beta1.VirtualClusterMode), string(v1beta1.SharedClusterMode), string(v1beta1.HCPClusterMode): return nil default: - return errors.New(`mode should be one of "shared" or "virtual"`) + return errors.New(`mode must be one of "shared", "virtual" or "hcp"`) } }, RunE: policyCreateAction(appCtx, config), diff --git a/docs/cli/k3kcli.adoc b/docs/cli/k3kcli.adoc index 85d43679..e1f94582 100644 --- a/docs/cli/k3kcli.adoc +++ b/docs/cli/k3kcli.adoc @@ -54,7 +54,7 @@ k3kcli cluster create [command options] NAME --kubeconfig-server string override the kubeconfig server host --labels stringArray Labels to add to the cluster object (e.g. key=value) --mirror-host-nodes Mirror Host Cluster Nodes - --mode string k3k mode type (shared, virtual) (default "shared") + --mode string k3k mode type (shared, virtual, hcp) (default "shared") -n, --namespace string namespace of the k3k cluster --persistence-type string persistence mode for the nodes (dynamic, ephemeral) (default "dynamic") --policy string The policy to create the cluster in diff --git a/docs/cli/k3kcli_cluster_create.md b/docs/cli/k3kcli_cluster_create.md index b57704be..8f3880c9 100644 --- a/docs/cli/k3kcli_cluster_create.md +++ b/docs/cli/k3kcli_cluster_create.md @@ -25,7 +25,7 @@ k3kcli cluster create [command options] NAME --kubeconfig-server string override the kubeconfig server host --labels stringArray Labels to add to the cluster object (e.g. key=value) --mirror-host-nodes Mirror Host Cluster Nodes - --mode string k3k mode type (shared, virtual) (default "shared") + --mode string k3k mode type (shared, virtual, hcp) (default "shared") -n, --namespace string namespace of the k3k cluster --persistence-type string persistence mode for the nodes (dynamic, ephemeral) (default "dynamic") --policy string The policy to create the cluster in diff --git a/docs/crds/crds.adoc b/docs/crds/crds.adoc index af975d23..eed155f9 100644 --- a/docs/crds/crds.adoc +++ b/docs/crds/crds.adoc @@ -133,8 +133,9 @@ _Underlying type:_ _string_ ClusterMode is the possible provisioning mode of a Cluster. -_Validation:_ -- Enum: [shared virtual] +Supported values: `shared`, `virtual`, `hcp`. + + _Appears In:_ @@ -177,8 +178,7 @@ _Appears In:_ | *`version`* __string__ | Version is the K3s version to use for the virtual nodes. + It should follow the K3s versioning convention (e.g., v1.28.2-k3s1). + If not specified, the Kubernetes version of the host node will be used. + | | -| *`mode`* __xref:{anchor_prefix}-github-com-rancher-k3k-pkg-apis-k3k-io-v1beta1-clustermode[$$ClusterMode$$]__ | Mode specifies the cluster provisioning mode: "shared" or "virtual". + -Defaults to "shared". This field is immutable. + | shared | Enum: [shared virtual] + +| *`mode`* __xref:{anchor_prefix}-github-com-rancher-k3k-pkg-apis-k3k-io-v1beta1-clustermode[$$ClusterMode$$]__ | Mode specifies the cluster provisioning mode. This field is immutable. + | shared | Enum: [shared virtual hcp] + | *`servers`* __integer__ | Servers specifies the number of K3s pods to run in server (control plane) mode. + Must be at least 1. Defaults to 1. + | 1 | @@ -548,8 +548,9 @@ _Underlying type:_ _string_ PodSecurityAdmissionLevel is the policy level applied to the pods in the namespace. -_Validation:_ -- Enum: [privileged baseline restricted] +Supported values: `privileged`, `baseline`, `restricted`. + + _Appears In:_ @@ -624,8 +625,7 @@ secret contents will be mounted. + | | | *`subPath`* __string__ | SubPath is an optional path within the secret to mount instead of the root. + When specified, only the specified key from the secret will be mounted as a file + at MountPath, keeping the parent directory writable. + | | -| *`role`* __string__ | Role is the type of the k3k pod that will be used to mount the secret. + -This can be 'server', 'agent', or 'all' (for both). + | | Enum: [server agent all] + +| *`role`* __string__ | Role is the type of the k3k pod that will be used to mount the secret. + | | Enum: [server agent all] + |=== @@ -797,7 +797,7 @@ to set defaults and constraints (min/max) + | | This includes both node affinity and pod affinity/anti-affinity rules. + | | | *`defaultAgentAffinity`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#affinity-v1-core[$$Affinity$$]__ | DefaultAgentAffinity specifies the affinity rules applied to agent pods of all clusters in the target Namespace. + This includes both node affinity and pod affinity/anti-affinity rules. + | | -| *`allowedMode`* __xref:{anchor_prefix}-github-com-rancher-k3k-pkg-apis-k3k-io-v1beta1-clustermode[$$ClusterMode$$]__ | AllowedMode specifies the allowed cluster provisioning mode. Defaults to "shared". + | shared | Enum: [shared virtual] + +| *`allowedMode`* __xref:{anchor_prefix}-github-com-rancher-k3k-pkg-apis-k3k-io-v1beta1-clustermode[$$ClusterMode$$]__ | AllowedMode specifies the allowed cluster provisioning mode. + | shared | Enum: [shared virtual hcp] + | *`disableNetworkPolicy`* __boolean__ | DisableNetworkPolicy indicates whether to disable the creation of a default network policy for cluster isolation. + | | | *`podSecurityAdmissionLevel`* __xref:{anchor_prefix}-github-com-rancher-k3k-pkg-apis-k3k-io-v1beta1-podsecurityadmissionlevel[$$PodSecurityAdmissionLevel$$]__ | PodSecurityAdmissionLevel specifies the pod security admission level applied to the pods in the namespace. + | | Enum: [privileged baseline restricted] + diff --git a/docs/crds/crds.md b/docs/crds/crds.md index 7a881bba..b6de132d 100644 --- a/docs/crds/crds.md +++ b/docs/crds/crds.md @@ -102,8 +102,9 @@ _Underlying type:_ _string_ ClusterMode is the possible provisioning mode of a Cluster. -_Validation:_ -- Enum: [shared virtual] +Supported values: `shared`, `virtual`, `hcp`. + + _Appears in:_ - [ClusterSpec](#clusterspec) @@ -138,7 +139,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | | `version` _string_ | Version is the K3s version to use for the virtual nodes.
It should follow the K3s versioning convention (e.g., v1.28.2-k3s1).
If not specified, the Kubernetes version of the host node will be used. | | | -| `mode` _[ClusterMode](#clustermode)_ | Mode specifies the cluster provisioning mode: "shared" or "virtual".
Defaults to "shared". This field is immutable. | shared | Enum: [shared virtual]
| +| `mode` _[ClusterMode](#clustermode)_ | Mode specifies the cluster provisioning mode. This field is immutable. | shared | Enum: [shared virtual hcp]
| | `servers` _integer_ | Servers specifies the number of K3s pods to run in server (control plane) mode.
Must be at least 1. Defaults to 1. | 1 | | | `agents` _integer_ | Agents specifies the number of K3s pods to run in agent (worker) mode.
Must be 0 or greater. Defaults to 0.
This field is ignored in "shared" mode. | 0 | | | `clusterCIDR` _string_ | ClusterCIDR is the CIDR range for pod IPs.
Defaults to 10.42.0.0/16 in shared mode and 10.52.0.0/16 in virtual mode.
This field is immutable. | | | @@ -410,8 +411,9 @@ _Underlying type:_ _string_ PodSecurityAdmissionLevel is the policy level applied to the pods in the namespace. -_Validation:_ -- Enum: [privileged baseline restricted] +Supported values: `privileged`, `baseline`, `restricted`. + + _Appears in:_ - [VirtualClusterPolicySpec](#virtualclusterpolicyspec) @@ -456,7 +458,7 @@ _Appears in:_ | `name` _string_ | Name is the name of the secret mount volume that will be used
as the name of volume and volume mount for the server or agent pod
if empty then the secret name will be used instead. | | MaxLength: 63
MinLength: 1
Pattern: `^[a-z0-9]([-a-z0-9]*[a-z0-9])?$`
| | `mountPath` _string_ | MountPath is the path within server and agent pods where the
secret contents will be mounted. | | | | `subPath` _string_ | SubPath is an optional path within the secret to mount instead of the root.
When specified, only the specified key from the secret will be mounted as a file
at MountPath, keeping the parent directory writable. | | | -| `role` _string_ | Role is the type of the k3k pod that will be used to mount the secret.
This can be 'server', 'agent', or 'all' (for both). | | Enum: [server agent all]
| +| `role` _string_ | Role is the type of the k3k pod that will be used to mount the secret. | | Enum: [server agent all]
| #### SecretSyncConfig @@ -592,7 +594,7 @@ _Appears in:_ | `defaultPriorityClass` _string_ | DefaultPriorityClass specifies the priorityClassName applied to all pods of all clusters in the target Namespace. | | | | `defaultServerAffinity` _[Affinity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#affinity-v1-core)_ | DefaultServerAffinity specifies the affinity rules applied to server pods of all clusters in the target Namespace.
This includes both node affinity and pod affinity/anti-affinity rules. | | | | `defaultAgentAffinity` _[Affinity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#affinity-v1-core)_ | DefaultAgentAffinity specifies the affinity rules applied to agent pods of all clusters in the target Namespace.
This includes both node affinity and pod affinity/anti-affinity rules. | | | -| `allowedMode` _[ClusterMode](#clustermode)_ | AllowedMode specifies the allowed cluster provisioning mode. Defaults to "shared". | shared | Enum: [shared virtual]
| +| `allowedMode` _[ClusterMode](#clustermode)_ | AllowedMode specifies the allowed cluster provisioning mode. | shared | Enum: [shared virtual hcp]
| | `disableNetworkPolicy` _boolean_ | DisableNetworkPolicy indicates whether to disable the creation of a default network policy for cluster isolation. | | | | `podSecurityAdmissionLevel` _[PodSecurityAdmissionLevel](#podsecurityadmissionlevel)_ | PodSecurityAdmissionLevel specifies the pod security admission level applied to the pods in the namespace. | | Enum: [privileged baseline restricted]
| | `sync` _[SyncConfig](#syncconfig)_ | Sync specifies the resources types that will be synced from virtual cluster to host cluster. | \{ \} | | diff --git a/examples/hcp-server.yaml b/examples/hcp-server.yaml new file mode 100644 index 00000000..bb984f8c --- /dev/null +++ b/examples/hcp-server.yaml @@ -0,0 +1,12 @@ +apiVersion: k3k.io/v1beta1 +kind: Cluster +metadata: + name: hcp-server +spec: + mode: hcp + tlsSANs: + - "" + servers: 1 + version: v1.33.1-k3s1 + expose: + nodePort: {} diff --git a/pkg/apis/k3k.io/v1beta1/types.go b/pkg/apis/k3k.io/v1beta1/types.go index 213ae6de..f7464f8f 100644 --- a/pkg/apis/k3k.io/v1beta1/types.go +++ b/pkg/apis/k3k.io/v1beta1/types.go @@ -45,11 +45,10 @@ type ClusterSpec struct { // +optional Version string `json:"version,omitempty"` - // Mode specifies the cluster provisioning mode: "shared" or "virtual". - // Defaults to "shared". This field is immutable. + // Mode specifies the cluster provisioning mode. This field is immutable. // - // +kubebuilder:default="shared" - // +kubebuilder:validation:Enum=shared;virtual + // +kubebuilder:default=shared + // +kubebuilder:validation:Enum=shared;virtual;hcp // +kubebuilder:validation:XValidation:message="mode is immutable",rule="self == oldSelf" // +optional Mode ClusterMode `json:"mode,omitempty"` @@ -262,10 +261,9 @@ type SecretMount struct { // +optional SubPath string `json:"subPath,omitempty"` // Role is the type of the k3k pod that will be used to mount the secret. - // This can be 'server', 'agent', or 'all' (for both). // - // +optional // +kubebuilder:validation:Enum=server;agent;all + // +optional Role string `json:"role,omitempty"` } @@ -422,8 +420,7 @@ type StorageClassSyncConfig struct { // ClusterMode is the possible provisioning mode of a Cluster. // -// +kubebuilder:validation:Enum=shared;virtual -// +kubebuilder:default="shared" +// Supported values: `shared`, `virtual`, `hcp`. type ClusterMode string const ( @@ -432,6 +429,11 @@ const ( // VirtualClusterMode represents a cluster that runs in a virtual environment. VirtualClusterMode = ClusterMode("virtual") + + // HCPClusterMode represents a Hosted Control Plane: an agentless K3s control + // plane managed by k3k inside the host cluster. End users join their own + // external nodes (BYO) using the standard K3s installer command. + HCPClusterMode = ClusterMode("hcp") ) // PersistenceMode is the storage mode of a Cluster. @@ -640,7 +642,7 @@ type ClusterStatus struct { // Phase is a high-level summary of the cluster's current lifecycle state. // - // +kubebuilder:default="Unknown" + // +kubebuilder:default=Unknown // +kubebuilder:validation:Enum=Pending;Provisioning;Ready;Failed;Terminating;Unknown // +optional Phase ClusterPhase `json:"phase,omitempty"` @@ -785,9 +787,10 @@ type VirtualClusterPolicySpec struct { // +optional DefaultAgentAffinity *corev1.Affinity `json:"defaultAgentAffinity,omitempty"` - // AllowedMode specifies the allowed cluster provisioning mode. Defaults to "shared". + // AllowedMode specifies the allowed cluster provisioning mode. // // +kubebuilder:default=shared + // +kubebuilder:validation:Enum=shared;virtual;hcp // +kubebuilder:validation:XValidation:message="mode is immutable",rule="self == oldSelf" // +optional AllowedMode ClusterMode `json:"allowedMode,omitempty"` @@ -799,6 +802,7 @@ type VirtualClusterPolicySpec struct { // PodSecurityAdmissionLevel specifies the pod security admission level applied to the pods in the namespace. // + // +kubebuilder:validation:Enum=privileged;baseline;restricted // +optional PodSecurityAdmissionLevel *PodSecurityAdmissionLevel `json:"podSecurityAdmissionLevel,omitempty"` @@ -831,7 +835,7 @@ type VirtualClusterPolicySpec struct { // PodSecurityAdmissionLevel is the policy level applied to the pods in the namespace. // -// +kubebuilder:validation:Enum=privileged;baseline;restricted +// Supported values: `privileged`, `baseline`, `restricted`. type PodSecurityAdmissionLevel string const ( diff --git a/pkg/controller/cluster/cluster.go b/pkg/controller/cluster/cluster.go index af686a45..281eec4e 100644 --- a/pkg/controller/cluster/cluster.go +++ b/pkg/controller/cluster/cluster.go @@ -401,9 +401,10 @@ func (c *ClusterReconciler) reconcile(ctx context.Context, cluster *v1beta1.Clus } } - // in virtual mode assign a default serviceCIDR - if cluster.Spec.Mode == v1beta1.VirtualClusterMode { - log.V(1).Info("assign default service CIDR for virtual mode") + // virtual and hcp modes both run a self-contained K3s control plane and + // need their own pod/service CIDR independent of the host cluster. + if cluster.Spec.Mode == v1beta1.VirtualClusterMode || cluster.Spec.Mode == v1beta1.HCPClusterMode { + log.V(1).Info("assign default service CIDR", "mode", cluster.Spec.Mode) cluster.Status.ServiceCIDR = defaultVirtualServiceCIDR } @@ -448,6 +449,23 @@ func (c *ClusterReconciler) reconcile(ctx context.Context, cluster *v1beta1.Clus return err } + // In hcp mode, validate that the cluster has an externally-routable + // API server endpoint (NodePort / LoadBalancer / Ingress) so external + // workers can join. The join command itself is printed by the CLI + // (`k3kcli cluster create` / `k3kcli kubeconfig generate`). + // We also own the default/kubernetes Endpoints/EndpointSlice inside the + // virtual cluster (the apiserver reconciler is disabled for HCP) so + // external-node pods can reach the in-cluster apiserver ClusterIP. + if cluster.Spec.Mode == v1beta1.HCPClusterMode { + if err := c.ensureHCPKubernetesEndpointSlice(ctx, cluster); err != nil { + return err + } + + if err := c.ensureHCPKubernetesEndpoints(ctx, cluster); err != nil { + return err + } + } + // Important: if you need to call the Server API of the Virtual Cluster // this needs to be done AFTER he kubeconfig has been generated @@ -909,6 +927,14 @@ func (c *ClusterReconciler) bindClusterRoles(ctx context.Context, cluster *v1bet } func (c *ClusterReconciler) ensureAgent(ctx context.Context, cluster *v1beta1.Cluster, serviceIP, token string) error { + // hcp mode is BYO-node by design: external (out-of-host-cluster) nodes + // join using the standard K3s installer command, which is printed by + // `k3kcli cluster create` / `k3kcli kubeconfig generate`. k3k therefore + // does not provision any agent pods on the host cluster. + if cluster.Spec.Mode == v1beta1.HCPClusterMode { + return nil + } + config := agent.NewConfig(cluster, c.Client) var agentEnsurer agent.ResourceEnsurer diff --git a/pkg/controller/cluster/hcp.go b/pkg/controller/cluster/hcp.go new file mode 100644 index 00000000..4cbe5679 --- /dev/null +++ b/pkg/controller/cluster/hcp.go @@ -0,0 +1,225 @@ +package cluster + +import ( + "context" + "fmt" + "net" + "strconv" + + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + corev1 "k8s.io/api/core/v1" + discoveryv1 "k8s.io/api/discovery/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrl "sigs.k8s.io/controller-runtime" + + "github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1" + "github.com/rancher/k3k/pkg/controller/cluster/server" +) + +// findNonLoopbackSAN returns the first non-loopback address from the given +// TLS SANs. Returns empty string if none is found. +func findNonLoopbackSAN(sans []string) string { + for _, san := range sans { + if san == "localhost" { + continue + } + + if ip := net.ParseIP(san); ip != nil && ip.IsLoopback() { + continue + } + + return san + } + + return "" +} + +// ensureHCPKubernetesEndpointSlice maintains the default/kubernetes Service +// EndpointSlice inside the virtual cluster, pointing it at the externally +// reachable host:port (NodePort / LoadBalancer / Ingress) so that pods +// scheduled on external worker nodes can reach the in-cluster apiserver +// ClusterIP. +// +// Background: the kube-apiserver normally reconciles default/kubernetes +// EndpointSlice to its own --advertise-address:--secure-port (the host-cluster +// pod IP and 6443). External worker nodes have no route to the host-cluster +// pod CIDR, so kube-proxy DNAT to that endpoint fails. We disable the +// apiserver reconciler in HCP mode (see serverOptions) and own this +// EndpointSlice object instead. +func (c *ClusterReconciler) ensureHCPKubernetesEndpointSlice(ctx context.Context, cluster *v1beta1.Cluster) error { + log := ctrl.LoggerFrom(ctx) + + url, err := server.ServerURL(ctx, c.Client, cluster, findNonLoopbackSAN(cluster.Spec.TLSSANs)) + if err != nil { + return err + } + + port, err := strconv.Atoi(url.Port()) + if err != nil { + return err + } + + addr, err := hcpEndpointAddress(ctx, url.Hostname()) + if err != nil { + return err + } + + var addressType discoveryv1.AddressType + + if ip := net.ParseIP(addr.IP); ip != nil { + if ip.To4() != nil { + addressType = discoveryv1.AddressTypeIPv4 + } else { + addressType = discoveryv1.AddressTypeIPv6 + } + } else { + return fmt.Errorf("invalid IP address %q", addr.IP) + } + + virtClient, err := newVirtualClient(ctx, c.Client, cluster.Name, cluster.Namespace) + if err != nil { + return fmt.Errorf("creating virtual cluster client: %w", err) + } + + endpointSlice := &discoveryv1.EndpointSlice{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kubernetes", + Namespace: metav1.NamespaceDefault, + }, + } + + _, err = controllerutil.CreateOrUpdate(ctx, virtClient, endpointSlice, func() error { + if endpointSlice.Labels == nil { + endpointSlice.Labels = make(map[string]string) + } + + // Ensure the service-name label is set + endpointSlice.Labels[discoveryv1.LabelServiceName] = "kubernetes" + endpointSlice.AddressType = addressType + + endpointSlice.Endpoints = []discoveryv1.Endpoint{ + {Addresses: []string{addr.IP}}, + } + + endpointSlice.Ports = []discoveryv1.EndpointPort{ + { + Name: new("https"), + Port: new(int32(port)), + Protocol: new(corev1.ProtocolTCP), + }, + } + + return nil + }) + if err != nil { + return fmt.Errorf("upserting default/kubernetes endpointslice in virtual cluster: %w", err) + } + + log.V(1).Info("HCP kubernetes endpointslice reconciled", "address", addr.IP, "host", url.Hostname(), "port", port) + + return nil +} + +func (c *ClusterReconciler) ensureHCPKubernetesEndpoints(ctx context.Context, cluster *v1beta1.Cluster) error { + log := ctrl.LoggerFrom(ctx) + + url, err := server.ServerURL(ctx, c.Client, cluster, findNonLoopbackSAN(cluster.Spec.TLSSANs)) + if err != nil { + return err + } + + addr, err := hcpEndpointAddress(ctx, url.Hostname()) + if err != nil { + return err + } + + virtClient, err := newVirtualClient(ctx, c.Client, cluster.Name, cluster.Namespace) + if err != nil { + return fmt.Errorf("creating virtual cluster client: %w", err) + } + + //nolint:staticcheck // SA1019 corev1.Endpoints is deprecated in v1.33+, but needed in the Conformance tests + // We are already using the discoveryv1.EndpointSlice + endpoints := &corev1.Endpoints{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kubernetes", + Namespace: metav1.NamespaceDefault, + }, + } + + port, err := strconv.Atoi(url.Port()) + if err != nil { + return err + } + + _, err = controllerutil.CreateOrUpdate(ctx, virtClient, endpoints, func() error { + if endpoints.Labels == nil { + endpoints.Labels = make(map[string]string) + } + + // Ensure the skip-mirror label is set + endpoints.Labels[discoveryv1.LabelSkipMirror] = "true" + + //nolint:staticcheck // SA1019 corev1.EndpointSubset is deprecated in v1.33+, but needed in the Conformance tests + endpoints.Subsets = []corev1.EndpointSubset{ + { + Addresses: []corev1.EndpointAddress{addr}, + Ports: []corev1.EndpointPort{ + { + Name: "https", + Port: int32(port), + Protocol: corev1.ProtocolTCP, + }, + }, + }, + } + + return nil + }) + if err != nil { + return fmt.Errorf("upserting default/kubernetes endpoints in virtual cluster: %w", err) + } + + log.V(1).Info("HCP kubernetes endpoints reconciled", "address", addr.IP, "host", url.Host, "port", port) + + return nil +} + +// hcpEndpointAddress builds a corev1.EndpointAddress from the externally +// reachable host. Endpoints require an IP; if the host is a DNS name we +// resolve it. The Hostname field is intentionally left unset: +// the kubernetes API validates it as a DNS-1123 label (no dots), +// so an FQDN like "host.example.com" would be rejected. +func hcpEndpointAddress(ctx context.Context, host string) (corev1.EndpointAddress, error) { + if ip := net.ParseIP(host); ip != nil { + if ip.IsLoopback() { + return corev1.EndpointAddress{}, fmt.Errorf("HCP endpoint host %q is a loopback address and cannot be used", host) + } + + return corev1.EndpointAddress{IP: host}, nil + } + + ipAddrs, err := net.DefaultResolver.LookupIPAddr(ctx, host) + if err != nil { + return corev1.EndpointAddress{}, fmt.Errorf("HCP endpoint host %q is not an IP and does not resolve: %w", host, err) + } + + var filteredIPs []net.IP + + for _, addr := range ipAddrs { + if !addr.IP.IsLoopback() { + filteredIPs = append(filteredIPs, addr.IP) + } + } + + if len(filteredIPs) == 0 { + return corev1.EndpointAddress{}, fmt.Errorf("HCP endpoint host %q resolved to no non-loopback IPs", host) + } + + if v4 := filteredIPs[0].To4(); v4 != nil { + return corev1.EndpointAddress{IP: v4.String()}, nil + } + + return corev1.EndpointAddress{IP: filteredIPs[0].String()}, nil +} diff --git a/pkg/controller/cluster/hcp_test.go b/pkg/controller/cluster/hcp_test.go new file mode 100644 index 00000000..572401f6 --- /dev/null +++ b/pkg/controller/cluster/hcp_test.go @@ -0,0 +1,132 @@ +package cluster + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/rancher/k3k/pkg/controller" +) + +func Test_findNonLoopbackSAN(t *testing.T) { + tests := []struct { + name string + sans []string + want string + }{ + { + name: "loopback first, external second", + sans: []string{"127.0.0.1", "10.0.0.100"}, + want: "10.0.0.100", + }, + { + name: "external first", + sans: []string{"10.0.0.100", "127.0.0.1"}, + want: "10.0.0.100", + }, + { + name: "only loopback", + sans: []string{"127.0.0.1", "::1"}, + want: "", + }, + { + name: "localhost hostname filtered", + sans: []string{"localhost", "example.com"}, + want: "example.com", + }, + { + name: "external hostname", + sans: []string{"hcp.example.com"}, + want: "hcp.example.com", + }, + { + name: "empty", + sans: []string{}, + want: "", + }, + { + name: "ipv6 loopback filtered", + sans: []string{"::1", "2001:db8::1"}, + want: "2001:db8::1", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := findNonLoopbackSAN(tt.sans) + assert.Equal(t, tt.want, got) + }) + } +} + +func Test_hcpEndpointAddress(t *testing.T) { + tests := []struct { + name string + input string + wantIP string + wantHostname string + wantErr bool + }{ + { + name: "ipv4 literal is passed through", + input: "10.144.101.195", + wantIP: "10.144.101.195", + wantHostname: "", + wantErr: false, + }, + { + name: "unresolvable hostname errors", + input: "definitely-not-a-real-host.invalid", + wantErr: true, + }, + { + name: "ipv4 loopback literal is rejected", + input: "127.0.0.1", + wantErr: true, + }, + { + name: "ipv6 loopback literal is rejected", + input: "::1", + wantErr: true, + }, + { + name: "ipv4 loopback in range is rejected", + input: "127.0.0.100", + wantErr: true, + }, + { + name: "valid ipv6 literal passes through", + input: "2001:db8::1", + wantIP: "2001:db8::1", + wantHostname: "", + wantErr: false, + }, + { + name: "localhost hostname filters loopbacks", + input: "localhost", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := hcpEndpointAddress(context.Background(), tt.input) + if tt.wantErr { + require.Error(t, err) + return + } + + require.NoError(t, err) + + assert.Equal(t, tt.wantIP, got.IP) + assert.Equal(t, tt.wantHostname, got.Hostname) + }) + } +} + +// Compile-time assertion: every reused exported name from the controller +// package below this test file must remain stable. If `controller.K3SImage` +// disappears (refactor), this guards the dependency. +var _ = controller.K3SImage diff --git a/pkg/controller/cluster/server/config.go b/pkg/controller/cluster/server/config.go index 52136afd..f4635003 100644 --- a/pkg/controller/cluster/server/config.go +++ b/pkg/controller/cluster/server/config.go @@ -11,7 +11,6 @@ import ( "github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1" "github.com/rancher/k3k/pkg/controller" - "github.com/rancher/k3k/pkg/controller/cluster/agent" ) // serverConfig are few options from k3s server options that will @@ -23,6 +22,7 @@ type serverConfig struct { DisableAgent bool `yaml:"disable-agent,omitempty"` Disable []string `yaml:"disable,omitempty"` EgressSelectorMode string `yaml:"egress-selector-mode,omitempty"` + KubeApiServerArg []string `yaml:"kube-apiserver-arg,omitempty"` Server string `yaml:"server,omitempty"` ServiceCIDR string `yaml:"service-cidr,omitempty"` TLSSAN []string `yaml:"tls-san,omitempty"` @@ -77,10 +77,29 @@ func buildServerConfig(cluster *v1beta1.Cluster, initServer bool, serviceIP, tok serverConfig.Server = "https://" + serviceIP } - if cluster.Spec.Mode != agent.VirtualNodeMode { + // shared and hcp modes both run K3s with --disable-agent (agentless server). + switch cluster.Spec.Mode { + case "", v1beta1.SharedClusterMode: serverConfig.DisableAgent = true serverConfig.EgressSelectorMode = "disabled" serverConfig.Disable = []string{"servicelb", "traefik", "metrics-server", "local-storage"} + case v1beta1.HCPClusterMode: + serverConfig.DisableAgent = true + // Tunnel apiserver egress through the k3s-agent WebSocket: the + // apiserver has no route to the virtual cluster's pod CIDR and + // bypasses kube-proxy when dialing pod IPs (webhooks, log/exec). + // "cluster" is the only safe mode — "agent" lets pod dials go + // direct (no route, fails); "pod" only permits pod IPs the agent + // has already watched, so a newly-created pod's IP is rejected + // and tears down the remotedialer session, making kubelet streams + // flaky. See k3s pkg/agent/tunnel/tunnel.go. + serverConfig.EgressSelectorMode = "cluster" + // Disable the apiserver's built-in endpoint reconciler so K3k can + // own default/kubernetes Endpoints and point it at the externally + // reachable host:port (NodePort / LB / Ingress). + serverConfig.KubeApiServerArg = append(serverConfig.KubeApiServerArg, "endpoint-reconciler-type=none") + case v1beta1.VirtualClusterMode: + // no extra config for virtual mode } return serverConfig diff --git a/pkg/controller/cluster/server/endpoint.go b/pkg/controller/cluster/server/endpoint.go new file mode 100644 index 00000000..92e2ac6d --- /dev/null +++ b/pkg/controller/cluster/server/endpoint.go @@ -0,0 +1,138 @@ +package server + +import ( + "context" + "fmt" + "net" + "net/url" + "slices" + "strconv" + + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + ctrl "sigs.k8s.io/controller-runtime" + + "github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1" +) + +// ServerURL generates the API server URL for the kubeconfig based on the service configuration. +// +// It handles internal vs external access patterns: +// - Internal access (hostServerIP == service.ClusterIP): uses the ClusterIP for direct pod-to-pod communication +// - External access (hostServerIP != service.ClusterIP): uses the appropriate external endpoint based on the service type +// +// Service type handling: +// - ClusterIP: uses service.Spec.ClusterIP (internal-only) +// - NodePort: uses hostServerIP:NodePort for external access, ClusterIP:Port for internal access +// - LoadBalancer: uses the LoadBalancer ingress IP, falling back to its hostname +// - Ingress (if configured): takes precedence over the service-based URL +// +// The hostServerIP parameter determines the access pattern: +// - Controller reconciliation: passes service.Spec.ClusterIP → internal access +// - CLI kubeconfig export: passes the external host → external access +func ServerURL(ctx context.Context, c client.Client, cluster *v1beta1.Cluster, hostServerIP string) (*url.URL, error) { + log := ctrl.LoggerFrom(ctx) + + key := types.NamespacedName{ + Name: ServiceName(cluster.Name), + Namespace: cluster.Namespace, + } + + // Check if ingress is configured + if cluster.Spec.Expose != nil && cluster.Spec.Expose.Ingress != nil { + key := types.NamespacedName{ + Name: IngressName(cluster.Name), + Namespace: cluster.Namespace, + } + + var k3kIngress networkingv1.Ingress + if err := c.Get(ctx, key, &k3kIngress); err != nil { + return nil, err + } + + if len(k3kIngress.Spec.Rules) > 0 && k3kIngress.Spec.Rules[0].Host != "" { + return &url.URL{ + Scheme: "https", + Host: k3kIngress.Spec.Rules[0].Host, + }, nil + } + + log.V(1).Info("Ingress has no rule with a host set, falling back to the service URL.") + } + + // Fall back to Service-based URL + var k3kService corev1.Service + if err := c.Get(ctx, key, &k3kService); err != nil { + return nil, err + } + + // init to hostServerIP and 443 port + host := hostServerIP + port := int32(443) + + // Use service port as default if available + if len(k3kService.Spec.Ports) > 0 { + port = k3kService.Spec.Ports[0].Port + } + + // Handle each service type separately + switch k3kService.Spec.Type { + case corev1.ServiceTypeClusterIP: + host = k3kService.Spec.ClusterIP + + case corev1.ServiceTypeNodePort: + // Only use NodePort if hostServerIP is NOT the ClusterIP + // If hostServerIP == ClusterIP, this is an internal connection, use ClusterIP + if hostServerIP != k3kService.Spec.ClusterIP { + if len(k3kService.Spec.Ports) > 0 { + port = k3kService.Spec.Ports[0].NodePort + } + } else { + // Internal connection: use ClusterIP + host = k3kService.Spec.ClusterIP + } + + case corev1.ServiceTypeLoadBalancer: + if len(k3kService.Status.LoadBalancer.Ingress) > 0 { + ingress := k3kService.Status.LoadBalancer.Ingress[0] + + switch { + case ingress.IP != "": + host = ingress.IP + case ingress.Hostname != "": + host = ingress.Hostname + default: + log.V(1).Info("No usable ingress address found in LoadBalancer service.") + } + } + } + + if !slices.Contains(cluster.Status.TLSSANs, host) { + log.V(1).Info(fmt.Sprintf("IP %s not in tlsSANs.", host)) + + if len(cluster.Spec.TLSSANs) > 0 { + log.V(1).Info("Using the first TLS SAN in the spec as a fallback: " + cluster.Spec.TLSSANs[0]) + + host = cluster.Spec.TLSSANs[0] + } else if len(cluster.Status.TLSSANs) > 0 { + log.V(1).Info("No explicit tlsSANs specified. Trying to use the first TLS SAN in the status: " + cluster.Status.TLSSANs[0]) + + host = cluster.Status.TLSSANs[0] + } else { + log.V(1).Info("IP not found in tlsSANs. This could cause issue with the certificate validation.") + } + } + + // Build URL with port only if not the default HTTPS port + if port != int32(443) { + host = net.JoinHostPort(host, strconv.Itoa(int(port))) + } + + return &url.URL{ + Scheme: "https", + Host: host, + }, nil +} diff --git a/pkg/controller/kubeconfig/kubeconfig_test.go b/pkg/controller/cluster/server/endpoint_test.go similarity index 95% rename from pkg/controller/kubeconfig/kubeconfig_test.go rename to pkg/controller/cluster/server/endpoint_test.go index 0f5ea5e2..c829e7fc 100644 --- a/pkg/controller/kubeconfig/kubeconfig_test.go +++ b/pkg/controller/cluster/server/endpoint_test.go @@ -1,6 +1,6 @@ -package kubeconfig +package server_test -// This file pins the behavior of getURLFromService() across the different +// This file pins the behavior of ServerURL() across the different // service types (ClusterIP, NodePort, LoadBalancer), Ingress exposure, and the // TLS SAN fallback logic. // @@ -64,7 +64,7 @@ func TestURLGeneration_ClusterIP(t *testing.T) { cluster, svc := createClusterIPService("test-cluster", "default", tt.servicePort) fakeClient := createFakeClient(t, cluster, svc) - url, err := getURLFromService(t.Context(), fakeClient, cluster, tt.hostServerIP) + url, err := server.ServerURL(t.Context(), fakeClient, cluster, tt.hostServerIP) require.NoError(t, err) assert.Equal(t, tt.expectedURL, url.String()) @@ -104,7 +104,7 @@ func TestURLGeneration_NodePort(t *testing.T) { cluster, svc := createNodePortService("test-cluster", "default", tt.servicePort, tt.nodePort) fakeClient := createFakeClient(t, cluster, svc) - url, err := getURLFromService(t.Context(), fakeClient, cluster, tt.hostServerIP) + url, err := server.ServerURL(t.Context(), fakeClient, cluster, tt.hostServerIP) require.NoError(t, err) assert.Equal(t, tt.expectedURL, url.String()) @@ -154,7 +154,7 @@ func TestURLGeneration_LoadBalancer(t *testing.T) { cluster, svc := createLoadBalancerService("test-cluster", "default", tt.servicePort, tt.lbIP, tt.lbHostname) fakeClient := createFakeClient(t, cluster, svc) - url, err := getURLFromService(t.Context(), fakeClient, cluster, tt.hostServerIP) + url, err := server.ServerURL(t.Context(), fakeClient, cluster, tt.hostServerIP) require.NoError(t, err) assert.Equal(t, tt.expectedURL, url.String()) @@ -181,7 +181,7 @@ func TestURLGeneration_Ingress(t *testing.T) { cluster, svc, ingress := createIngressService("test-cluster", "default", tt.ingressHost) fakeClient := createFakeClient(t, cluster, svc, ingress) - url, err := getURLFromService(t.Context(), fakeClient, cluster, "10.0.0.1") + url, err := server.ServerURL(t.Context(), fakeClient, cluster, "10.0.0.1") require.NoError(t, err) assert.Equal(t, tt.expectedURL, url.String()) @@ -256,7 +256,7 @@ func TestURLGeneration_TLSSANs(t *testing.T) { fakeClient := createFakeClient(t, cluster, svc) - url, err := getURLFromService(t.Context(), fakeClient, cluster, tt.hostServerIP) + url, err := server.ServerURL(t.Context(), fakeClient, cluster, tt.hostServerIP) require.NoError(t, err) assert.Equal(t, tt.expectedURL, url.String()) diff --git a/pkg/controller/cluster/server/server.go b/pkg/controller/cluster/server/server.go index 330ceefa..8ea304cb 100644 --- a/pkg/controller/cluster/server/server.go +++ b/pkg/controller/cluster/server/server.go @@ -22,7 +22,6 @@ import ( "github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1" "github.com/rancher/k3k/pkg/controller" - "github.com/rancher/k3k/pkg/controller/cluster/agent" "github.com/rancher/k3k/pkg/controller/cluster/mounts" ) @@ -250,8 +249,10 @@ func (s *Server) podSpec(ctx context.Context, image, name string, persistent boo }, }, } - // start the pod unprivileged in shared mode - if s.mode == agent.VirtualNodeMode { + // virtual mode runs an embedded kubelet inside the server pod and therefore + // requires Privileged. shared and hcp modes are agentless (no kubelet) and + // run unprivileged. + if s.mode == string(v1beta1.VirtualClusterMode) { podSpec.Containers[0].SecurityContext = &corev1.SecurityContext{ Privileged: ptr.To(true), } diff --git a/pkg/controller/cluster/server/template.go b/pkg/controller/cluster/server/template.go index 334f946f..08a4fdd9 100644 --- a/pkg/controller/cluster/server/template.go +++ b/pkg/controller/cluster/server/template.go @@ -19,7 +19,7 @@ safe_mode() { CURRENT_IP=$(cat /var/lib/rancher/k3s/k3k-node-ip) fi - if [ -z "$CURRENT_IP" ] || [ "$CURRENT_IP" = "$POD_IP" ] || [ {{.K3K_MODE}} != "virtual" ]; then + if [ -z "$CURRENT_IP" ] || [ "$CURRENT_IP" = "$POD_IP" ] || [ "{{.K3K_MODE}}" != "virtual" ]; then return fi @@ -116,7 +116,8 @@ configure_cgroups() { fi # only configure the cgroups if the runtime used is the default and the mode is virtual - if [ -n "$runtime_class" ] || [ "{{.K3K_MODE}}" != "virtual" ]; then + # shared and hcp run agentless (no kubelet) and don't need cgroup overrides. + if [ -n "$runtime_class" ] || [ "{{.K3K_MODE}}" != "virtual" ]; then return fi diff --git a/pkg/controller/kubeconfig/kubeconfig.go b/pkg/controller/kubeconfig/kubeconfig.go index b7a2cfc4..565514ba 100644 --- a/pkg/controller/kubeconfig/kubeconfig.go +++ b/pkg/controller/kubeconfig/kubeconfig.go @@ -3,22 +3,13 @@ package kubeconfig import ( "context" "crypto/x509" - "fmt" - "net" - "net/url" - "slices" - "strconv" "time" - "k8s.io/apimachinery/pkg/types" "k8s.io/apiserver/pkg/authentication/user" "sigs.k8s.io/controller-runtime/pkg/client" certutil "github.com/rancher/dynamiclistener/cert" - corev1 "k8s.io/api/core/v1" - networkingv1 "k8s.io/api/networking/v1" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" - ctrl "sigs.k8s.io/controller-runtime" "github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1" "github.com/rancher/k3k/pkg/controller" @@ -63,7 +54,7 @@ func (k *KubeConfig) Generate(ctx context.Context, client client.Client, cluster return nil, err } - serverURL, err := getURLFromService(ctx, client, cluster, hostServerIP) + serverURL, err := server.ServerURL(ctx, client, cluster, hostServerIP) if err != nil { return nil, err } @@ -95,119 +86,3 @@ func NewConfig(url string, serverCA, clientCert, clientKey []byte) *clientcmdapi return config } - -// getURLFromService generates the API server URL for the kubeconfig based on the service configuration. -// -// It handles internal vs external access patterns: -// - Internal access (hostServerIP == service.ClusterIP): uses the ClusterIP for direct pod-to-pod communication -// - External access (hostServerIP != service.ClusterIP): uses the appropriate external endpoint based on the service type -// -// Service type handling: -// - ClusterIP: uses service.Spec.ClusterIP (internal-only) -// - NodePort: uses hostServerIP:NodePort for external access, ClusterIP:Port for internal access -// - LoadBalancer: uses the LoadBalancer ingress IP, falling back to its hostname -// - Ingress (if configured): takes precedence over the service-based URL -// -// The hostServerIP parameter determines the access pattern: -// - Controller reconciliation: passes service.Spec.ClusterIP → internal access -// - CLI kubeconfig export: passes the external host → external access -func getURLFromService(ctx context.Context, c client.Client, cluster *v1beta1.Cluster, hostServerIP string) (*url.URL, error) { - log := ctrl.LoggerFrom(ctx) - - key := types.NamespacedName{ - Name: server.ServiceName(cluster.Name), - Namespace: cluster.Namespace, - } - - // Check if ingress is configured - if cluster.Spec.Expose != nil && cluster.Spec.Expose.Ingress != nil { - key := types.NamespacedName{ - Name: server.IngressName(cluster.Name), - Namespace: cluster.Namespace, - } - - var k3kIngress networkingv1.Ingress - if err := c.Get(ctx, key, &k3kIngress); err != nil { - return nil, err - } - - if len(k3kIngress.Spec.Rules) > 0 && k3kIngress.Spec.Rules[0].Host != "" { - return url.Parse(fmt.Sprintf("https://%s", k3kIngress.Spec.Rules[0].Host)) - } - - log.V(1).Info("Ingress has no rule with a host set, falling back to the service URL.") - } - - // Fall back to Service-based URL - var k3kService corev1.Service - if err := c.Get(ctx, key, &k3kService); err != nil { - return nil, err - } - - // init to hostServerIP and 443 port - ip := hostServerIP - port := int32(443) - - // Use service port as default if available - if len(k3kService.Spec.Ports) > 0 { - port = k3kService.Spec.Ports[0].Port - } - - // Handle each service type separately - switch k3kService.Spec.Type { - case corev1.ServiceTypeClusterIP: - ip = k3kService.Spec.ClusterIP - - case corev1.ServiceTypeNodePort: - // Only use NodePort if hostServerIP is NOT the ClusterIP - // If hostServerIP == ClusterIP, this is an internal connection, use ClusterIP - if hostServerIP != k3kService.Spec.ClusterIP { - if len(k3kService.Spec.Ports) > 0 { - port = k3kService.Spec.Ports[0].NodePort - } - } else { - // Internal connection: use ClusterIP - ip = k3kService.Spec.ClusterIP - } - - case corev1.ServiceTypeLoadBalancer: - if len(k3kService.Status.LoadBalancer.Ingress) > 0 { - ingress := k3kService.Status.LoadBalancer.Ingress[0] - - switch { - case ingress.IP != "": - ip = ingress.IP - case ingress.Hostname != "": - ip = ingress.Hostname - default: - log.V(1).Info("No usable ingress address found in LoadBalancer service.") - } - } - } - - if !slices.Contains(cluster.Status.TLSSANs, ip) { - log.V(1).Info(fmt.Sprintf("IP %s not in tlsSANs.", ip)) - - if len(cluster.Spec.TLSSANs) > 0 { - log.V(1).Info("Using the first TLS SAN in the spec as a fallback: " + cluster.Spec.TLSSANs[0]) - - ip = cluster.Spec.TLSSANs[0] - } else if len(cluster.Status.TLSSANs) > 0 { - log.V(1).Info("No explicit tlsSANs specified. Trying to use the first TLS SAN in the status: " + cluster.Status.TLSSANs[0]) - - ip = cluster.Status.TLSSANs[0] - } else { - log.V(1).Info("IP not found in tlsSANs. This could cause issue with the certificate validation.") - } - } - - // Build URL with port only if not the default HTTPS port - var rawURL string - if port != int32(443) { - rawURL = fmt.Sprintf("https://%s", net.JoinHostPort(ip, strconv.Itoa(int(port)))) - } else { - rawURL = fmt.Sprintf("https://%s", ip) - } - - return url.Parse(rawURL) -} diff --git a/tests/e2e/cluster_create_test.go b/tests/e2e/cluster_create_test.go index dcd6f702..bd7ba927 100644 --- a/tests/e2e/cluster_create_test.go +++ b/tests/e2e/cluster_create_test.go @@ -5,9 +5,11 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1" + k3kcluster "github.com/rancher/k3k/pkg/controller/cluster" fwk3k "github.com/rancher/k3k/tests/framework/k3k" . "github.com/onsi/ginkgo/v2" @@ -136,3 +138,93 @@ var _ = When("creating a shared mode cluster", Label(e2eTestLabel), Label(slowTe Should(Succeed()) }) }) + +var _ = When("creating an HCP mode cluster", Label(e2eTestLabel), Label(slowTestsLabel), func() { + var virtualCluster *VirtualCluster + + BeforeEach(func() { + namespace := fwk3k.CreateNamespace(k8s) + + DeferCleanup(func() { + fwk3k.DeleteNamespaces(k8s, namespace.Name) + }) + + cluster := NewCluster(namespace.Name) + cluster.Spec.Mode = v1beta1.HCPClusterMode + + CreateCluster(cluster) + client, restConfig := NewVirtualK8sClientAndConfig(cluster) + + virtualCluster = &VirtualCluster{ + Cluster: cluster, + RestConfig: restConfig, + Client: client, + } + }) + + It("is up and running", func() { + Eventually(func(g Gomega) { + ctx := GinkgoT().Context() + key := client.ObjectKeyFromObject(virtualCluster.Cluster) + g.Expect(k8sClient.Get(ctx, key, virtualCluster.Cluster)).To(Succeed()) + g.Expect(virtualCluster.Cluster.Status.Phase).To(BeEquivalentTo(v1beta1.ClusterReady)) + }). + WithTimeout(time.Minute). + WithPolling(time.Second). + Should(Succeed()) + }) + + It("creates a populated token secret", func() { + ctx := GinkgoT().Context() + + Eventually(func(g Gomega) { + var tokenSecret corev1.Secret + + key := client.ObjectKey{ + Name: k3kcluster.TokenSecretName(virtualCluster.Cluster.Name), + Namespace: virtualCluster.Cluster.Namespace, + } + + g.Expect(k8sClient.Get(ctx, key, &tokenSecret)).To(Succeed()) + g.Expect(tokenSecret.Data["token"]).NotTo(BeEmpty()) + }). + WithTimeout(time.Minute). + WithPolling(time.Second). + Should(Succeed()) + }) + + It("reconciles default/kubernetes Endpoints and EndpointSlice", func() { + ctx := GinkgoT().Context() + + Eventually(func(g Gomega) { + key := client.ObjectKeyFromObject(virtualCluster.Cluster) + g.Expect(k8sClient.Get(ctx, key, virtualCluster.Cluster)).To(Succeed()) + + endpoints, err := virtualCluster.Client.CoreV1().Endpoints("default").Get(ctx, "kubernetes", metav1.GetOptions{}) + g.Expect(err).To(Not(HaveOccurred())) + g.Expect(endpoints.Subsets).To(HaveLen(1)) + g.Expect(endpoints.Subsets[0].Addresses).To(HaveLen(1)) + g.Expect(virtualCluster.Cluster.Status.TLSSANs).To(ContainElement(endpoints.Subsets[0].Addresses[0].IP)) + }). + WithTimeout(time.Minute). + WithPolling(time.Second). + Should(Succeed()) + + Eventually(func(g Gomega) { + key := client.ObjectKeyFromObject(virtualCluster.Cluster) + g.Expect(k8sClient.Get(ctx, key, virtualCluster.Cluster)).To(Succeed()) + + slices, err := virtualCluster.Client.DiscoveryV1().EndpointSlices("default").List(ctx, metav1.ListOptions{ + LabelSelector: "kubernetes.io/service-name=kubernetes", + }) + g.Expect(err).To(Not(HaveOccurred())) + g.Expect(slices.Items).To(HaveLen(1)) + g.Expect(slices.Items[0].Endpoints).To(HaveLen(1)) + g.Expect(slices.Items[0].Endpoints[0].Addresses).To(HaveLen(1)) + g.Expect(virtualCluster.Cluster.Status.TLSSANs).To(ContainElement(slices.Items[0].Endpoints[0].Addresses[0])) + }). + WithTimeout(time.Minute). + WithPolling(time.Second). + Should(Succeed()) + }) +}) diff --git a/tests/e2e/common_test.go b/tests/e2e/common_test.go index d91bcfc3..76d948a2 100644 --- a/tests/e2e/common_test.go +++ b/tests/e2e/common_test.go @@ -294,7 +294,8 @@ func (c *VirtualCluster) NewNginxPod(namespace string) (*corev1.Pod, string) { By(fmt.Sprintf("Nginx Pod is running (%s/%s)", nginxPod.Namespace, nginxPod.Name)) - // only check the pod on the host cluster if the mode is shared mode + // only check the pod on the host cluster if the mode is shared mode. + // hcp is agentless and BYO-node, so no host-side pod mirror exists. if c.Cluster.Spec.Mode != v1beta1.SharedClusterMode { return nginxPod, "" } diff --git a/tests/e2e/tests_suite_test.go b/tests/e2e/tests_suite_test.go index 8e513645..2a0487e3 100644 --- a/tests/e2e/tests_suite_test.go +++ b/tests/e2e/tests_suite_test.go @@ -75,6 +75,8 @@ var _ = BeforeSuite(func() { initKubernetesClient(ctx) + GinkgoWriter.Println("Checking K3k deployment status") + patchPVC(ctx, k8s) }) @@ -87,6 +89,8 @@ func initKubernetesClient(ctx context.Context) { restcfg = config.RestConfig k8s = config.Clientset k8sClient = config.Client + + GinkgoWriter.Println("Host IP: " + hostIP) } func patchPVC(ctx context.Context, clientset *kubernetes.Clientset) { diff --git a/tests/framework/client/config.go b/tests/framework/client/config.go index b0fd6177..5bc848ec 100644 --- a/tests/framework/client/config.go +++ b/tests/framework/client/config.go @@ -3,6 +3,7 @@ package client import ( "context" "fmt" + "net" "net/url" "os" @@ -93,6 +94,48 @@ func getServerIP(cfg *rest.Config) (string, error) { return "", fmt.Errorf("failed to parse REST config host: %w", err) } - // If Host includes a port, u.Hostname() extracts just the hostname part - return u.Hostname(), nil + host := u.Hostname() + + if isLoopbackHost(host) { + if ip, ok := firstNonLoopbackIPv4(); ok { + return ip, nil + } + } + + return host, nil +} + +func isLoopbackHost(host string) bool { + if host == "" || host == "localhost" { + return true + } + + if ip := net.ParseIP(host); ip != nil && ip.IsLoopback() { + return true + } + + return false +} + +func firstNonLoopbackIPv4() (string, bool) { + addrs, err := net.InterfaceAddrs() + if err != nil { + return "", false + } + + for _, addr := range addrs { + ipNet, ok := addr.(*net.IPNet) + if !ok { + continue + } + + ip := ipNet.IP.To4() + if ip == nil || ip.IsLoopback() || !ip.IsGlobalUnicast() { + continue + } + + return ip.String(), true + } + + return "", false }