Merge branch 'pr876' into copilot/fix-code-review-comment

This commit is contained in:
copilot-swe-agent[bot]
2026-06-17 12:39:35 +00:00
committed by GitHub
27 changed files with 1523 additions and 166 deletions
+407
View File
@@ -0,0 +1,407 @@
name: Conformance Tests - HCP Mode
on:
schedule:
- cron: "0 1 * * *"
workflow_dispatch:
inputs:
k3k_version:
description: 'K3k version to test (e.g. v1.1.0). Leave empty to build from source.'
required: false
type: string
k8s_version:
description: 'Kubernetes version to test'
required: false
type: choice
options:
- ""
- "v1.34.6"
- "v1.35.3"
permissions:
contents: read
env:
K8S_VERSIONS: "v1.34.6,v1.35.3"
HELM_VERSION: v4.1.3
HELM_CHECKSUM_AMD64: 02ce9722d541238f81459938b84cf47df2fdf1187493b4bfb2346754d82a4700
jobs:
setup:
runs-on: ubuntu-latest
outputs:
k8s_versions: ${{ steps.set-matrix.outputs.k8s_versions }}
steps:
- id: set-matrix
run: |
if [[ -z "${{ inputs.k8s_version }}" ]]; then
JSON_ARRAY=$(jq -nc '"${{ env.K8S_VERSIONS }}" | split(",")')
echo "k8s_versions=${JSON_ARRAY}" >> "$GITHUB_OUTPUT"
else
echo "k8s_versions=[\"${{ inputs.k8s_version }}\"]" >> "$GITHUB_OUTPUT"
fi
conformance:
needs: setup
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
k8s_version: ${{ fromJSON(needs.setup.outputs.k8s_versions) }}
env:
KUBERNETES_VERSION: ${{ matrix.k8s_version }}
steps:
- name: Checkout code
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
with:
fetch-depth: 0
fetch-tags: true
- uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6
with:
go-version-file: go.mod
- name: Install helm
env:
FILENAME: helm.tar.gz
run: |
curl -sSfL -o ${{ env.FILENAME }} https://get.helm.sh/helm-${{ env.HELM_VERSION }}-linux-amd64.tar.gz
echo "${{ env.HELM_CHECKSUM_AMD64 }} ${{ env.FILENAME }}" | sha256sum --check
tar -xvzf ${{ env.FILENAME }} linux-amd64/helm
sudo install -m 755 linux-amd64/helm /usr/local/bin/helm
rm -fr "${{ env.FILENAME }}" linux-amd64/helm
- name: Install hydrophone
run: go install sigs.k8s.io/hydrophone@3de3e886a2f6f09635d8b981c195490af1584d97 #v0.7.0
- name: Install k3s
env:
KUBECONFIG: /etc/rancher/k3s/k3s.yaml
K3S_HOST_VERSION: ${{ env.KUBERNETES_VERSION }}+k3s1
run: |
curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=${K3S_HOST_VERSION} INSTALL_K3S_EXEC="--write-kubeconfig-mode=777" sh -s -
kubectl cluster-info
kubectl get nodes
echo "KUBECONFIG=${KUBECONFIG}" >> $GITHUB_ENV
- name: Setup K3k (from source)
if: inputs.k3k_version == ''
run: |
export REPO=ttl.sh/$(uuidgen)
export VERSION=1h
make build
make package
make push
make install
# add k3kcli to $PATH
echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
- name: Setup K3k (from release)
if: inputs.k3k_version != ''
run: |
K3K_VERSION="${{ inputs.k3k_version }}"
CHART_VERSION="${K3K_VERSION#v}"
helm repo add k3k https://rancher.github.io/k3k
helm repo update
helm install --namespace k3k-system --create-namespace --version "${CHART_VERSION}" k3k k3k/k3k
wget -qO k3kcli "https://github.com/rancher/k3k/releases/download/${{ inputs.k3k_version }}/k3kcli-linux-amd64"
sudo mv k3kcli /usr/local/bin/k3kcli
sudo chmod +x /usr/local/bin/k3kcli
- name: Wait for K3k controller
run: |
echo "Wait for K3k controller deployment to be available"
kubectl wait -n k3k-system deployment -l "app.kubernetes.io/name=k3k" --for=condition=Available --timeout=5m
- name: Check k3kcli
run: k3kcli -v
- name: Create virtual cluster
run: |
kubectl create namespace k3k-mycluster
cat <<EOF | kubectl apply -f -
apiVersion: k3k.io/v1beta1
kind: Cluster
metadata:
name: mycluster
namespace: k3k-mycluster
spec:
mode: hcp
tlsSANs:
- "127.0.0.1"
- "192.168.100.1"
expose:
nodePort:
serverPort: 30001
EOF
echo "Wait for bootstrap secret to be available"
kubectl wait -n k3k-mycluster --for=create secret k3k-mycluster-bootstrap --timeout=5m
k3kcli kubeconfig generate --name mycluster
export KUBECONFIG=${{ github.workspace }}/k3k-mycluster-mycluster-kubeconfig.yaml
kubectl cluster-info
kubectl get nodes
kubectl get pods -A
- name: Install Virtualization Dependencies
run: |
sudo apt-get update
sudo apt-get install -y qemu-kvm qemu-utils cloud-image-utils wget
sudo usermod -aG kvm $USER
kvm-ok
- name: Set up bridge network for VMs
run: |
# Create a bridge so both VMs share an L2 segment with unique routable IPs.
# Required because QEMU `-net user` gives every VM the same 10.0.2.15 NAT
# address, which breaks flannel VXLAN between workers.
sudo ip link add name k3kbr0 type bridge
sudo ip addr add 192.168.100.1/24 dev k3kbr0
sudo ip link set k3kbr0 up
# NAT outbound so VMs can reach the internet (image pulls etc).
sudo sysctl -w net.ipv4.ip_forward=1
sudo iptables -t nat -A POSTROUTING -s 192.168.100.0/24 ! -o k3kbr0 -j MASQUERADE
sudo iptables -A FORWARD -i k3kbr0 -j ACCEPT
sudo iptables -A FORWARD -o k3kbr0 -j ACCEPT
# One TAP per VM, attached to the bridge.
sudo ip tuntap add tap-w1 mode tap
sudo ip link set tap-w1 master k3kbr0
sudo ip link set tap-w1 up
sudo ip tuntap add tap-w2 mode tap
sudo ip link set tap-w2 master k3kbr0
sudo ip link set tap-w2 up
- name: Download Base Cloud Image
run: |
# PINNED TO UBUNTU 24.04 LTS (noble).
#
# Newer Ubuntu releases (tested: 26.04 "resolute") ship a stricter
# `cri-containerd.apparmor.d` profile that denies inter-thread signal
# delivery. The BIND ISC library used by `nslookup` relies on those
# signals during shutdown (`isc_app_ctxshutdown()` calls `kill()`),
# so when AppArmor denies them nslookup exits 139 with
# "kill: Permission denied". The conformance tests
# `[sig-network] Services should be able to change the type from
# {NodePort,ClusterIP} to ExternalName`
# both run `nslookup` from an exec pod and fail in that case.
#
# Before bumping past 24.04, verify those two conformance tests still
# pass — or that the containerd AppArmor profile on the newer release
# has been relaxed to allow intra-pod signals.
wget -q https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img -O ubuntu-cloudimg.img
- name: Generate SSH Key and Cloud-Init Seeds
run: |
ssh-keygen -t rsa -b 4096 -f ./id_rsa -N ""
PUBKEY="$(cat ./id_rsa.pub)"
# Per-VM cloud-init: each worker gets a unique hostname and a static IP
# on the bridge subnet via netplan.
for i in 1 2; do
IP="192.168.100.1${i}" # 192.168.100.11 / 192.168.100.12
cat <<EOF > user-data-${i}
#cloud-config
hostname: worker-${i}
preserve_hostname: false
manage_etc_hosts: true
# Stop cloud-init from generating its own DHCP netplan that would
# conflict with the static one we write in write_files below.
network:
config: disabled
users:
- name: ubuntu
ssh_authorized_keys:
- ${PUBKEY}
sudo: ['ALL=(ALL) NOPASSWD:ALL']
shell: /bin/bash
write_files:
- path: /etc/netplan/50-static.yaml
permissions: '0600'
content: |
network:
version: 2
ethernets:
ens3:
dhcp4: false
addresses: [${IP}/24]
routes:
- to: default
via: 192.168.100.1
nameservers:
addresses: [8.8.8.8, 1.1.1.1]
runcmd:
- netplan apply
EOF
cloud-localds seed-${i}.img user-data-${i}
done
- name: Create Worker Disks
run: |
qemu-img create -f qcow2 -b ubuntu-cloudimg.img -F qcow2 worker-1.qcow2 20G
qemu-img create -f qcow2 -b ubuntu-cloudimg.img -F qcow2 worker-2.qcow2 20G
- name: Launch Worker VMs
run: |
# Launch Worker 1 — attached to tap-w1 on k3kbr0.
sudo qemu-system-x86_64 \
-m 2048 -smp 2 -cpu host -enable-kvm -nographic \
-drive file=worker-1.qcow2,if=virtio \
-drive file=seed-1.img,format=raw,if=virtio \
-netdev tap,id=net0,ifname=tap-w1,script=no,downscript=no \
-device virtio-net-pci,netdev=net0,mac=52:54:00:12:34:56 \
&
# Wait a moment before launching the second VM
sleep 5
# Launch Worker 2 — attached to tap-w2 on k3kbr0.
sudo qemu-system-x86_64 \
-m 2048 -smp 2 -cpu host -enable-kvm -nographic \
-drive file=worker-2.qcow2,if=virtio \
-drive file=seed-2.img,format=raw,if=virtio \
-netdev tap,id=net0,ifname=tap-w2,script=no,downscript=no \
-device virtio-net-pci,netdev=net0,mac=52:54:00:12:34:57 \
&
- name: Wait for SSH Availability
run: |
echo "Waiting for Worker 1 (192.168.100.11) to respond..."
timeout 180s bash -c '
until ssh -i ./id_rsa -o StrictHostKeyChecking=no -o ConnectTimeout=2 ubuntu@192.168.100.11 true 2>/dev/null; do sleep 3; done
'
echo "Waiting for Worker 2 (192.168.100.12) to respond..."
timeout 180s bash -c '
until ssh -i ./id_rsa -o StrictHostKeyChecking=no -o ConnectTimeout=2 ubuntu@192.168.100.12 true 2>/dev/null; do sleep 3; done
'
echo "Both VMs are up and running!"
echo "Testing connectivity from VMs to K3k API server..."
ssh -i ./id_rsa -o StrictHostKeyChecking=no ubuntu@192.168.100.11 \
"curl -kv --max-time 10 https://192.168.100.1:30001/readyz || echo 'Worker 1 connectivity test failed'"
ssh -i ./id_rsa -o StrictHostKeyChecking=no ubuntu@192.168.100.12 \
"curl -kv --max-time 10 https://192.168.100.1:30001/readyz || echo 'Worker 2 connectivity test failed'"
- name: Verify Worker VM Configuration
run: |
for IP in 192.168.100.11 192.168.100.12; do
echo "=== Worker at ${IP} ==="
ssh -i ./id_rsa -o StrictHostKeyChecking=no ubuntu@${IP} \
"echo 'Hostname:' \$(hostname) && \
echo 'IP Address:' \$(ip -4 addr show ens3 | grep -oP '(?<=inet\s)\d+(\.\d+){3}') && \
echo 'Gateway:' \$(ip route | grep default)"
echo ""
done
- name: Join Workers to K3k Control Plane
env:
K3S_WORKER_VERSION: ${{ env.KUBERNETES_VERSION }}+k3s1
run: |
K3S_TOKEN=$(kubectl get secret -n k3k-mycluster k3k-mycluster-token -o jsonpath='{.data.token}' | base64 -d)
echo "Registering Worker 1 (k3s ${K3S_WORKER_VERSION})..."
ssh -i ./id_rsa -o StrictHostKeyChecking=no ubuntu@192.168.100.11 \
"curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=${K3S_WORKER_VERSION} K3S_URL=https://192.168.100.1:30001 K3S_TOKEN=${K3S_TOKEN} sh -"
echo "Registering Worker 2 (k3s ${K3S_WORKER_VERSION})..."
ssh -i ./id_rsa -o StrictHostKeyChecking=no ubuntu@192.168.100.12 \
"curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=${K3S_WORKER_VERSION} K3S_URL=https://192.168.100.1:30001 K3S_TOKEN=${K3S_TOKEN} sh -"
- name: Verify Cluster Nodes
env:
KUBECONFIG: ${{ github.workspace }}/k3k-mycluster-mycluster-kubeconfig.yaml
run: |
echo "Monitoring K3k Virtual Cluster for Node registration..."
kubectl get pod -A
kubectl get nodes
timeout 180s bash -c '
until [ $(kubectl get nodes --no-headers 2>/dev/null | grep -c "Ready") -eq 2 ]; do
echo "Waiting for both worker nodes to show Ready..."
kubectl get nodes || true
sleep 5
done
'
- name: Run conformance tests
run: |
hydrophone --conformance --parallel 4 \
--kubeconfig ${{ github.workspace }}/k3k-mycluster-mycluster-kubeconfig.yaml \
--output-dir /tmp
- name: Collect logs
if: always()
env:
KUBECONFIG: /etc/rancher/k3s/k3s.yaml
run: |
journalctl -u k3s -o cat --no-pager > /tmp/k3s.log
kubectl logs -n k3k-system -l "app.kubernetes.io/name=k3k" --tail=-1 > /tmp/k3k.log
- name: Archive K3s logs
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
if: always()
with:
name: k3s-${{ matrix.k8s_version }}-logs
path: /tmp/k3s.log
- name: Archive K3k logs
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
if: always()
with:
name: k3k-${{ matrix.k8s_version }}-logs
path: /tmp/k3k.log
- name: Archive conformance logs
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
if: always()
with:
name: conformance-${{ matrix.k8s_version }}-logs
path: /tmp/e2e.log
- name: Archive results
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
if: always()
with:
name: conformance-${{ matrix.k8s_version }}-results
path: /tmp/junit_01.xml
- name: Job Summary
if: always()
run: |
echo '## 📊 Conformance Tests Results (${{ matrix.k8s_version }})' >> $GITHUB_STEP_SUMMARY
echo '| Passed | Failed | Pending | Skipped |' >> $GITHUB_STEP_SUMMARY
echo '|---|---|---|---|' >> $GITHUB_STEP_SUMMARY
RESULTS=$(tail -10 /tmp/e2e.log | grep -E "Passed .* Failed .* Pending .* Skipped" | cut -d '-' -f 3)
RESULTS=$(echo $RESULTS | grep -oE '[0-9]+' | xargs | sed 's/ / | /g')
echo "| $RESULTS |" >> $GITHUB_STEP_SUMMARY
# only include failed tests section if there are any
if grep -q '\[FAIL\]' /tmp/e2e.log; then
echo '' >> $GITHUB_STEP_SUMMARY
echo '### Failed Tests' >> $GITHUB_STEP_SUMMARY
echo '```' >> $GITHUB_STEP_SUMMARY
grep '\[FAIL\]' /tmp/e2e.log >> $GITHUB_STEP_SUMMARY
echo '```' >> $GITHUB_STEP_SUMMARY
fi
+5
View File
@@ -11,3 +11,8 @@ __debug*
cover.out
covcounters.**
covmeta.**
.claude/
.claude-flow/
.swarm
.mcp.json
CLAUDE.md
@@ -1289,17 +1289,14 @@ spec:
are mirrored into the virtual cluster.
type: boolean
mode:
allOf:
- enum:
- shared
- virtual
- enum:
- shared
- virtual
default: shared
description: |-
Mode specifies the cluster provisioning mode: "shared" or "virtual".
Mode specifies the cluster provisioning mode: "shared", "virtual" or "hcp".
Defaults to "shared". This field is immutable.
enum:
- shared
- virtual
- hcp
type: string
x-kubernetes-validations:
- message: mode is immutable
@@ -1419,7 +1416,7 @@ spec:
role:
description: |-
Role is the type of the k3k pod that will be used to mount the secret.
This can be 'server', 'agent', or 'all' (for both).
This can be `server`, `agent`, or `all` (for both).
enum:
- server
- agent
@@ -55,6 +55,7 @@ spec:
enum:
- shared
- virtual
- hcp
type: string
x-kubernetes-validations:
- message: mode is immutable
+33 -11
View File
@@ -5,7 +5,6 @@ import (
"context"
"errors"
"fmt"
"net/url"
"os"
"strings"
"text/template"
@@ -84,10 +83,14 @@ func createAction(appCtx *AppContext, config *CreateConfig) func(cmd *cobra.Comm
return errors.New("invalid cluster name")
}
if config.mode == string(v1beta1.SharedClusterMode) && config.agents != 0 {
if config.agents > 0 && config.mode != string(v1beta1.VirtualClusterMode) {
return errors.New("invalid flag, --agents flag is only allowed in virtual mode")
}
if config.mode == string(v1beta1.HCPClusterMode) {
logrus.Warn("HCP (Hosted Control Plane) mode is experimental.")
}
namespace := appCtx.Namespace(name)
if err := createNamespace(ctx, client, namespace, config.policy); err != nil {
@@ -128,17 +131,12 @@ func createAction(appCtx *AppContext, config *CreateConfig) func(cmd *cobra.Comm
}
// add Host IP address as an extra TLS-SAN to expose the k3k cluster
url, err := url.Parse(appCtx.RestConfig.Host)
host, err := resolveServerHost(appCtx.RestConfig.Host, config.kubeconfigServerHost)
if err != nil {
return err
}
host := strings.Split(url.Host, ":")
if config.kubeconfigServerHost != "" {
host = []string{config.kubeconfigServerHost}
}
cluster.Spec.TLSSANs = []string{host[0]}
cluster.Spec.TLSSANs = []string{host}
if err := client.Create(ctx, cluster); err != nil {
if apierrors.IsAlreadyExists(err) {
@@ -179,16 +177,40 @@ func createAction(appCtx *AppContext, config *CreateConfig) func(cmd *cobra.Comm
var kubeconfig *clientcmdapi.Config
if err := retry.OnError(availableBackoff, apierrors.IsNotFound, func() error {
kubeconfig, err = cfg.Generate(ctx, client, cluster, host[0], 0)
kubeconfig, err = cfg.Generate(ctx, client, cluster, host, 0)
return err
}); err != nil {
return err
}
return writeKubeconfigFile(cluster, kubeconfig, "")
if err := writeKubeconfigFile(cluster, kubeconfig, ""); err != nil {
return err
}
if cluster.Spec.Mode == v1beta1.HCPClusterMode {
printHCPJoinInstructions(cluster, kubeconfig)
}
return nil
}
}
func printHCPJoinInstructions(cluster *v1beta1.Cluster, kc *clientcmdapi.Config) {
tokenSecretName := k3kcluster.TokenSecretName(cluster.Name)
serverURL := kc.Clusters["default"].Server
logrus.Infof(`To join an external worker node to this HCP cluster:
1. On this machine, fetch the cluster token:
kubectl get secret -n %s %s -o jsonpath='{.data.token}' | base64 -d
2. On the worker node, run (replace <TOKEN> with the value from step 1):
curl -sfL https://get.k3s.io | K3S_URL=%s K3S_TOKEN=<TOKEN> sh -
`, cluster.Namespace, tokenSecretName, serverURL)
}
func newCluster(name, namespace string, config *CreateConfig) (*v1beta1.Cluster, error) {
var storageRequestSize *resource.Quantity
if config.storageRequestSize != "" {
+3 -3
View File
@@ -27,7 +27,7 @@ func createFlags(cmd *cobra.Command, cfg *CreateConfig) {
cmd.Flags().StringArrayVar(&cfg.labels, "labels", []string{}, "Labels to add to the cluster object (e.g. key=value)")
cmd.Flags().StringArrayVar(&cfg.annotations, "annotations", []string{}, "Annotations to add to the cluster object (e.g. key=value)")
cmd.Flags().StringVar(&cfg.version, "version", "", "k3s version")
cmd.Flags().StringVar(&cfg.mode, "mode", "shared", "k3k mode type (shared, virtual)")
cmd.Flags().StringVar(&cfg.mode, "mode", "shared", "k3k mode type (shared, virtual, hcp)")
cmd.Flags().StringVar(&cfg.kubeconfigServerHost, "kubeconfig-server", "", "override the kubeconfig server host")
cmd.Flags().StringVar(&cfg.policy, "policy", "", "The policy to create the cluster in")
cmd.Flags().StringVar(&cfg.customCertsPath, "custom-certs", "", "The path for custom certificate directory")
@@ -54,10 +54,10 @@ func validateCreateConfig(cfg *CreateConfig) error {
if cfg.mode != "" {
switch cfg.mode {
case string(v1beta1.VirtualClusterMode), string(v1beta1.SharedClusterMode):
case string(v1beta1.VirtualClusterMode), string(v1beta1.SharedClusterMode), string(v1beta1.HCPClusterMode):
return nil
default:
return errors.New(`mode should be one of "shared" or "virtual"`)
return errors.New(`mode should be one of "shared", "virtual" or "hcp"`)
}
}
+27 -6
View File
@@ -5,7 +5,6 @@ import (
"net/url"
"os"
"path/filepath"
"strings"
"time"
"github.com/sirupsen/logrus"
@@ -89,14 +88,12 @@ func generate(appCtx *AppContext, cfg *GenerateKubeconfigConfig) func(cmd *cobra
return err
}
url, err := url.Parse(appCtx.RestConfig.Host)
host, err := resolveServerHost(appCtx.RestConfig.Host, cfg.kubeconfigServerHost)
if err != nil {
return err
}
host := strings.Split(url.Host, ":")
if cfg.kubeconfigServerHost != "" {
host = []string{cfg.kubeconfigServerHost}
cfg.altNames = append(cfg.altNames, cfg.kubeconfigServerHost)
}
@@ -118,16 +115,40 @@ func generate(appCtx *AppContext, cfg *GenerateKubeconfigConfig) func(cmd *cobra
var kubeconfig *clientcmdapi.Config
if err := retry.OnError(controller.Backoff, apierrors.IsNotFound, func() error {
kubeconfig, err = kubeCfg.Generate(ctx, client, &cluster, host[0], 0)
kubeconfig, err = kubeCfg.Generate(ctx, client, &cluster, host, 0)
return err
}); err != nil {
return err
}
return writeKubeconfigFile(&cluster, kubeconfig, cfg.configName)
if err := writeKubeconfigFile(&cluster, kubeconfig, cfg.configName); err != nil {
return err
}
if cluster.Spec.Mode == v1beta1.HCPClusterMode {
printHCPJoinInstructions(&cluster, kubeconfig)
}
return nil
}
}
// resolveServerHost returns the host that should be embedded in the kubeconfig
// server URL and used as the TLS-SAN. If override is set it takes precedence;
// otherwise the host is extracted from restConfigHost.
func resolveServerHost(restConfigHost, override string) (string, error) {
if override != "" {
return override, nil
}
u, err := url.Parse(restConfigHost)
if err != nil {
return "", err
}
return u.Hostname(), nil
}
func writeKubeconfigFile(cluster *v1beta1.Cluster, kubeconfig *clientcmdapi.Config, configName string) error {
if configName == "" {
configName = cluster.Namespace + "-" + cluster.Name + "-kubeconfig.yaml"
+2 -2
View File
@@ -34,10 +34,10 @@ func NewPolicyCreateCmd(appCtx *AppContext) *cobra.Command {
Example: "k3kcli policy create [command options] NAME",
PreRunE: func(cmd *cobra.Command, args []string) error {
switch config.mode {
case string(v1beta1.VirtualClusterMode), string(v1beta1.SharedClusterMode):
case string(v1beta1.VirtualClusterMode), string(v1beta1.SharedClusterMode), string(v1beta1.HCPClusterMode):
return nil
default:
return errors.New(`mode should be one of "shared" or "virtual"`)
return errors.New(`mode should be one of "shared", "virtual" or "hcp"`)
}
},
RunE: policyCreateAction(appCtx, config),
+1 -1
View File
@@ -54,7 +54,7 @@ k3kcli cluster create [command options] NAME
--kubeconfig-server string override the kubeconfig server host
--labels stringArray Labels to add to the cluster object (e.g. key=value)
--mirror-host-nodes Mirror Host Cluster Nodes
--mode string k3k mode type (shared, virtual) (default "shared")
--mode string k3k mode type (shared, virtual, hcp) (default "shared")
-n, --namespace string namespace of the k3k cluster
--persistence-type string persistence mode for the nodes (dynamic, ephemeral) (default "dynamic")
--policy string The policy to create the cluster in
+1 -1
View File
@@ -25,7 +25,7 @@ k3kcli cluster create [command options] NAME
--kubeconfig-server string override the kubeconfig server host
--labels stringArray Labels to add to the cluster object (e.g. key=value)
--mirror-host-nodes Mirror Host Cluster Nodes
--mode string k3k mode type (shared, virtual) (default "shared")
--mode string k3k mode type (shared, virtual, hcp) (default "shared")
-n, --namespace string namespace of the k3k cluster
--persistence-type string persistence mode for the nodes (dynamic, ephemeral) (default "dynamic")
--policy string The policy to create the cluster in
+10 -8
View File
@@ -133,8 +133,9 @@ _Underlying type:_ _string_
ClusterMode is the possible provisioning mode of a Cluster.
_Validation:_
- Enum: [shared virtual]
Supported values: `shared`, `virtual`, `hcp`.
_Appears In:_
@@ -177,8 +178,8 @@ _Appears In:_
| *`version`* __string__ | Version is the K3s version to use for the virtual nodes. +
It should follow the K3s versioning convention (e.g., v1.28.2-k3s1). +
If not specified, the Kubernetes version of the host node will be used. + | |
| *`mode`* __xref:{anchor_prefix}-github-com-rancher-k3k-pkg-apis-k3k-io-v1beta1-clustermode[$$ClusterMode$$]__ | Mode specifies the cluster provisioning mode: "shared" or "virtual". +
Defaults to "shared". This field is immutable. + | shared | Enum: [shared virtual] +
| *`mode`* __xref:{anchor_prefix}-github-com-rancher-k3k-pkg-apis-k3k-io-v1beta1-clustermode[$$ClusterMode$$]__ | Mode specifies the cluster provisioning mode: "shared", "virtual" or "hcp". +
Defaults to "shared". This field is immutable. + | shared | Enum: [shared virtual hcp] +
| *`servers`* __integer__ | Servers specifies the number of K3s pods to run in server (control plane) mode. +
Must be at least 1. Defaults to 1. + | 1 |
@@ -547,8 +548,9 @@ _Underlying type:_ _string_
PodSecurityAdmissionLevel is the policy level applied to the pods in the namespace.
_Validation:_
- Enum: [privileged baseline restricted]
Supported values: `privileged`, `baseline`, `restricted`.
_Appears In:_
@@ -618,7 +620,7 @@ secret contents will be mounted. + | |
When specified, only the specified key from the secret will be mounted as a file +
at MountPath, keeping the parent directory writable. + | |
| *`role`* __string__ | Role is the type of the k3k pod that will be used to mount the secret. +
This can be 'server', 'agent', or 'all' (for both). + | | Enum: [server agent all] +
This can be `server`, `agent`, or `all` (for both). + | | Enum: [server agent all] +
|===
@@ -790,7 +792,7 @@ to set defaults and constraints (min/max) + | |
This includes both node affinity and pod affinity/anti-affinity rules. + | |
| *`defaultAgentAffinity`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#affinity-v1-core[$$Affinity$$]__ | DefaultAgentAffinity specifies the affinity rules applied to agent pods of all clusters in the target Namespace. +
This includes both node affinity and pod affinity/anti-affinity rules. + | |
| *`allowedMode`* __xref:{anchor_prefix}-github-com-rancher-k3k-pkg-apis-k3k-io-v1beta1-clustermode[$$ClusterMode$$]__ | AllowedMode specifies the allowed cluster provisioning mode. Defaults to "shared". + | shared | Enum: [shared virtual] +
| *`allowedMode`* __xref:{anchor_prefix}-github-com-rancher-k3k-pkg-apis-k3k-io-v1beta1-clustermode[$$ClusterMode$$]__ | AllowedMode specifies the allowed cluster provisioning mode. Defaults to "shared". + | shared | Enum: [shared virtual hcp] +
| *`disableNetworkPolicy`* __boolean__ | DisableNetworkPolicy indicates whether to disable the creation of a default network policy for cluster isolation. + | |
| *`podSecurityAdmissionLevel`* __xref:{anchor_prefix}-github-com-rancher-k3k-pkg-apis-k3k-io-v1beta1-podsecurityadmissionlevel[$$PodSecurityAdmissionLevel$$]__ | PodSecurityAdmissionLevel specifies the pod security admission level applied to the pods in the namespace. + | | Enum: [privileged baseline restricted] +
+9 -7
View File
@@ -102,8 +102,9 @@ _Underlying type:_ _string_
ClusterMode is the possible provisioning mode of a Cluster.
_Validation:_
- Enum: [shared virtual]
Supported values: `shared`, `virtual`, `hcp`.
_Appears in:_
- [ClusterSpec](#clusterspec)
@@ -138,7 +139,7 @@ _Appears in:_
| Field | Description | Default | Validation |
| --- | --- | --- | --- |
| `version` _string_ | Version is the K3s version to use for the virtual nodes.<br />It should follow the K3s versioning convention (e.g., v1.28.2-k3s1).<br />If not specified, the Kubernetes version of the host node will be used. | | |
| `mode` _[ClusterMode](#clustermode)_ | Mode specifies the cluster provisioning mode: "shared" or "virtual".<br />Defaults to "shared". This field is immutable. | shared | Enum: [shared virtual] <br /> |
| `mode` _[ClusterMode](#clustermode)_ | Mode specifies the cluster provisioning mode: "shared", "virtual" or "hcp".<br />Defaults to "shared". This field is immutable. | shared | Enum: [shared virtual hcp] <br /> |
| `servers` _integer_ | Servers specifies the number of K3s pods to run in server (control plane) mode.<br />Must be at least 1. Defaults to 1. | 1 | |
| `agents` _integer_ | Agents specifies the number of K3s pods to run in agent (worker) mode.<br />Must be 0 or greater. Defaults to 0.<br />This field is ignored in "shared" mode. | 0 | |
| `clusterCIDR` _string_ | ClusterCIDR is the CIDR range for pod IPs.<br />Defaults to 10.42.0.0/16 in shared mode and 10.52.0.0/16 in virtual mode.<br />This field is immutable. | | |
@@ -409,8 +410,9 @@ _Underlying type:_ _string_
PodSecurityAdmissionLevel is the policy level applied to the pods in the namespace.
_Validation:_
- Enum: [privileged baseline restricted]
Supported values: `privileged`, `baseline`, `restricted`.
_Appears in:_
- [VirtualClusterPolicySpec](#virtualclusterpolicyspec)
@@ -454,7 +456,7 @@ _Appears in:_
| `optional` _boolean_ | optional field specify whether the Secret or its keys must be defined | | |
| `mountPath` _string_ | MountPath is the path within server and agent pods where the<br />secret contents will be mounted. | | |
| `subPath` _string_ | SubPath is an optional path within the secret to mount instead of the root.<br />When specified, only the specified key from the secret will be mounted as a file<br />at MountPath, keeping the parent directory writable. | | |
| `role` _string_ | Role is the type of the k3k pod that will be used to mount the secret.<br />This can be 'server', 'agent', or 'all' (for both). | | Enum: [server agent all] <br /> |
| `role` _string_ | Role is the type of the k3k pod that will be used to mount the secret.<br />This can be `server`, `agent`, or `all` (for both). | | Enum: [server agent all] <br /> |
#### SecretSyncConfig
@@ -590,7 +592,7 @@ _Appears in:_
| `defaultPriorityClass` _string_ | DefaultPriorityClass specifies the priorityClassName applied to all pods of all clusters in the target Namespace. | | |
| `defaultServerAffinity` _[Affinity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#affinity-v1-core)_ | DefaultServerAffinity specifies the affinity rules applied to server pods of all clusters in the target Namespace.<br />This includes both node affinity and pod affinity/anti-affinity rules. | | |
| `defaultAgentAffinity` _[Affinity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#affinity-v1-core)_ | DefaultAgentAffinity specifies the affinity rules applied to agent pods of all clusters in the target Namespace.<br />This includes both node affinity and pod affinity/anti-affinity rules. | | |
| `allowedMode` _[ClusterMode](#clustermode)_ | AllowedMode specifies the allowed cluster provisioning mode. Defaults to "shared". | shared | Enum: [shared virtual] <br /> |
| `allowedMode` _[ClusterMode](#clustermode)_ | AllowedMode specifies the allowed cluster provisioning mode. Defaults to "shared". | shared | Enum: [shared virtual hcp] <br /> |
| `disableNetworkPolicy` _boolean_ | DisableNetworkPolicy indicates whether to disable the creation of a default network policy for cluster isolation. | | |
| `podSecurityAdmissionLevel` _[PodSecurityAdmissionLevel](#podsecurityadmissionlevel)_ | PodSecurityAdmissionLevel specifies the pod security admission level applied to the pods in the namespace. | | Enum: [privileged baseline restricted] <br /> |
| `sync` _[SyncConfig](#syncconfig)_ | Sync specifies the resources types that will be synced from virtual cluster to host cluster. | \{ \} | |
+10
View File
@@ -0,0 +1,10 @@
apiVersion: k3k.io/v1beta1
kind: Cluster
metadata:
name: hcp-server
spec:
mode: hcp
servers: 1
version: v1.33.1-k3s1
expose:
nodePort: {}
+15 -9
View File
@@ -45,11 +45,11 @@ type ClusterSpec struct {
// +optional
Version string `json:"version,omitempty"`
// Mode specifies the cluster provisioning mode: "shared" or "virtual".
// Mode specifies the cluster provisioning mode: "shared", "virtual" or "hcp".
// Defaults to "shared". This field is immutable.
//
// +kubebuilder:default="shared"
// +kubebuilder:validation:Enum=shared;virtual
// +kubebuilder:default=shared
// +kubebuilder:validation:Enum=shared;virtual;hcp
// +kubebuilder:validation:XValidation:message="mode is immutable",rule="self == oldSelf"
// +optional
Mode ClusterMode `json:"mode,omitempty"`
@@ -253,10 +253,10 @@ type SecretMount struct {
// +optional
SubPath string `json:"subPath,omitempty"`
// Role is the type of the k3k pod that will be used to mount the secret.
// This can be 'server', 'agent', or 'all' (for both).
// This can be `server`, `agent`, or `all` (for both).
//
// +optional
// +kubebuilder:validation:Enum=server;agent;all
// +optional
Role string `json:"role,omitempty"`
}
@@ -413,8 +413,7 @@ type StorageClassSyncConfig struct {
// ClusterMode is the possible provisioning mode of a Cluster.
//
// +kubebuilder:validation:Enum=shared;virtual
// +kubebuilder:default="shared"
// Supported values: `shared`, `virtual`, `hcp`.
type ClusterMode string
const (
@@ -423,6 +422,11 @@ const (
// VirtualClusterMode represents a cluster that runs in a virtual environment.
VirtualClusterMode = ClusterMode("virtual")
// HCPClusterMode represents a Hosted Control Plane: an agentless K3s control
// plane managed by k3k inside the host cluster. End users join their own
// external nodes (BYO) using the standard K3s installer command.
HCPClusterMode = ClusterMode("hcp")
)
// PersistenceMode is the storage mode of a Cluster.
@@ -626,7 +630,7 @@ type ClusterStatus struct {
// Phase is a high-level summary of the cluster's current lifecycle state.
//
// +kubebuilder:default="Unknown"
// +kubebuilder:default=Unknown
// +kubebuilder:validation:Enum=Pending;Provisioning;Ready;Failed;Terminating;Unknown
// +optional
Phase ClusterPhase `json:"phase,omitempty"`
@@ -774,6 +778,7 @@ type VirtualClusterPolicySpec struct {
// AllowedMode specifies the allowed cluster provisioning mode. Defaults to "shared".
//
// +kubebuilder:default=shared
// +kubebuilder:validation:Enum=shared;virtual;hcp
// +kubebuilder:validation:XValidation:message="mode is immutable",rule="self == oldSelf"
// +optional
AllowedMode ClusterMode `json:"allowedMode,omitempty"`
@@ -785,6 +790,7 @@ type VirtualClusterPolicySpec struct {
// PodSecurityAdmissionLevel specifies the pod security admission level applied to the pods in the namespace.
//
// +kubebuilder:validation:Enum=privileged;baseline;restricted
// +optional
PodSecurityAdmissionLevel *PodSecurityAdmissionLevel `json:"podSecurityAdmissionLevel,omitempty"`
@@ -817,7 +823,7 @@ type VirtualClusterPolicySpec struct {
// PodSecurityAdmissionLevel is the policy level applied to the pods in the namespace.
//
// +kubebuilder:validation:Enum=privileged;baseline;restricted
// Supported values: `privileged`, `baseline`, `restricted`.
type PodSecurityAdmissionLevel string
const (
+33 -3
View File
@@ -400,9 +400,10 @@ func (c *ClusterReconciler) reconcile(ctx context.Context, cluster *v1beta1.Clus
}
}
// in virtual mode assign a default serviceCIDR
if cluster.Spec.Mode == v1beta1.VirtualClusterMode {
log.V(1).Info("assign default service CIDR for virtual mode")
// virtual and hcp modes both run a self-contained K3s control plane and
// need their own pod/service CIDR independent of the host cluster.
if cluster.Spec.Mode == v1beta1.VirtualClusterMode || cluster.Spec.Mode == v1beta1.HCPClusterMode {
log.V(1).Info("assign default service CIDR", "mode", cluster.Spec.Mode)
cluster.Status.ServiceCIDR = defaultVirtualServiceCIDR
}
@@ -447,6 +448,27 @@ func (c *ClusterReconciler) reconcile(ctx context.Context, cluster *v1beta1.Clus
return err
}
// In hcp mode, validate that the cluster has an externally-routable
// API server endpoint (NodePort / LoadBalancer / Ingress) so external
// workers can join. The join command itself is printed by the CLI
// (`k3kcli cluster create` / `k3kcli kubeconfig generate`).
// We also own the default/kubernetes Endpoints/EndpointSlice inside the
// virtual cluster (the apiserver reconciler is disabled for HCP) so
// external-node pods can reach the in-cluster apiserver ClusterIP.
if cluster.Spec.Mode == v1beta1.HCPClusterMode {
if err := c.ensureHCPRegistration(ctx, cluster); err != nil {
return err
}
if err := c.ensureHCPKubernetesEndpointSlice(ctx, cluster); err != nil {
return err
}
if err := c.ensureHCPKubernetesEndpoints(ctx, cluster); err != nil {
return err
}
}
// Important: if you need to call the Server API of the Virtual Cluster
// this needs to be done AFTER he kubeconfig has been generated
@@ -896,6 +918,14 @@ func (c *ClusterReconciler) bindClusterRoles(ctx context.Context, cluster *v1bet
}
func (c *ClusterReconciler) ensureAgent(ctx context.Context, cluster *v1beta1.Cluster, serviceIP, token string) error {
// hcp mode is BYO-node by design: external (out-of-host-cluster) nodes
// join using the standard K3s installer command, which is printed by
// `k3kcli cluster create` / `k3kcli kubeconfig generate`. k3k therefore
// does not provision any agent pods on the host cluster.
if cluster.Spec.Mode == v1beta1.HCPClusterMode {
return nil
}
config := agent.NewConfig(cluster, c.Client, c.Scheme)
var agentEnsurer agent.ResourceEnsurer
+325
View File
@@ -0,0 +1,325 @@
package cluster
import (
"context"
"errors"
"fmt"
"net"
"net/url"
"strconv"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
corev1 "k8s.io/api/core/v1"
discoveryv1 "k8s.io/api/discovery/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
ctrl "sigs.k8s.io/controller-runtime"
"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
"github.com/rancher/k3k/pkg/controller/cluster/server"
)
// ErrHCPNoExternalEndpoint is returned by ensureHCPRegistration when an
// HCP-mode cluster has no externally-routable endpoint (no NodePort,
// LoadBalancer or Ingress) so external worker nodes cannot reach the API
// server. updateStatus translates it into a Ready=False condition with
// reason HCPNoExternalEndpoint instead of failing the reconcile outright.
var ErrHCPNoExternalEndpoint = errors.New("HCP cluster has no external endpoint")
// ensureHCPRegistration verifies that an HCP-mode cluster exposes an
// externally-routable API server endpoint so external worker nodes can join.
// Join instructions (the `curl ... | sh -` line) are printed by the CLI
// (`k3kcli cluster create` / `k3kcli kubeconfig generate`); the controller
// does not persist them on the Cluster object.
//
// Returns ErrHCPNoExternalEndpoint when no NodePort, LoadBalancer or Ingress
// is configured, which updateStatus surfaces as Ready=False with reason
// HCPNoExternalEndpoint.
func (c *ClusterReconciler) ensureHCPRegistration(ctx context.Context, cluster *v1beta1.Cluster) error {
log := ctrl.LoggerFrom(ctx)
_, external, err := server.ServerURL(ctx, c.Client, cluster, selectNonLoopbackSAN(cluster), 0)
if err != nil {
return err
}
if !external {
log.Info("HCP cluster has no externally-routable endpoint",
"cluster", cluster.Name, "namespace", cluster.Namespace)
return ErrHCPNoExternalEndpoint
}
return nil
}
// selectNonLoopbackSAN returns the first non-loopback address from the
// cluster's TLS SANs, preferring spec.TLSSANs then falling back to status.TLSSANs.
// Returns empty string if no non-loopback address is found.
func selectNonLoopbackSAN(cluster *v1beta1.Cluster) string {
// Try spec.TLSSANs first (user-provided values)
for _, san := range cluster.Spec.TLSSANs {
if ip := net.ParseIP(san); ip != nil && ip.IsLoopback() {
continue
}
if san == "localhost" {
continue
}
return san // Found a non-loopback address
}
// Fall back to status.TLSSANs (computed values)
for _, san := range cluster.Status.TLSSANs {
if ip := net.ParseIP(san); ip != nil && ip.IsLoopback() {
continue
}
if san == "localhost" {
continue
}
return san
}
return ""
}
// ensureHCPKubernetesEndpointSlice maintains the default/kubernetes Service
// EndpointSlice inside the virtual cluster, pointing it at the externally
// reachable host:port (NodePort / LoadBalancer / Ingress) so that pods
// scheduled on external worker nodes can reach the in-cluster apiserver
// ClusterIP.
//
// Background: the kube-apiserver normally reconciles default/kubernetes
// EndpointSlice to its own --advertise-address:--secure-port (the host-cluster
// pod IP and 6443). External worker nodes have no route to the host-cluster
// pod CIDR, so kube-proxy DNAT to that endpoint fails. We disable the
// apiserver reconciler in HCP mode (see serverOptions) and own this
// EndpointSlice object instead.
func (c *ClusterReconciler) ensureHCPKubernetesEndpointSlice(ctx context.Context, cluster *v1beta1.Cluster) error {
log := ctrl.LoggerFrom(ctx)
rawURL, external, err := server.ServerURL(ctx, c.Client, cluster, selectNonLoopbackSAN(cluster), 0)
if err != nil {
return err
}
if !external {
// Defensive: reconcile would have already short-circuited with
// ErrHCPNoExternalEndpoint via ensureHCPRegistration before reaching
// here, but skip gracefully if invoked directly.
return nil
}
host, port, err := parseHCPHostPort(rawURL)
if err != nil {
return fmt.Errorf("parsing HCP server URL %q: %w", rawURL, err)
}
addr, err := hcpEndpointAddress(ctx, host)
if err != nil {
return err
}
var addressType discoveryv1.AddressType
if ip := net.ParseIP(addr.IP); ip != nil {
if ip.To4() != nil {
addressType = discoveryv1.AddressTypeIPv4
} else {
addressType = discoveryv1.AddressTypeIPv6
}
} else {
return fmt.Errorf("invalid IP address %q", addr.IP)
}
virtClient, err := newVirtualClient(ctx, c.Client, cluster.Name, cluster.Namespace)
if err != nil {
return fmt.Errorf("creating virtual cluster client: %w", err)
}
endpointSlice := &discoveryv1.EndpointSlice{
ObjectMeta: metav1.ObjectMeta{
Name: "kubernetes",
Namespace: metav1.NamespaceDefault,
},
}
_, err = controllerutil.CreateOrUpdate(ctx, virtClient, endpointSlice, func() error {
if endpointSlice.Labels == nil {
endpointSlice.Labels = make(map[string]string)
}
// Ensure the service-name label is set
endpointSlice.Labels[discoveryv1.LabelServiceName] = "kubernetes"
endpointSlice.AddressType = addressType
endpointSlice.Endpoints = []discoveryv1.Endpoint{
{Addresses: []string{addr.IP}},
}
portName := "https"
endpointSlice.Ports = []discoveryv1.EndpointPort{
{
Name: &portName,
Port: &port,
Protocol: new(corev1.ProtocolTCP),
},
}
return nil
})
if err != nil {
return fmt.Errorf("upserting default/kubernetes endpointslice in virtual cluster: %w", err)
}
log.V(1).Info("HCP kubernetes endpointslice reconciled",
"address", addr.IP, "host", host, "port", port)
return nil
}
func (c *ClusterReconciler) ensureHCPKubernetesEndpoints(ctx context.Context, cluster *v1beta1.Cluster) error {
log := ctrl.LoggerFrom(ctx)
rawURL, external, err := server.ServerURL(ctx, c.Client, cluster, selectNonLoopbackSAN(cluster), 0)
if err != nil {
return err
}
if !external {
// Defensive: reconcile would have already short-circuited with
// ErrHCPNoExternalEndpoint via ensureHCPRegistration before reaching
// here, but skip gracefully if invoked directly.
return nil
}
host, port, err := parseHCPHostPort(rawURL)
if err != nil {
return fmt.Errorf("parsing HCP server URL %q: %w", rawURL, err)
}
addr, err := hcpEndpointAddress(ctx, host)
if err != nil {
return err
}
virtClient, err := newVirtualClient(ctx, c.Client, cluster.Name, cluster.Namespace)
if err != nil {
return fmt.Errorf("creating virtual cluster client: %w", err)
}
//nolint:staticcheck // SA1019 corev1.Endpoints is deprecated in v1.33+, but needed in the Conformance tests
// We are already using the discoveryv1.EndpointSlice
endpoints := &corev1.Endpoints{
ObjectMeta: metav1.ObjectMeta{
Name: "kubernetes",
Namespace: metav1.NamespaceDefault,
},
}
_, err = controllerutil.CreateOrUpdate(ctx, virtClient, endpoints, func() error {
if endpoints.Labels == nil {
endpoints.Labels = make(map[string]string)
}
// Ensure the skip-mirror label is set
endpoints.Labels[discoveryv1.LabelSkipMirror] = "true"
//nolint:staticcheck // SA1019 corev1.EndpointSubset is deprecated in v1.33+, but needed in the Conformance tests
endpoints.Subsets = []corev1.EndpointSubset{
{
Addresses: []corev1.EndpointAddress{addr},
Ports: []corev1.EndpointPort{
{
Name: "https",
Port: port,
Protocol: corev1.ProtocolTCP,
},
},
},
}
return nil
})
if err != nil {
return fmt.Errorf("upserting default/kubernetes endpoints in virtual cluster: %w", err)
}
log.V(1).Info("HCP kubernetes endpoints reconciled", "address", addr.IP, "host", host, "port", port)
return nil
}
// parseHCPHostPort extracts the host and port from a server URL produced by
// server.ServerURL. The port defaults to 443 when omitted.
func parseHCPHostPort(rawURL string) (string, int32, error) {
u, err := url.Parse(rawURL)
if err != nil {
return "", 0, err
}
host := u.Hostname()
if host == "" {
return "", 0, fmt.Errorf("missing host in URL %q", rawURL)
}
portStr := u.Port()
var port int32 = 443
if portStr != "" {
p, err := strconv.Atoi(portStr)
if err != nil {
return "", 0, fmt.Errorf("invalid port in URL %q: %w", rawURL, err)
}
if p <= 0 || p > 65535 {
return "", 0, fmt.Errorf("port %d out of range in URL %q", p, rawURL)
}
port = int32(p)
}
return host, port, nil
}
// hcpEndpointAddress builds a corev1.EndpointAddress from the externally
// reachable host. Endpoints require an IP; if the host is a DNS name we
// resolve it. The Hostname field is intentionally left unset:
// the kubernetes API validates it as a DNS-1123 label (no dots),
// so an FQDN like "host.example.com" would be rejected.
func hcpEndpointAddress(ctx context.Context, host string) (corev1.EndpointAddress, error) {
if ip := net.ParseIP(host); ip != nil {
if ip.IsLoopback() {
return corev1.EndpointAddress{}, fmt.Errorf("HCP endpoint host %q is a loopback address and cannot be used", host)
}
return corev1.EndpointAddress{IP: host}, nil
}
ipAddrs, err := net.DefaultResolver.LookupIPAddr(ctx, host)
if err != nil {
return corev1.EndpointAddress{}, fmt.Errorf("HCP endpoint host %q is not an IP and does not resolve: %w", host, err)
}
var filteredIPs []net.IP
for _, addr := range ipAddrs {
if !addr.IP.IsLoopback() {
filteredIPs = append(filteredIPs, addr.IP)
}
}
if len(filteredIPs) == 0 {
return corev1.EndpointAddress{}, fmt.Errorf("HCP endpoint host %q resolved to no non-loopback IPs", host)
}
if v4 := filteredIPs[0].To4(); v4 != nil {
return corev1.EndpointAddress{IP: v4.String()}, nil
}
return corev1.EndpointAddress{IP: filteredIPs[0].String()}, nil
}
+279
View File
@@ -0,0 +1,279 @@
package cluster
import (
"context"
"errors"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"k8s.io/apimachinery/pkg/runtime"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
"github.com/rancher/k3k/pkg/controller"
"github.com/rancher/k3k/pkg/controller/cluster/server"
)
func Test_ensureHCPRegistration(t *testing.T) {
scheme := runtime.NewScheme()
require.NoError(t, corev1.AddToScheme(scheme))
require.NoError(t, v1beta1.AddToScheme(scheme))
cluster := &v1beta1.Cluster{
ObjectMeta: metav1.ObjectMeta{Name: "demo", Namespace: "team-a"},
Spec: v1beta1.ClusterSpec{
Mode: v1beta1.HCPClusterMode,
Version: "v1.33.1-k3s1",
TLSSANs: []string{"hcp.example.com"},
},
Status: v1beta1.ClusterStatus{
TLSSANs: []string{"hcp.example.com"},
},
}
t.Run("clusterip-only service returns ErrHCPNoExternalEndpoint", func(t *testing.T) {
svc := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: server.ServiceName(cluster.Name),
Namespace: cluster.Namespace,
},
Spec: corev1.ServiceSpec{
Type: corev1.ServiceTypeClusterIP,
ClusterIP: "10.43.0.50",
Ports: []corev1.ServicePort{
{Name: "k3s-server-port", Port: 443},
},
},
}
fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(svc).Build()
r := &ClusterReconciler{Client: fakeClient}
c := cluster.DeepCopy()
err := r.ensureHCPRegistration(context.Background(), c)
require.Error(t, err)
assert.True(t, errors.Is(err, ErrHCPNoExternalEndpoint))
})
t.Run("nodeport service returns no error", func(t *testing.T) {
svc := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: server.ServiceName(cluster.Name),
Namespace: cluster.Namespace,
},
Spec: corev1.ServiceSpec{
Type: corev1.ServiceTypeNodePort,
ClusterIP: "10.43.0.50",
Ports: []corev1.ServicePort{
{Name: "k3s-server-port", Port: 443, NodePort: 30443},
},
},
}
fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(svc).Build()
r := &ClusterReconciler{Client: fakeClient}
c := cluster.DeepCopy()
assert.NoError(t, r.ensureHCPRegistration(context.Background(), c))
})
}
func Test_parseHCPHostPort(t *testing.T) {
tests := []struct {
name string
url string
wantHost string
wantPort int32
wantErr bool
}{
{
name: "ip with explicit port",
url: "https://10.144.101.195:30337",
wantHost: "10.144.101.195",
wantPort: 30337,
},
{
name: "hostname without port defaults to 443",
url: "https://hcp.example.com",
wantHost: "hcp.example.com",
wantPort: 443,
},
{
name: "hostname with explicit port",
url: "https://hcp.example.com:6443",
wantHost: "hcp.example.com",
wantPort: 6443,
},
{
name: "missing host",
url: "https://",
wantErr: true,
},
{
name: "non-numeric port",
url: "https://host:abc",
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
host, port, err := parseHCPHostPort(tt.url)
if tt.wantErr {
require.Error(t, err)
return
}
require.NoError(t, err)
assert.Equal(t, tt.wantHost, host)
assert.Equal(t, tt.wantPort, port)
})
}
}
func Test_selectNonLoopbackSAN(t *testing.T) {
tests := []struct {
name string
specSANs []string
statusSANs []string
want string
}{
{
name: "spec with loopback first, external second",
specSANs: []string{"127.0.0.1", "10.0.0.100"},
statusSANs: []string{},
want: "10.0.0.100",
},
{
name: "spec with external first",
specSANs: []string{"10.0.0.100", "127.0.0.1"},
statusSANs: []string{},
want: "10.0.0.100",
},
{
name: "spec empty, status with loopback first, external second",
specSANs: []string{},
statusSANs: []string{"127.0.0.1", "10.43.0.50"},
want: "10.43.0.50",
},
{
name: "spec with only loopback",
specSANs: []string{"127.0.0.1", "::1"},
statusSANs: []string{},
want: "",
},
{
name: "spec with localhost hostname",
specSANs: []string{"localhost", "example.com"},
statusSANs: []string{},
want: "example.com",
},
{
name: "spec with external hostname",
specSANs: []string{"hcp.example.com"},
statusSANs: []string{},
want: "hcp.example.com",
},
{
name: "empty spec and status",
specSANs: []string{},
statusSANs: []string{},
want: "",
},
{
name: "ipv6 loopback filtered",
specSANs: []string{"::1", "2001:db8::1"},
statusSANs: []string{},
want: "2001:db8::1",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
cluster := &v1beta1.Cluster{
Spec: v1beta1.ClusterSpec{
TLSSANs: tt.specSANs,
},
Status: v1beta1.ClusterStatus{
TLSSANs: tt.statusSANs,
},
}
got := selectNonLoopbackSAN(cluster)
assert.Equal(t, tt.want, got)
})
}
}
func Test_hcpEndpointAddress(t *testing.T) {
tests := []struct {
name string
input string
wantIP string
wantHostname string
wantErr bool
}{
{
name: "ipv4 literal is passed through",
input: "10.144.101.195",
wantIP: "10.144.101.195",
wantHostname: "",
wantErr: false,
},
{
name: "unresolvable hostname errors",
input: "definitely-not-a-real-host.invalid",
wantErr: true,
},
{
name: "ipv4 loopback literal is rejected",
input: "127.0.0.1",
wantErr: true,
},
{
name: "ipv6 loopback literal is rejected",
input: "::1",
wantErr: true,
},
{
name: "ipv4 loopback in range is rejected",
input: "127.0.0.100",
wantErr: true,
},
{
name: "valid ipv6 literal passes through",
input: "2001:db8::1",
wantIP: "2001:db8::1",
wantHostname: "",
wantErr: false,
},
{
name: "localhost hostname filters loopbacks",
input: "localhost",
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := hcpEndpointAddress(context.Background(), tt.input)
if tt.wantErr {
require.Error(t, err)
return
}
require.NoError(t, err)
assert.Equal(t, tt.wantIP, got.IP)
assert.Equal(t, tt.wantHostname, got.Hostname)
})
}
}
// Compile-time assertion: every reused exported name from the controller
// package below this test file must remain stable. If `controller.K3SImage`
// disappears (refactor), this guards the dependency.
var _ = controller.K3SImage
+21 -2
View File
@@ -11,7 +11,6 @@ import (
"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
"github.com/rancher/k3k/pkg/controller"
"github.com/rancher/k3k/pkg/controller/cluster/agent"
)
// serverConfig are few options from k3s server options that will
@@ -23,6 +22,7 @@ type serverConfig struct {
DisableAgent bool `yaml:"disable-agent,omitempty"`
Disable []string `yaml:"disable,omitempty"`
EgressSelectorMode string `yaml:"egress-selector-mode,omitempty"`
KubeApiServerArg []string `yaml:"kube-apiserver-arg,omitempty"`
Server string `yaml:"server,omitempty"`
ServiceCIDR string `yaml:"service-cidr,omitempty"`
TLSSAN []string `yaml:"tls-san,omitempty"`
@@ -77,10 +77,29 @@ func buildServerConfig(cluster *v1beta1.Cluster, initServer bool, serviceIP, tok
serverConfig.Server = "https://" + serviceIP
}
if cluster.Spec.Mode != agent.VirtualNodeMode {
// shared and hcp modes both run K3s with --disable-agent (agentless server).
switch cluster.Spec.Mode {
case "", v1beta1.SharedClusterMode:
serverConfig.DisableAgent = true
serverConfig.EgressSelectorMode = "disabled"
serverConfig.Disable = []string{"servicelb", "traefik", "metrics-server", "local-storage"}
case v1beta1.HCPClusterMode:
serverConfig.DisableAgent = true
// Tunnel apiserver egress through the k3s-agent WebSocket: the
// apiserver has no route to the virtual cluster's pod CIDR and
// bypasses kube-proxy when dialing pod IPs (webhooks, log/exec).
// "cluster" is the only safe mode — "agent" lets pod dials go
// direct (no route, fails); "pod" only permits pod IPs the agent
// has already watched, so a newly-created pod's IP is rejected
// and tears down the remotedialer session, making kubelet streams
// flaky. See k3s pkg/agent/tunnel/tunnel.go.
serverConfig.EgressSelectorMode = "cluster"
// Disable the apiserver's built-in endpoint reconciler so K3k can
// own default/kubernetes Endpoints and point it at the externally
// reachable host:port (NodePort / LB / Ingress).
serverConfig.KubeApiServerArg = append(serverConfig.KubeApiServerArg, "endpoint-reconciler-type=none")
case v1beta1.VirtualClusterMode:
// no extra config for virtual mode
}
return serverConfig
+10 -4
View File
@@ -2,9 +2,10 @@ package server
import (
"fmt"
"reflect"
"testing"
"github.com/stretchr/testify/assert"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
@@ -43,6 +44,9 @@ func Test_BuildServerConfig(t *testing.T) {
Name: testClusterName,
Namespace: testClusterNamespace,
},
Spec: v1beta1.ClusterSpec{
Mode: v1beta1.SharedClusterMode,
},
Status: v1beta1.ClusterStatus{
ClusterCIDR: defaultSharedClusterCIDR,
ServiceCIDR: defaultSharedServiceCIDR,
@@ -71,6 +75,9 @@ func Test_BuildServerConfig(t *testing.T) {
Name: testClusterName,
Namespace: testClusterNamespace,
},
Spec: v1beta1.ClusterSpec{
Mode: v1beta1.SharedClusterMode,
},
Status: v1beta1.ClusterStatus{
ClusterCIDR: defaultSharedClusterCIDR,
ServiceCIDR: defaultSharedServiceCIDR,
@@ -158,6 +165,7 @@ func Test_BuildServerConfig(t *testing.T) {
Namespace: testClusterNamespace,
},
Spec: v1beta1.ClusterSpec{
Mode: v1beta1.SharedClusterMode,
ClusterDNS: testClusterDNS,
},
Status: v1beta1.ClusterStatus{
@@ -186,9 +194,7 @@ func Test_BuildServerConfig(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
serverConfig := buildServerConfig(tt.args.cluster, tt.args.initServer, tt.args.serviceIP, tt.args.token)
if !reflect.DeepEqual(tt.expectedData, serverConfig) {
t.Errorf("found %v, expected %v", serverConfig, tt.expectedData)
}
assert.Equal(t, tt.expectedData, serverConfig)
})
}
}
+124
View File
@@ -0,0 +1,124 @@
package server
import (
"context"
"fmt"
"net"
"slices"
"strconv"
"github.com/sirupsen/logrus"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/client"
corev1 "k8s.io/api/core/v1"
networkingv1 "k8s.io/api/networking/v1"
"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
)
// ServerURL returns the URL at which the K3s API server of a virtual cluster
// is reachable. The second return value reports whether that URL is routable
// from outside the host cluster (true for NodePort/LoadBalancer/Ingress, false
// for plain ClusterIP exposition).
//
// hostServerIP is used as the address when the underlying Service is a
// NodePort. serverPort, when non-zero, overrides the port discovered from the
// Service.
func ServerURL(ctx context.Context, c client.Client, cluster *v1beta1.Cluster, hostServerIP string, serverPort int) (string, bool, error) {
key := types.NamespacedName{
Name: ServiceName(cluster.Name),
Namespace: cluster.Namespace,
}
var k3kService corev1.Service
if err := c.Get(ctx, key, &k3kService); err != nil {
return "", false, err
}
ip := k3kService.Spec.ClusterIP
port := int32(httpsPort)
external := false
if len(k3kService.Spec.Ports) == 0 {
logrus.Warn("No ports exposed by the cluster service.")
}
switch k3kService.Spec.Type {
case corev1.ServiceTypeNodePort:
ip = hostServerIP
external = true
if len(k3kService.Spec.Ports) > 0 {
port = k3kService.Spec.Ports[0].NodePort
}
case corev1.ServiceTypeLoadBalancer:
if len(k3kService.Status.LoadBalancer.Ingress) > 0 {
ingress := k3kService.Status.LoadBalancer.Ingress[0]
switch {
case ingress.IP != "":
ip = ingress.IP
external = true
case ingress.Hostname != "":
ip = ingress.Hostname
external = true
}
}
if !external {
logrus.Warn("No usable ingress address found in LoadBalancer service.")
}
if len(k3kService.Spec.Ports) > 0 {
port = k3kService.Spec.Ports[0].Port
}
}
if serverPort != 0 {
port = int32(serverPort)
}
if !slices.Contains(cluster.Status.TLSSANs, ip) {
logrus.Warnf("IP %s not in tlsSANs.", ip)
if len(cluster.Spec.TLSSANs) > 0 {
logrus.Warnf("Using the first TLS SAN in the spec as a fallback: %s", cluster.Spec.TLSSANs[0])
ip = cluster.Spec.TLSSANs[0]
} else if len(cluster.Status.TLSSANs) > 0 {
logrus.Warnf("No explicit tlsSANs specified. Trying to use the first TLS SAN in the status: %s", cluster.Status.TLSSANs[0])
ip = cluster.Status.TLSSANs[0]
} else {
logrus.Warn("IP not found in tlsSANs. This could cause issue with the certificate validation.")
}
}
host := ip
if port != httpsPort {
host = net.JoinHostPort(ip, strconv.Itoa(int(port)))
}
url := "https://" + host
// if ingress is specified, use the ingress host
if cluster.Spec.Expose != nil && cluster.Spec.Expose.Ingress != nil {
var k3kIngress networkingv1.Ingress
ingressKey := types.NamespacedName{
Name: IngressName(cluster.Name),
Namespace: cluster.Namespace,
}
if err := c.Get(ctx, ingressKey, &k3kIngress); err != nil {
return "", external, err
}
if len(k3kIngress.Spec.Rules) > 0 {
url = fmt.Sprintf("https://%s", k3kIngress.Spec.Rules[0].Host)
external = true
}
}
return url, external, nil
}
@@ -0,0 +1,80 @@
package server
import (
"context"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"k8s.io/apimachinery/pkg/runtime"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
)
func Test_ServerURL_LoadBalancer(t *testing.T) {
scheme := runtime.NewScheme()
require.NoError(t, corev1.AddToScheme(scheme))
require.NoError(t, v1beta1.AddToScheme(scheme))
cluster := &v1beta1.Cluster{
ObjectMeta: metav1.ObjectMeta{Name: "demo", Namespace: "team-a"},
Status: v1beta1.ClusterStatus{TLSSANs: []string{"203.0.113.10", "lb.example.com"}},
}
tests := []struct {
name string
ingress []corev1.LoadBalancerIngress
wantURL string
wantExternal bool
}{
{
name: "no ingress yet (LB still provisioning)",
ingress: nil,
wantExternal: false,
},
{
name: "ingress with IP",
ingress: []corev1.LoadBalancerIngress{{IP: "203.0.113.10"}},
wantURL: "https://203.0.113.10",
wantExternal: true,
},
{
name: "ingress with hostname only",
ingress: []corev1.LoadBalancerIngress{{Hostname: "lb.example.com"}},
wantURL: "https://lb.example.com",
wantExternal: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
svc := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: ServiceName(cluster.Name),
Namespace: cluster.Namespace,
},
Spec: corev1.ServiceSpec{
Type: corev1.ServiceTypeLoadBalancer,
ClusterIP: "10.43.0.50",
Ports: []corev1.ServicePort{{Name: "k3s-server-port", Port: 443}},
},
Status: corev1.ServiceStatus{
LoadBalancer: corev1.LoadBalancerStatus{Ingress: tt.ingress},
},
}
c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(svc).Build()
url, external, err := ServerURL(context.Background(), c, cluster, "", 0)
require.NoError(t, err)
assert.Equal(t, tt.wantExternal, external)
if tt.wantExternal {
assert.Equal(t, tt.wantURL, url)
}
})
}
}
+4 -3
View File
@@ -22,7 +22,6 @@ import (
"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
"github.com/rancher/k3k/pkg/controller"
"github.com/rancher/k3k/pkg/controller/cluster/agent"
"github.com/rancher/k3k/pkg/controller/cluster/mounts"
)
@@ -250,8 +249,10 @@ func (s *Server) podSpec(ctx context.Context, image, name string, persistent boo
},
},
}
// start the pod unprivileged in shared mode
if s.mode == agent.VirtualNodeMode {
// virtual mode runs an embedded kubelet inside the server pod and therefore
// requires Privileged. shared and hcp modes are agentless (no kubelet) and
// run unprivileged.
if s.mode == string(v1beta1.VirtualClusterMode) {
podSpec.Containers[0].SecurityContext = &corev1.SecurityContext{
Privileged: ptr.To(true),
}
+3 -2
View File
@@ -19,7 +19,7 @@ safe_mode() {
CURRENT_IP=$(cat /var/lib/rancher/k3s/k3k-node-ip)
fi
if [ -z "$CURRENT_IP" ] || [ "$CURRENT_IP" = "$POD_IP" ] || [ {{.K3K_MODE}} != "virtual" ]; then
if [ -z "$CURRENT_IP" ] || [ "$CURRENT_IP" = "$POD_IP" ] || [ "{{.K3K_MODE}}" != "virtual" ]; then
return
fi
@@ -116,7 +116,8 @@ configure_cgroups() {
fi
# only configure the cgroups if the runtime used is the default and the mode is virtual
if [ -n "$runtime_class" ] || [ "{{.K3K_MODE}}" != "virtual" ]; then
# shared and hcp run agentless (no kubelet) and don't need cgroup overrides.
if [ -n "$runtime_class" ] || [ "{{.K3K_MODE}}" != "virtual" ]; then
return
fi
+18 -5
View File
@@ -19,11 +19,12 @@ const (
ConditionReady = "Ready"
// Condition Reasons
ReasonValidationFailed = "ValidationFailed"
ReasonProvisioning = "Provisioning"
ReasonProvisioned = "Provisioned"
ReasonProvisioningFailed = "ProvisioningFailed"
ReasonTerminating = "Terminating"
ReasonValidationFailed = "ValidationFailed"
ReasonProvisioning = "Provisioning"
ReasonProvisioned = "Provisioned"
ReasonProvisioningFailed = "ProvisioningFailed"
ReasonTerminating = "Terminating"
ReasonHCPNoExternalEndpoint = "HCPNoExternalEndpoint"
)
func (c *ClusterReconciler) updateStatus(ctx context.Context, cluster *v1beta1.Cluster, reconcileErr error) {
@@ -69,6 +70,18 @@ func (c *ClusterReconciler) updateStatus(ctx context.Context, cluster *v1beta1.C
return
}
if errors.Is(reconcileErr, ErrHCPNoExternalEndpoint) {
cluster.Status.Phase = v1beta1.ClusterPending
meta.SetStatusCondition(&cluster.Status.Conditions, metav1.Condition{
Type: ConditionReady,
Status: metav1.ConditionFalse,
Reason: ReasonHCPNoExternalEndpoint,
Message: "HCP cluster has no external endpoint; set spec.expose.nodePort, spec.expose.loadBalancer or spec.expose.ingress so external nodes can reach the API server",
})
return
}
// If there's an error, but it's not a validation error, the cluster is in a failed state.
if reconcileErr != nil {
cluster.Status.Phase = v1beta1.ClusterFailed
+1 -89
View File
@@ -3,18 +3,12 @@ package kubeconfig
import (
"context"
"crypto/x509"
"fmt"
"slices"
"time"
"github.com/sirupsen/logrus"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apiserver/pkg/authentication/user"
"sigs.k8s.io/controller-runtime/pkg/client"
certutil "github.com/rancher/dynamiclistener/cert"
corev1 "k8s.io/api/core/v1"
networkingv1 "k8s.io/api/networking/v1"
clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
@@ -60,7 +54,7 @@ func (k *KubeConfig) Generate(ctx context.Context, client client.Client, cluster
return nil, err
}
url, err := getURLFromService(ctx, client, cluster, hostServerIP, port)
url, _, err := server.ServerURL(ctx, client, cluster, hostServerIP, port)
if err != nil {
return nil, err
}
@@ -92,85 +86,3 @@ func NewConfig(url string, serverCA, clientCert, clientKey []byte) *clientcmdapi
return config
}
func getURLFromService(ctx context.Context, client client.Client, cluster *v1beta1.Cluster, hostServerIP string, serverPort int) (string, error) {
// get the server service to extract the right IP
key := types.NamespacedName{
Name: server.ServiceName(cluster.Name),
Namespace: cluster.Namespace,
}
var k3kService corev1.Service
if err := client.Get(ctx, key, &k3kService); err != nil {
return "", err
}
ip := k3kService.Spec.ClusterIP
port := int32(443)
if len(k3kService.Spec.Ports) == 0 {
logrus.Warn("No ports exposed by the cluster service.")
}
switch k3kService.Spec.Type {
case corev1.ServiceTypeNodePort:
ip = hostServerIP
if len(k3kService.Spec.Ports) > 0 {
port = k3kService.Spec.Ports[0].NodePort
}
case corev1.ServiceTypeLoadBalancer:
if len(k3kService.Status.LoadBalancer.Ingress) > 0 {
ip = k3kService.Status.LoadBalancer.Ingress[0].IP
} else {
logrus.Warn("No ingress found in LoadBalancer service.")
}
if len(k3kService.Spec.Ports) > 0 {
port = k3kService.Spec.Ports[0].Port
}
}
if serverPort != 0 {
port = int32(serverPort)
}
if !slices.Contains(cluster.Status.TLSSANs, ip) {
logrus.Warnf("IP %s not in tlsSANs.", ip)
if len(cluster.Spec.TLSSANs) > 0 {
logrus.Warnf("Using the first TLS SAN in the spec as a fallback: %s", cluster.Spec.TLSSANs[0])
ip = cluster.Spec.TLSSANs[0]
} else if len(cluster.Status.TLSSANs) > 0 {
logrus.Warnf("No explicit tlsSANs specified. Trying to use the first TLS SAN in the status: %s", cluster.Status.TLSSANs[0])
ip = cluster.Status.TLSSANs[0]
} else {
logrus.Warn("IP not found in tlsSANs. This could cause issue with the certificate validation.")
}
}
url := "https://" + ip
if port != 443 {
url = fmt.Sprintf("%s:%d", url, port)
}
// if ingress is specified, use the ingress host
if cluster.Spec.Expose != nil && cluster.Spec.Expose.Ingress != nil {
var k3kIngress networkingv1.Ingress
ingressKey := types.NamespacedName{
Name: server.IngressName(cluster.Name),
Namespace: cluster.Namespace,
}
if err := client.Get(ctx, ingressKey, &k3kIngress); err != nil {
return "", err
}
url = fmt.Sprintf("https://%s", k3kIngress.Spec.Rules[0].Host)
}
return url, nil
}
+93
View File
@@ -5,8 +5,11 @@ import (
"sigs.k8s.io/controller-runtime/pkg/client"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
k3kcluster "github.com/rancher/k3k/pkg/controller/cluster"
fwk3k "github.com/rancher/k3k/tests/framework/k3k"
. "github.com/onsi/ginkgo/v2"
@@ -61,3 +64,93 @@ var _ = When("creating a shared mode cluster", Label(e2eTestLabel), Label(slowTe
Should(Succeed())
})
})
var _ = When("creating an HCP mode cluster", Label(e2eTestLabel), Label(slowTestsLabel), func() {
var virtualCluster *VirtualCluster
BeforeEach(func() {
namespace := fwk3k.CreateNamespace(k8s)
DeferCleanup(func() {
fwk3k.DeleteNamespaces(k8s, namespace.Name)
})
cluster := NewCluster(namespace.Name)
cluster.Spec.Mode = v1beta1.HCPClusterMode
CreateCluster(cluster)
client, restConfig := NewVirtualK8sClientAndConfig(cluster)
virtualCluster = &VirtualCluster{
Cluster: cluster,
RestConfig: restConfig,
Client: client,
}
})
It("is up and running", func() {
Eventually(func(g Gomega) {
ctx := GinkgoT().Context()
key := client.ObjectKeyFromObject(virtualCluster.Cluster)
g.Expect(k8sClient.Get(ctx, key, virtualCluster.Cluster)).To(Succeed())
g.Expect(virtualCluster.Cluster.Status.Phase).To(BeEquivalentTo(v1beta1.ClusterReady))
}).
WithTimeout(time.Minute).
WithPolling(time.Second).
Should(Succeed())
})
It("creates a populated token secret", func() {
ctx := GinkgoT().Context()
Eventually(func(g Gomega) {
var tokenSecret corev1.Secret
key := client.ObjectKey{
Name: k3kcluster.TokenSecretName(virtualCluster.Cluster.Name),
Namespace: virtualCluster.Cluster.Namespace,
}
g.Expect(k8sClient.Get(ctx, key, &tokenSecret)).To(Succeed())
g.Expect(tokenSecret.Data["token"]).NotTo(BeEmpty())
}).
WithTimeout(time.Minute).
WithPolling(time.Second).
Should(Succeed())
})
It("reconciles default/kubernetes Endpoints and EndpointSlice", func() {
ctx := GinkgoT().Context()
Eventually(func(g Gomega) {
key := client.ObjectKeyFromObject(virtualCluster.Cluster)
g.Expect(k8sClient.Get(ctx, key, virtualCluster.Cluster)).To(Succeed())
endpoints, err := virtualCluster.Client.CoreV1().Endpoints("default").Get(ctx, "kubernetes", metav1.GetOptions{})
g.Expect(err).To(Not(HaveOccurred()))
g.Expect(endpoints.Subsets).To(HaveLen(1))
g.Expect(endpoints.Subsets[0].Addresses).To(HaveLen(1))
g.Expect(virtualCluster.Cluster.Status.TLSSANs).To(ContainElement(endpoints.Subsets[0].Addresses[0].IP))
}).
WithTimeout(time.Minute).
WithPolling(time.Second).
Should(Succeed())
Eventually(func(g Gomega) {
key := client.ObjectKeyFromObject(virtualCluster.Cluster)
g.Expect(k8sClient.Get(ctx, key, virtualCluster.Cluster)).To(Succeed())
slices, err := virtualCluster.Client.DiscoveryV1().EndpointSlices("default").List(ctx, metav1.ListOptions{
LabelSelector: "kubernetes.io/service-name=kubernetes",
})
g.Expect(err).To(Not(HaveOccurred()))
g.Expect(slices.Items).To(HaveLen(1))
g.Expect(slices.Items[0].Endpoints).To(HaveLen(1))
g.Expect(slices.Items[0].Endpoints[0].Addresses).To(HaveLen(1))
g.Expect(virtualCluster.Cluster.Status.TLSSANs).To(ContainElement(slices.Items[0].Endpoints[0].Addresses[0]))
}).
WithTimeout(time.Minute).
WithPolling(time.Second).
Should(Succeed())
})
})
+2 -1
View File
@@ -288,7 +288,8 @@ func (c *VirtualCluster) NewNginxPod(namespace string) (*corev1.Pod, string) {
By(fmt.Sprintf("Nginx Pod is running (%s/%s)", nginxPod.Namespace, nginxPod.Name))
// only check the pod on the host cluster if the mode is shared mode
// only check the pod on the host cluster if the mode is shared mode.
// hcp is agentless and BYO-node, so no host-side pod mirror exists.
if c.Cluster.Spec.Mode != v1beta1.SharedClusterMode {
return nginxPod, ""
}