github/k3k
1
0
Fork 0
mirror of https://github.com/rancher/k3k.git synced 2026-08-18 03:46:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

Move e2e tests to a multi-node cluster, add kubelet restart test (#1010)

Browse Source 
* Add E2E tests for multi-node setup and kubelet restart scenarios

* Update Kubernetes version options in conformance tests workflow

* Enhance VM provisioning action to support dynamic bridge CIDR prefix and pin package versions
This commit is contained in:
Enrico Candino
2026-07-14 11:21:17 +02:00
committed by GitHub
parent be742939c6
commit 07cfe77318
4 changed files with 503 additions and 168 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/actions/provision-k3s-vm-workers/action.yml
+201
View File
@@ -0,0 +1,201 @@
name: 'Provision k3s VM workers'
description: >
Launches real QEMU/KVM worker VMs on a bridged network and joins them as k3s agents to a given
k3s server. Used to turn a single-node CI host into a genuine multi-node cluster (unlike
container-based nodes, e.g. k3d, these are independent kernels/machines).
inputs:
worker-count:
description: 'Number of worker VMs to launch'
required: false
default: '2'
bridge-cidr-prefix:
description: 'First three octets of the bridge subnet the VMs are attached to'
required: false
default: '192.168.100'
k3s-url:
description: 'K3s server URL the workers should join (e.g. https://192.168.100.1:6443)'
required: true
k3s-token:
description: 'K3s token to join the server with'
required: true
k3s-version:
description: 'K3s version to install on the workers (e.g. v1.34.6+k3s1)'
required: true
runs:
using: 'composite'
steps:
- name: Install Virtualization Dependencies
shell: bash
run: |
echo "CIDR_PREFIX=${{ inputs.bridge-cidr-prefix }}" >> "$GITHUB_ENV"
sudo apt-get update
# Pinned to the versions available on the runner's Ubuntu 24.04 (noble) image at the
# time of writing. These come from Ubuntu's regular archive, which does not retain
# superseded versions — a routine security update to any of these will make the pin
# unresolvable and fail the install below. If that happens, check the versions
# currently available (logged on every run) and update the pins to match.
echo "Available versions for pinned packages:"
apt-cache madison qemu-kvm qemu-utils cloud-image-utils
sudo apt-get install -y \
qemu-kvm=1:8.2.2+ds-0ubuntu1.17 \
qemu-utils=1:8.2.2+ds-0ubuntu1.17 \
cloud-image-utils=0.33-1
sudo usermod -aG kvm $USER
kvm-ok
- name: Set up bridge network for VMs
shell: bash
run: |
# Create a bridge so both VMs share an L2 segment with unique routable IPs.
# Required because QEMU `-net user` gives every VM the same 10.0.2.15 NAT
# address, which breaks flannel VXLAN between workers.
sudo ip link add name k3kbr0 type bridge
sudo ip addr add ${CIDR_PREFIX}.1/24 dev k3kbr0
sudo ip link set k3kbr0 up
# NAT outbound so VMs can reach the internet (image pulls etc).
sudo sysctl -w net.ipv4.ip_forward=1
sudo iptables -t nat -A POSTROUTING -s ${CIDR_PREFIX}.0/24 ! -o k3kbr0 -j MASQUERADE
sudo iptables -A FORWARD -i k3kbr0 -j ACCEPT
sudo iptables -A FORWARD -o k3kbr0 -j ACCEPT
# One TAP per VM, attached to the bridge.
for i in $(seq 1 ${{ inputs.worker-count }}); do
sudo ip tuntap add tap-w${i} mode tap
sudo ip link set tap-w${i} master k3kbr0
sudo ip link set tap-w${i} up
done
- name: Download Base Cloud Image
shell: bash
run: |
# PINNED TO UBUNTU 24.04 LTS (noble).
#
# Newer Ubuntu releases (tested: 26.04 "resolute") ship a stricter
# `cri-containerd.apparmor.d` profile that denies inter-thread signal
# delivery. The BIND ISC library used by `nslookup` relies on those
# signals during shutdown (`isc_app_ctxshutdown()` calls `kill()`),
# so when AppArmor denies them nslookup exits 139 with
# "kill: Permission denied". The conformance tests
# `[sig-network] Services should be able to change the type from
# {NodePort,ClusterIP} to ExternalName`
# both run `nslookup` from an exec pod and fail in that case.
#
# Before bumping past 24.04, verify those two conformance tests still
# pass — or that the containerd AppArmor profile on the newer release
# has been relaxed to allow intra-pod signals.
wget -q https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img -O ubuntu-cloudimg.img
- name: Generate SSH Key and Cloud-Init Seeds
shell: bash
run: |
ssh-keygen -t rsa -b 4096 -f ./id_rsa -N ""
PUBKEY="$(cat ./id_rsa.pub)"
# Per-VM cloud-init: each worker gets a unique hostname and a static IP
# on the bridge subnet via netplan.
for i in $(seq 1 ${{ inputs.worker-count }}); do
IP="${CIDR_PREFIX}.1${i}" # e.g. 192.168.100.11 / 192.168.100.12 / ...
cat <<EOF > user-data-${i}
#cloud-config
hostname: worker-${i}
preserve_hostname: false
manage_etc_hosts: true
# Stop cloud-init from generating its own DHCP netplan that would
# conflict with the static one we write in write_files below.
network:
config: disabled
users:
- name: ubuntu
ssh_authorized_keys:
- ${PUBKEY}
sudo: ['ALL=(ALL) NOPASSWD:ALL']
shell: /bin/bash
write_files:
- path: /etc/netplan/50-static.yaml
permissions: '0600'
content: |
network:
version: 2
ethernets:
ens3:
dhcp4: false
addresses: [${IP}/24]
routes:
- to: default
via: ${CIDR_PREFIX}.1
nameservers:
addresses: [8.8.8.8, 1.1.1.1]
runcmd:
- netplan apply
EOF
cloud-localds seed-${i}.img user-data-${i}
done
- name: Create Worker Disks
shell: bash
run: |
for i in $(seq 1 ${{ inputs.worker-count }}); do
qemu-img create -f qcow2 -b ubuntu-cloudimg.img -F qcow2 worker-${i}.qcow2 20G
done
- name: Launch Worker VMs
shell: bash
run: |
for i in $(seq 1 ${{ inputs.worker-count }}); do
# Each VM gets a unique MAC, attached to its own tap on k3kbr0.
MAC=$(printf '52:54:00:12:34:%02x' $((85 + i)))
sudo qemu-system-x86_64 \
-m 2048 -smp 2 -cpu host -enable-kvm -nographic \
-drive file=worker-${i}.qcow2,if=virtio \
-drive file=seed-${i}.img,format=raw,if=virtio \
-netdev tap,id=net0,ifname=tap-w${i},script=no,downscript=no \
-device virtio-net-pci,netdev=net0,mac=${MAC} \
&
# Wait a moment before launching the next VM
sleep 5
done
- name: Wait for SSH Availability
shell: bash
run: |
for i in $(seq 1 ${{ inputs.worker-count }}); do
IP="${CIDR_PREFIX}.1${i}"
echo "Waiting for Worker ${i} (${IP}) to respond..."
timeout 180s bash -c "
until ssh -i ./id_rsa -o StrictHostKeyChecking=no -o ConnectTimeout=2 ubuntu@${IP} true 2>/dev/null; do sleep 3; done
"
done
echo "All VMs are up and running!"
- name: Verify Worker VM Configuration
shell: bash
run: |
for i in $(seq 1 ${{ inputs.worker-count }}); do
IP="${CIDR_PREFIX}.1${i}"
echo "=== Worker at ${IP} ==="
ssh -i ./id_rsa -o StrictHostKeyChecking=no ubuntu@${IP} \
"echo 'Hostname:' \$(hostname) && \
echo 'IP Address:' \$(ip -4 addr show ens3 | grep -oP '(?<=inet\s)\d+(\.\d+){3}') && \
echo 'Gateway:' \$(ip route | grep default)"
echo ""
done
- name: Join Workers to K3s Cluster
shell: bash
run: |
for i in $(seq 1 ${{ inputs.worker-count }}); do
IP="${CIDR_PREFIX}.1${i}"
echo "Registering Worker ${i} (k3s ${{ inputs.k3s-version }})..."
ssh -i ./id_rsa -o StrictHostKeyChecking=no ubuntu@${IP} \
"curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=${{ inputs.k3s-version }} K3S_URL=${{ inputs.k3s-url }} K3S_TOKEN=${{ inputs.k3s-token }} sh -"
done
.github/workflows/test-conformance-hcp.yaml
+17 -163
View File
@@ -15,14 +15,15 @@ on:
type: choice
options:
- ""
- "v1.34.6"
- "v1.35.3"
- "v1.34.9"
- "v1.35.6"
- "v1.36.2"
permissions:
contents: read
env:
K8S_VERSIONS: "v1.34.6,v1.35.3"
K8S_VERSIONS: "v1.34.9,v1.35.6,v1.36.2"
HELM_VERSION: v4.1.3
HELM_CHECKSUM_AMD64: 02ce9722d541238f81459938b84cf47df2fdf1187493b4bfb2346754d82a4700
@@ -157,176 +158,29 @@ jobs:
kubectl get nodes
kubectl get pods -A
- name: Install Virtualization Dependencies
- name: Read K3k virtual cluster token
id: k3s-token
run: |
sudo apt-get update
sudo apt-get install -y qemu-kvm qemu-utils cloud-image-utils wget
sudo usermod -aG kvm $USER
TOKEN=$(kubectl get secret -n k3k-mycluster k3k-mycluster-token -o jsonpath='{.data.token}' | base64 -d)
echo "::add-mask::${TOKEN}"
echo "token=${TOKEN}" >> "$GITHUB_OUTPUT"
kvm-ok
- name: Provision worker VMs and join them to the K3k virtual cluster
uses: ./.github/actions/provision-k3s-vm-workers
with:
worker-count: 2
k3s-url: https://192.168.100.1:30001
k3s-token: ${{ steps.k3s-token.outputs.token }}
k3s-version: ${{ env.KUBERNETES_VERSION }}+k3s1
- name: Set up bridge network for VMs
- name: Test connectivity from VMs to K3k API server
run: |
# Create a bridge so both VMs share an L2 segment with unique routable IPs.
# Required because QEMU `-net user` gives every VM the same 10.0.2.15 NAT
# address, which breaks flannel VXLAN between workers.
sudo ip link add name k3kbr0 type bridge
sudo ip addr add 192.168.100.1/24 dev k3kbr0
sudo ip link set k3kbr0 up
# NAT outbound so VMs can reach the internet (image pulls etc).
sudo sysctl -w net.ipv4.ip_forward=1
sudo iptables -t nat -A POSTROUTING -s 192.168.100.0/24 ! -o k3kbr0 -j MASQUERADE
sudo iptables -A FORWARD -i k3kbr0 -j ACCEPT
sudo iptables -A FORWARD -o k3kbr0 -j ACCEPT
# One TAP per VM, attached to the bridge.
sudo ip tuntap add tap-w1 mode tap
sudo ip link set tap-w1 master k3kbr0
sudo ip link set tap-w1 up
sudo ip tuntap add tap-w2 mode tap
sudo ip link set tap-w2 master k3kbr0
sudo ip link set tap-w2 up
- name: Download Base Cloud Image
run: |
# PINNED TO UBUNTU 24.04 LTS (noble).
#
# Newer Ubuntu releases (tested: 26.04 "resolute") ship a stricter
# `cri-containerd.apparmor.d` profile that denies inter-thread signal
# delivery. The BIND ISC library used by `nslookup` relies on those
# signals during shutdown (`isc_app_ctxshutdown()` calls `kill()`),
# so when AppArmor denies them nslookup exits 139 with
# "kill: Permission denied". The conformance tests
# `[sig-network] Services should be able to change the type from
# {NodePort,ClusterIP} to ExternalName`
# both run `nslookup` from an exec pod and fail in that case.
#
# Before bumping past 24.04, verify those two conformance tests still
# pass — or that the containerd AppArmor profile on the newer release
# has been relaxed to allow intra-pod signals.
wget -q https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img -O ubuntu-cloudimg.img
- name: Generate SSH Key and Cloud-Init Seeds
run: |
ssh-keygen -t rsa -b 4096 -f ./id_rsa -N ""
PUBKEY="$(cat ./id_rsa.pub)"
# Per-VM cloud-init: each worker gets a unique hostname and a static IP
# on the bridge subnet via netplan.
for i in 1 2; do
IP="192.168.100.1${i}" # 192.168.100.11 / 192.168.100.12
cat <<EOF > user-data-${i}
#cloud-config
hostname: worker-${i}
preserve_hostname: false
manage_etc_hosts: true
# Stop cloud-init from generating its own DHCP netplan that would
# conflict with the static one we write in write_files below.
network:
config: disabled
users:
- name: ubuntu
ssh_authorized_keys:
- ${PUBKEY}
sudo: ['ALL=(ALL) NOPASSWD:ALL']
shell: /bin/bash
write_files:
- path: /etc/netplan/50-static.yaml
permissions: '0600'
content: |
network:
version: 2
ethernets:
ens3:
dhcp4: false
addresses: [${IP}/24]
routes:
- to: default
via: 192.168.100.1
nameservers:
addresses: [8.8.8.8, 1.1.1.1]
runcmd:
- netplan apply
EOF
cloud-localds seed-${i}.img user-data-${i}
done
- name: Create Worker Disks
run: |
qemu-img create -f qcow2 -b ubuntu-cloudimg.img -F qcow2 worker-1.qcow2 20G
qemu-img create -f qcow2 -b ubuntu-cloudimg.img -F qcow2 worker-2.qcow2 20G
- name: Launch Worker VMs
run: |
# Launch Worker 1 — attached to tap-w1 on k3kbr0.
sudo qemu-system-x86_64 \
-m 2048 -smp 2 -cpu host -enable-kvm -nographic \
-drive file=worker-1.qcow2,if=virtio \
-drive file=seed-1.img,format=raw,if=virtio \
-netdev tap,id=net0,ifname=tap-w1,script=no,downscript=no \
-device virtio-net-pci,netdev=net0,mac=52:54:00:12:34:56 \
&
# Wait a moment before launching the second VM
sleep 5
# Launch Worker 2 — attached to tap-w2 on k3kbr0.
sudo qemu-system-x86_64 \
-m 2048 -smp 2 -cpu host -enable-kvm -nographic \
-drive file=worker-2.qcow2,if=virtio \
-drive file=seed-2.img,format=raw,if=virtio \
-netdev tap,id=net0,ifname=tap-w2,script=no,downscript=no \
-device virtio-net-pci,netdev=net0,mac=52:54:00:12:34:57 \
&
- name: Wait for SSH Availability
run: |
echo "Waiting for Worker 1 (192.168.100.11) to respond..."
timeout 180s bash -c '
until ssh -i ./id_rsa -o StrictHostKeyChecking=no -o ConnectTimeout=2 ubuntu@192.168.100.11 true 2>/dev/null; do sleep 3; done
'
echo "Waiting for Worker 2 (192.168.100.12) to respond..."
timeout 180s bash -c '
until ssh -i ./id_rsa -o StrictHostKeyChecking=no -o ConnectTimeout=2 ubuntu@192.168.100.12 true 2>/dev/null; do sleep 3; done
'
echo "Both VMs are up and running!"
echo "Testing connectivity from VMs to K3k API server..."
ssh -i ./id_rsa -o StrictHostKeyChecking=no ubuntu@192.168.100.11 \
"curl -kv --max-time 10 https://192.168.100.1:30001/readyz || echo 'Worker 1 connectivity test failed'"
ssh -i ./id_rsa -o StrictHostKeyChecking=no ubuntu@192.168.100.12 \
"curl -kv --max-time 10 https://192.168.100.1:30001/readyz || echo 'Worker 2 connectivity test failed'"
- name: Verify Worker VM Configuration
run: |
for IP in 192.168.100.11 192.168.100.12; do
echo "=== Worker at ${IP} ==="
ssh -i ./id_rsa -o StrictHostKeyChecking=no ubuntu@${IP} \
"echo 'Hostname:' \$(hostname) && \
echo 'IP Address:' \$(ip -4 addr show ens3 | grep -oP '(?<=inet\s)\d+(\.\d+){3}') && \
echo 'Gateway:' \$(ip route | grep default)"
echo ""
done
- name: Join Workers to K3k Control Plane
env:
K3S_WORKER_VERSION: ${{ env.KUBERNETES_VERSION }}+k3s1
run: |
K3S_TOKEN=$(kubectl get secret -n k3k-mycluster k3k-mycluster-token -o jsonpath='{.data.token}' | base64 -d)
echo "Registering Worker 1 (k3s ${K3S_WORKER_VERSION})..."
ssh -i ./id_rsa -o StrictHostKeyChecking=no ubuntu@192.168.100.11 \
"curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=${K3S_WORKER_VERSION} K3S_URL=https://192.168.100.1:30001 K3S_TOKEN=${K3S_TOKEN} sh -"
echo "Registering Worker 2 (k3s ${K3S_WORKER_VERSION})..."
ssh -i ./id_rsa -o StrictHostKeyChecking=no ubuntu@192.168.100.12 \
"curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=${K3S_WORKER_VERSION} K3S_URL=https://192.168.100.1:30001 K3S_TOKEN=${K3S_TOKEN} sh -"
- name: Verify Cluster Nodes
env:
KUBECONFIG: ${{ github.workspace }}/k3k-mycluster-mycluster-kubeconfig.yaml
.github/workflows/test-e2e.yaml
+94 -5
View File
@@ -43,7 +43,38 @@ jobs:
- name: Install k3s
run: |
curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=${K3S_HOST_VERSION} INSTALL_K3S_EXEC="--write-kubeconfig-mode=777" sh -s -
curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=${K3S_HOST_VERSION} INSTALL_K3S_EXEC="--write-kubeconfig-mode=777" sh -s -
- name: Read host k3s node token
id: k3s-token
run: |
TOKEN=$(sudo cat /var/lib/rancher/k3s/server/node-token)
echo "::add-mask::${TOKEN}"
echo "token=${TOKEN}" >> "$GITHUB_OUTPUT"
- name: Provision worker VMs and join them to the host cluster
uses: ./.github/actions/provision-k3s-vm-workers
env:
K3S_HOST_VERSION: ${{ env.K3S_HOST_VERSION }}
with:
worker-count: 2
k3s-url: https://192.168.100.1:6443
k3s-token: ${{ steps.k3s-token.outputs.token }}
k3s-version: ${{ env.K3S_HOST_VERSION }}
- name: Wait for all host nodes to be Ready
env:
KUBECONFIG: /etc/rancher/k3s/k3s.yaml
run: |
kubectl get nodes
timeout 180s bash -c '
until [ $(kubectl get nodes --no-headers 2>/dev/null | grep -c "Ready") -eq 3 ]; do
echo "Waiting for all 3 nodes to show Ready..."
kubectl get nodes || true
sleep 5
done
'
- name: Build and package and push dev images
env:
@@ -65,7 +96,7 @@ jobs:
- name: Convert coverage data
run: go tool covdata textfmt -i=${GOCOVERDIR} -o ${GOCOVERDIR}/cover.out
- name: Upload coverage reports to Codecov (controller)
uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7
with:
@@ -88,6 +119,12 @@ jobs:
journalctl -u k3s -o cat --no-pager > /tmp/k3s.log
kubectl logs -n k3k-system -l "app.kubernetes.io/name=k3k" --tail=-1 > /tmp/k3k.log
for i in 1 2; do
IP="192.168.100.1${i}"
ssh -i ./id_rsa -o StrictHostKeyChecking=no ubuntu@${IP} \
"sudo journalctl -u k3s-agent -o cat --no-pager" > /tmp/worker-${i}.log || true
done
- name: Archive k3s logs
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
if: always()
@@ -101,6 +138,14 @@ jobs:
with:
name: e2e-k3k-logs
path: /tmp/k3k.log
- name: Archive worker VM logs
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
if: always()
with:
name: e2e-worker-logs
path: /tmp/worker-*.log
tests-e2e-slow:
runs-on: ubuntu-latest
@@ -130,7 +175,38 @@ jobs:
- name: Install k3s
run: |
curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=${K3S_HOST_VERSION} INSTALL_K3S_EXEC="--write-kubeconfig-mode=777" sh -s -
curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=${K3S_HOST_VERSION} INSTALL_K3S_EXEC="--write-kubeconfig-mode=777" sh -s -
- name: Read host k3s node token
id: k3s-token
run: |
TOKEN=$(sudo cat /var/lib/rancher/k3s/server/node-token)
echo "::add-mask::${TOKEN}"
echo "token=${TOKEN}" >> "$GITHUB_OUTPUT"
- name: Provision worker VMs and join them to the host cluster
uses: ./.github/actions/provision-k3s-vm-workers
env:
K3S_HOST_VERSION: ${{ env.K3S_HOST_VERSION }}
with:
worker-count: 2
k3s-url: https://192.168.100.1:6443
k3s-token: ${{ steps.k3s-token.outputs.token }}
k3s-version: ${{ env.K3S_HOST_VERSION }}
- name: Wait for all host nodes to be Ready
env:
KUBECONFIG: /etc/rancher/k3s/k3s.yaml
run: |
kubectl get nodes
timeout 180s bash -c '
until [ $(kubectl get nodes --no-headers 2>/dev/null | grep -c "Ready") -eq 3 ]; do
echo "Waiting for all 3 nodes to show Ready..."
kubectl get nodes || true
sleep 5
done
'
- name: Build and package and push dev images
env:
@@ -152,7 +228,7 @@ jobs:
- name: Convert coverage data
run: go tool covdata textfmt -i=${GOCOVERDIR} -o ${GOCOVERDIR}/cover.out
- name: Upload coverage reports to Codecov (controller)
uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7
with:
@@ -175,6 +251,12 @@ jobs:
journalctl -u k3s -o cat --no-pager > /tmp/k3s.log
kubectl logs -n k3k-system -l "app.kubernetes.io/name=k3k" --tail=-1 > /tmp/k3k.log
for i in 1 2; do
IP="192.168.100.1${i}"
ssh -i ./id_rsa -o StrictHostKeyChecking=no ubuntu@${IP} \
"sudo journalctl -u k3s-agent -o cat --no-pager" > /tmp/worker-${i}.log || true
done
- name: Archive k3s logs
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
if: always()
@@ -187,4 +269,11 @@ jobs:
if: always()
with:
name: e2e-slow-k3k-logs
path: /tmp/k3k.log
path: /tmp/k3k.log
- name: Archive worker VM logs
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
if: always()
with:
name: e2e-slow-worker-logs
path: /tmp/worker-*.log