diff --git a/.github/actions/provision-k3s-vm-workers/action.yml b/.github/actions/provision-k3s-vm-workers/action.yml new file mode 100644 index 00000000..6b28e6dd --- /dev/null +++ b/.github/actions/provision-k3s-vm-workers/action.yml @@ -0,0 +1,201 @@ +name: 'Provision k3s VM workers' +description: > + Launches real QEMU/KVM worker VMs on a bridged network and joins them as k3s agents to a given + k3s server. Used to turn a single-node CI host into a genuine multi-node cluster (unlike + container-based nodes, e.g. k3d, these are independent kernels/machines). + +inputs: + worker-count: + description: 'Number of worker VMs to launch' + required: false + default: '2' + bridge-cidr-prefix: + description: 'First three octets of the bridge subnet the VMs are attached to' + required: false + default: '192.168.100' + k3s-url: + description: 'K3s server URL the workers should join (e.g. https://192.168.100.1:6443)' + required: true + k3s-token: + description: 'K3s token to join the server with' + required: true + k3s-version: + description: 'K3s version to install on the workers (e.g. v1.34.6+k3s1)' + required: true + +runs: + using: 'composite' + steps: + - name: Install Virtualization Dependencies + shell: bash + run: | + echo "CIDR_PREFIX=${{ inputs.bridge-cidr-prefix }}" >> "$GITHUB_ENV" + + sudo apt-get update + + # Pinned to the versions available on the runner's Ubuntu 24.04 (noble) image at the + # time of writing. These come from Ubuntu's regular archive, which does not retain + # superseded versions — a routine security update to any of these will make the pin + # unresolvable and fail the install below. If that happens, check the versions + # currently available (logged on every run) and update the pins to match. + echo "Available versions for pinned packages:" + apt-cache madison qemu-kvm qemu-utils cloud-image-utils + + sudo apt-get install -y \ + qemu-kvm=1:8.2.2+ds-0ubuntu1.17 \ + qemu-utils=1:8.2.2+ds-0ubuntu1.17 \ + cloud-image-utils=0.33-1 + sudo usermod -aG kvm $USER + + kvm-ok + + - name: Set up bridge network for VMs + shell: bash + run: | + # Create a bridge so both VMs share an L2 segment with unique routable IPs. + # Required because QEMU `-net user` gives every VM the same 10.0.2.15 NAT + # address, which breaks flannel VXLAN between workers. + sudo ip link add name k3kbr0 type bridge + sudo ip addr add ${CIDR_PREFIX}.1/24 dev k3kbr0 + sudo ip link set k3kbr0 up + + # NAT outbound so VMs can reach the internet (image pulls etc). + sudo sysctl -w net.ipv4.ip_forward=1 + sudo iptables -t nat -A POSTROUTING -s ${CIDR_PREFIX}.0/24 ! -o k3kbr0 -j MASQUERADE + sudo iptables -A FORWARD -i k3kbr0 -j ACCEPT + sudo iptables -A FORWARD -o k3kbr0 -j ACCEPT + + # One TAP per VM, attached to the bridge. + for i in $(seq 1 ${{ inputs.worker-count }}); do + sudo ip tuntap add tap-w${i} mode tap + sudo ip link set tap-w${i} master k3kbr0 + sudo ip link set tap-w${i} up + done + + - name: Download Base Cloud Image + shell: bash + run: | + # PINNED TO UBUNTU 24.04 LTS (noble). + # + # Newer Ubuntu releases (tested: 26.04 "resolute") ship a stricter + # `cri-containerd.apparmor.d` profile that denies inter-thread signal + # delivery. The BIND ISC library used by `nslookup` relies on those + # signals during shutdown (`isc_app_ctxshutdown()` calls `kill()`), + # so when AppArmor denies them nslookup exits 139 with + # "kill: Permission denied". The conformance tests + # `[sig-network] Services should be able to change the type from + # {NodePort,ClusterIP} to ExternalName` + # both run `nslookup` from an exec pod and fail in that case. + # + # Before bumping past 24.04, verify those two conformance tests still + # pass — or that the containerd AppArmor profile on the newer release + # has been relaxed to allow intra-pod signals. + wget -q https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img -O ubuntu-cloudimg.img + + - name: Generate SSH Key and Cloud-Init Seeds + shell: bash + run: | + ssh-keygen -t rsa -b 4096 -f ./id_rsa -N "" + PUBKEY="$(cat ./id_rsa.pub)" + + # Per-VM cloud-init: each worker gets a unique hostname and a static IP + # on the bridge subnet via netplan. + for i in $(seq 1 ${{ inputs.worker-count }}); do + IP="${CIDR_PREFIX}.1${i}" # e.g. 192.168.100.11 / 192.168.100.12 / ... + cat < user-data-${i} + #cloud-config + hostname: worker-${i} + preserve_hostname: false + manage_etc_hosts: true + # Stop cloud-init from generating its own DHCP netplan that would + # conflict with the static one we write in write_files below. + network: + config: disabled + users: + - name: ubuntu + ssh_authorized_keys: + - ${PUBKEY} + sudo: ['ALL=(ALL) NOPASSWD:ALL'] + shell: /bin/bash + write_files: + - path: /etc/netplan/50-static.yaml + permissions: '0600' + content: | + network: + version: 2 + ethernets: + ens3: + dhcp4: false + addresses: [${IP}/24] + routes: + - to: default + via: ${CIDR_PREFIX}.1 + nameservers: + addresses: [8.8.8.8, 1.1.1.1] + runcmd: + - netplan apply + EOF + cloud-localds seed-${i}.img user-data-${i} + done + + - name: Create Worker Disks + shell: bash + run: | + for i in $(seq 1 ${{ inputs.worker-count }}); do + qemu-img create -f qcow2 -b ubuntu-cloudimg.img -F qcow2 worker-${i}.qcow2 20G + done + + - name: Launch Worker VMs + shell: bash + run: | + for i in $(seq 1 ${{ inputs.worker-count }}); do + # Each VM gets a unique MAC, attached to its own tap on k3kbr0. + MAC=$(printf '52:54:00:12:34:%02x' $((85 + i))) + + sudo qemu-system-x86_64 \ + -m 2048 -smp 2 -cpu host -enable-kvm -nographic \ + -drive file=worker-${i}.qcow2,if=virtio \ + -drive file=seed-${i}.img,format=raw,if=virtio \ + -netdev tap,id=net0,ifname=tap-w${i},script=no,downscript=no \ + -device virtio-net-pci,netdev=net0,mac=${MAC} \ + & + + # Wait a moment before launching the next VM + sleep 5 + done + + - name: Wait for SSH Availability + shell: bash + run: | + for i in $(seq 1 ${{ inputs.worker-count }}); do + IP="${CIDR_PREFIX}.1${i}" + echo "Waiting for Worker ${i} (${IP}) to respond..." + timeout 180s bash -c " + until ssh -i ./id_rsa -o StrictHostKeyChecking=no -o ConnectTimeout=2 ubuntu@${IP} true 2>/dev/null; do sleep 3; done + " + done + + echo "All VMs are up and running!" + + - name: Verify Worker VM Configuration + shell: bash + run: | + for i in $(seq 1 ${{ inputs.worker-count }}); do + IP="${CIDR_PREFIX}.1${i}" + echo "=== Worker at ${IP} ===" + ssh -i ./id_rsa -o StrictHostKeyChecking=no ubuntu@${IP} \ + "echo 'Hostname:' \$(hostname) && \ + echo 'IP Address:' \$(ip -4 addr show ens3 | grep -oP '(?<=inet\s)\d+(\.\d+){3}') && \ + echo 'Gateway:' \$(ip route | grep default)" + echo "" + done + + - name: Join Workers to K3s Cluster + shell: bash + run: | + for i in $(seq 1 ${{ inputs.worker-count }}); do + IP="${CIDR_PREFIX}.1${i}" + echo "Registering Worker ${i} (k3s ${{ inputs.k3s-version }})..." + ssh -i ./id_rsa -o StrictHostKeyChecking=no ubuntu@${IP} \ + "curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=${{ inputs.k3s-version }} K3S_URL=${{ inputs.k3s-url }} K3S_TOKEN=${{ inputs.k3s-token }} sh -" + done diff --git a/.github/workflows/test-conformance-hcp.yaml b/.github/workflows/test-conformance-hcp.yaml index b927d16d..8849355e 100644 --- a/.github/workflows/test-conformance-hcp.yaml +++ b/.github/workflows/test-conformance-hcp.yaml @@ -15,14 +15,15 @@ on: type: choice options: - "" - - "v1.34.6" - - "v1.35.3" + - "v1.34.9" + - "v1.35.6" + - "v1.36.2" permissions: contents: read env: - K8S_VERSIONS: "v1.34.6,v1.35.3" + K8S_VERSIONS: "v1.34.9,v1.35.6,v1.36.2" HELM_VERSION: v4.1.3 HELM_CHECKSUM_AMD64: 02ce9722d541238f81459938b84cf47df2fdf1187493b4bfb2346754d82a4700 @@ -157,176 +158,29 @@ jobs: kubectl get nodes kubectl get pods -A - - name: Install Virtualization Dependencies + - name: Read K3k virtual cluster token + id: k3s-token run: | - sudo apt-get update - sudo apt-get install -y qemu-kvm qemu-utils cloud-image-utils wget - sudo usermod -aG kvm $USER + TOKEN=$(kubectl get secret -n k3k-mycluster k3k-mycluster-token -o jsonpath='{.data.token}' | base64 -d) + echo "::add-mask::${TOKEN}" + echo "token=${TOKEN}" >> "$GITHUB_OUTPUT" - kvm-ok + - name: Provision worker VMs and join them to the K3k virtual cluster + uses: ./.github/actions/provision-k3s-vm-workers + with: + worker-count: 2 + k3s-url: https://192.168.100.1:30001 + k3s-token: ${{ steps.k3s-token.outputs.token }} + k3s-version: ${{ env.KUBERNETES_VERSION }}+k3s1 - - name: Set up bridge network for VMs + - name: Test connectivity from VMs to K3k API server run: | - # Create a bridge so both VMs share an L2 segment with unique routable IPs. - # Required because QEMU `-net user` gives every VM the same 10.0.2.15 NAT - # address, which breaks flannel VXLAN between workers. - sudo ip link add name k3kbr0 type bridge - sudo ip addr add 192.168.100.1/24 dev k3kbr0 - sudo ip link set k3kbr0 up - - # NAT outbound so VMs can reach the internet (image pulls etc). - sudo sysctl -w net.ipv4.ip_forward=1 - sudo iptables -t nat -A POSTROUTING -s 192.168.100.0/24 ! -o k3kbr0 -j MASQUERADE - sudo iptables -A FORWARD -i k3kbr0 -j ACCEPT - sudo iptables -A FORWARD -o k3kbr0 -j ACCEPT - - # One TAP per VM, attached to the bridge. - sudo ip tuntap add tap-w1 mode tap - sudo ip link set tap-w1 master k3kbr0 - sudo ip link set tap-w1 up - - sudo ip tuntap add tap-w2 mode tap - sudo ip link set tap-w2 master k3kbr0 - sudo ip link set tap-w2 up - - - name: Download Base Cloud Image - run: | - # PINNED TO UBUNTU 24.04 LTS (noble). - # - # Newer Ubuntu releases (tested: 26.04 "resolute") ship a stricter - # `cri-containerd.apparmor.d` profile that denies inter-thread signal - # delivery. The BIND ISC library used by `nslookup` relies on those - # signals during shutdown (`isc_app_ctxshutdown()` calls `kill()`), - # so when AppArmor denies them nslookup exits 139 with - # "kill: Permission denied". The conformance tests - # `[sig-network] Services should be able to change the type from - # {NodePort,ClusterIP} to ExternalName` - # both run `nslookup` from an exec pod and fail in that case. - # - # Before bumping past 24.04, verify those two conformance tests still - # pass — or that the containerd AppArmor profile on the newer release - # has been relaxed to allow intra-pod signals. - wget -q https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img -O ubuntu-cloudimg.img - - - name: Generate SSH Key and Cloud-Init Seeds - run: | - ssh-keygen -t rsa -b 4096 -f ./id_rsa -N "" - PUBKEY="$(cat ./id_rsa.pub)" - - # Per-VM cloud-init: each worker gets a unique hostname and a static IP - # on the bridge subnet via netplan. - for i in 1 2; do - IP="192.168.100.1${i}" # 192.168.100.11 / 192.168.100.12 - cat < user-data-${i} - #cloud-config - hostname: worker-${i} - preserve_hostname: false - manage_etc_hosts: true - # Stop cloud-init from generating its own DHCP netplan that would - # conflict with the static one we write in write_files below. - network: - config: disabled - users: - - name: ubuntu - ssh_authorized_keys: - - ${PUBKEY} - sudo: ['ALL=(ALL) NOPASSWD:ALL'] - shell: /bin/bash - write_files: - - path: /etc/netplan/50-static.yaml - permissions: '0600' - content: | - network: - version: 2 - ethernets: - ens3: - dhcp4: false - addresses: [${IP}/24] - routes: - - to: default - via: 192.168.100.1 - nameservers: - addresses: [8.8.8.8, 1.1.1.1] - runcmd: - - netplan apply - EOF - cloud-localds seed-${i}.img user-data-${i} - done - - - name: Create Worker Disks - run: | - qemu-img create -f qcow2 -b ubuntu-cloudimg.img -F qcow2 worker-1.qcow2 20G - qemu-img create -f qcow2 -b ubuntu-cloudimg.img -F qcow2 worker-2.qcow2 20G - - - name: Launch Worker VMs - run: | - # Launch Worker 1 — attached to tap-w1 on k3kbr0. - sudo qemu-system-x86_64 \ - -m 2048 -smp 2 -cpu host -enable-kvm -nographic \ - -drive file=worker-1.qcow2,if=virtio \ - -drive file=seed-1.img,format=raw,if=virtio \ - -netdev tap,id=net0,ifname=tap-w1,script=no,downscript=no \ - -device virtio-net-pci,netdev=net0,mac=52:54:00:12:34:56 \ - & - - # Wait a moment before launching the second VM - sleep 5 - - # Launch Worker 2 — attached to tap-w2 on k3kbr0. - sudo qemu-system-x86_64 \ - -m 2048 -smp 2 -cpu host -enable-kvm -nographic \ - -drive file=worker-2.qcow2,if=virtio \ - -drive file=seed-2.img,format=raw,if=virtio \ - -netdev tap,id=net0,ifname=tap-w2,script=no,downscript=no \ - -device virtio-net-pci,netdev=net0,mac=52:54:00:12:34:57 \ - & - - - name: Wait for SSH Availability - run: | - echo "Waiting for Worker 1 (192.168.100.11) to respond..." - timeout 180s bash -c ' - until ssh -i ./id_rsa -o StrictHostKeyChecking=no -o ConnectTimeout=2 ubuntu@192.168.100.11 true 2>/dev/null; do sleep 3; done - ' - - echo "Waiting for Worker 2 (192.168.100.12) to respond..." - timeout 180s bash -c ' - until ssh -i ./id_rsa -o StrictHostKeyChecking=no -o ConnectTimeout=2 ubuntu@192.168.100.12 true 2>/dev/null; do sleep 3; done - ' - - echo "Both VMs are up and running!" - - echo "Testing connectivity from VMs to K3k API server..." ssh -i ./id_rsa -o StrictHostKeyChecking=no ubuntu@192.168.100.11 \ "curl -kv --max-time 10 https://192.168.100.1:30001/readyz || echo 'Worker 1 connectivity test failed'" ssh -i ./id_rsa -o StrictHostKeyChecking=no ubuntu@192.168.100.12 \ "curl -kv --max-time 10 https://192.168.100.1:30001/readyz || echo 'Worker 2 connectivity test failed'" - - name: Verify Worker VM Configuration - run: | - for IP in 192.168.100.11 192.168.100.12; do - echo "=== Worker at ${IP} ===" - ssh -i ./id_rsa -o StrictHostKeyChecking=no ubuntu@${IP} \ - "echo 'Hostname:' \$(hostname) && \ - echo 'IP Address:' \$(ip -4 addr show ens3 | grep -oP '(?<=inet\s)\d+(\.\d+){3}') && \ - echo 'Gateway:' \$(ip route | grep default)" - echo "" - done - - - name: Join Workers to K3k Control Plane - env: - K3S_WORKER_VERSION: ${{ env.KUBERNETES_VERSION }}+k3s1 - run: | - K3S_TOKEN=$(kubectl get secret -n k3k-mycluster k3k-mycluster-token -o jsonpath='{.data.token}' | base64 -d) - - echo "Registering Worker 1 (k3s ${K3S_WORKER_VERSION})..." - ssh -i ./id_rsa -o StrictHostKeyChecking=no ubuntu@192.168.100.11 \ - "curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=${K3S_WORKER_VERSION} K3S_URL=https://192.168.100.1:30001 K3S_TOKEN=${K3S_TOKEN} sh -" - - echo "Registering Worker 2 (k3s ${K3S_WORKER_VERSION})..." - ssh -i ./id_rsa -o StrictHostKeyChecking=no ubuntu@192.168.100.12 \ - "curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=${K3S_WORKER_VERSION} K3S_URL=https://192.168.100.1:30001 K3S_TOKEN=${K3S_TOKEN} sh -" - - name: Verify Cluster Nodes env: KUBECONFIG: ${{ github.workspace }}/k3k-mycluster-mycluster-kubeconfig.yaml diff --git a/.github/workflows/test-e2e.yaml b/.github/workflows/test-e2e.yaml index 102a63e1..ab52f83b 100644 --- a/.github/workflows/test-e2e.yaml +++ b/.github/workflows/test-e2e.yaml @@ -43,7 +43,38 @@ jobs: - name: Install k3s run: | - curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=${K3S_HOST_VERSION} INSTALL_K3S_EXEC="--write-kubeconfig-mode=777" sh -s - + curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=${K3S_HOST_VERSION} INSTALL_K3S_EXEC="--write-kubeconfig-mode=777" sh -s - + + - name: Read host k3s node token + id: k3s-token + run: | + TOKEN=$(sudo cat /var/lib/rancher/k3s/server/node-token) + echo "::add-mask::${TOKEN}" + echo "token=${TOKEN}" >> "$GITHUB_OUTPUT" + + - name: Provision worker VMs and join them to the host cluster + uses: ./.github/actions/provision-k3s-vm-workers + env: + K3S_HOST_VERSION: ${{ env.K3S_HOST_VERSION }} + with: + worker-count: 2 + k3s-url: https://192.168.100.1:6443 + k3s-token: ${{ steps.k3s-token.outputs.token }} + k3s-version: ${{ env.K3S_HOST_VERSION }} + + - name: Wait for all host nodes to be Ready + env: + KUBECONFIG: /etc/rancher/k3s/k3s.yaml + run: | + kubectl get nodes + + timeout 180s bash -c ' + until [ $(kubectl get nodes --no-headers 2>/dev/null | grep -c "Ready") -eq 3 ]; do + echo "Waiting for all 3 nodes to show Ready..." + kubectl get nodes || true + sleep 5 + done + ' - name: Build and package and push dev images env: @@ -65,7 +96,7 @@ jobs: - name: Convert coverage data run: go tool covdata textfmt -i=${GOCOVERDIR} -o ${GOCOVERDIR}/cover.out - + - name: Upload coverage reports to Codecov (controller) uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7 with: @@ -88,6 +119,12 @@ jobs: journalctl -u k3s -o cat --no-pager > /tmp/k3s.log kubectl logs -n k3k-system -l "app.kubernetes.io/name=k3k" --tail=-1 > /tmp/k3k.log + for i in 1 2; do + IP="192.168.100.1${i}" + ssh -i ./id_rsa -o StrictHostKeyChecking=no ubuntu@${IP} \ + "sudo journalctl -u k3s-agent -o cat --no-pager" > /tmp/worker-${i}.log || true + done + - name: Archive k3s logs uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -101,6 +138,14 @@ jobs: with: name: e2e-k3k-logs path: /tmp/k3k.log + + - name: Archive worker VM logs + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + if: always() + with: + name: e2e-worker-logs + path: /tmp/worker-*.log + tests-e2e-slow: runs-on: ubuntu-latest @@ -130,7 +175,38 @@ jobs: - name: Install k3s run: | - curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=${K3S_HOST_VERSION} INSTALL_K3S_EXEC="--write-kubeconfig-mode=777" sh -s - + curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=${K3S_HOST_VERSION} INSTALL_K3S_EXEC="--write-kubeconfig-mode=777" sh -s - + + - name: Read host k3s node token + id: k3s-token + run: | + TOKEN=$(sudo cat /var/lib/rancher/k3s/server/node-token) + echo "::add-mask::${TOKEN}" + echo "token=${TOKEN}" >> "$GITHUB_OUTPUT" + + - name: Provision worker VMs and join them to the host cluster + uses: ./.github/actions/provision-k3s-vm-workers + env: + K3S_HOST_VERSION: ${{ env.K3S_HOST_VERSION }} + with: + worker-count: 2 + k3s-url: https://192.168.100.1:6443 + k3s-token: ${{ steps.k3s-token.outputs.token }} + k3s-version: ${{ env.K3S_HOST_VERSION }} + + - name: Wait for all host nodes to be Ready + env: + KUBECONFIG: /etc/rancher/k3s/k3s.yaml + run: | + kubectl get nodes + + timeout 180s bash -c ' + until [ $(kubectl get nodes --no-headers 2>/dev/null | grep -c "Ready") -eq 3 ]; do + echo "Waiting for all 3 nodes to show Ready..." + kubectl get nodes || true + sleep 5 + done + ' - name: Build and package and push dev images env: @@ -152,7 +228,7 @@ jobs: - name: Convert coverage data run: go tool covdata textfmt -i=${GOCOVERDIR} -o ${GOCOVERDIR}/cover.out - + - name: Upload coverage reports to Codecov (controller) uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7 with: @@ -175,6 +251,12 @@ jobs: journalctl -u k3s -o cat --no-pager > /tmp/k3s.log kubectl logs -n k3k-system -l "app.kubernetes.io/name=k3k" --tail=-1 > /tmp/k3k.log + for i in 1 2; do + IP="192.168.100.1${i}" + ssh -i ./id_rsa -o StrictHostKeyChecking=no ubuntu@${IP} \ + "sudo journalctl -u k3s-agent -o cat --no-pager" > /tmp/worker-${i}.log || true + done + - name: Archive k3s logs uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -187,4 +269,11 @@ jobs: if: always() with: name: e2e-slow-k3k-logs - path: /tmp/k3k.log \ No newline at end of file + path: /tmp/k3k.log + + - name: Archive worker VM logs + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + if: always() + with: + name: e2e-slow-worker-logs + path: /tmp/worker-*.log diff --git a/tests/e2e/cluster_kubelet_restart_test.go b/tests/e2e/cluster_kubelet_restart_test.go new file mode 100644 index 00000000..c5164c2a --- /dev/null +++ b/tests/e2e/cluster_kubelet_restart_test.go @@ -0,0 +1,191 @@ +package k3k_test + +import ( + "context" + "time" + + "k8s.io/apimachinery/pkg/types" + "k8s.io/kubernetes/pkg/api/v1/pod" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/rancher/k3k/k3k-kubelet/translate" + fwk3k "github.com/rancher/k3k/tests/framework/k3k" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Context("In a shared cluster", Label(e2eTestLabel), Label(slowTestsLabel), Ordered, func() { + var ( + virtualCluster *VirtualCluster + translator *translate.ToHostTranslator + ) + + BeforeAll(func() { + virtualCluster = NewVirtualCluster() + translator = translate.NewHostTranslator(virtualCluster.Cluster) + + DeferCleanup(func() { + fwk3k.DeleteNamespaces(k8s, virtualCluster.Cluster.Namespace) + }) + }) + + When("restarting the k3k-kubelet", func() { + var ( + virtualPod *corev1.Pod + hostPodUID types.UID + ) + + BeforeAll(func() { + ctx := context.Background() + + p := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "nginx-", + Namespace: "default", + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: "nginx", + Image: "nginx", + }}, + }, + } + + var err error + + virtualPod, err = virtualCluster.Client.CoreV1().Pods(p.Namespace).Create(ctx, p, metav1.CreateOptions{}) + Expect(err).To(Not(HaveOccurred())) + + By("Waiting for the Pod to be Running in the virtual cluster") + + Eventually(func(g Gomega) { + vPod, err := virtualCluster.Client.CoreV1().Pods(virtualPod.Namespace).Get(ctx, virtualPod.Name, metav1.GetOptions{}) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(vPod.Status.Phase).To(Equal(corev1.PodRunning)) + }). + WithPolling(time.Second). + WithTimeout(time.Minute). + Should(Succeed()) + + By("Waiting for the Pod to be Running in the host cluster") + + Eventually(func(g Gomega) { + hostPodName := translator.NamespacedName(virtualPod) + + hPod, err := k8s.CoreV1().Pods(hostPodName.Namespace).Get(ctx, hostPodName.Name, metav1.GetOptions{}) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(hPod.Status.Phase).To(Equal(corev1.PodRunning)) + }). + WithPolling(time.Second). + WithTimeout(time.Minute). + Should(Succeed()) + + By("Updating a label on the Pod, to exercise the tracking-metadata-refresh path before the restart") + + virtualPod, err = virtualCluster.Client.CoreV1().Pods(virtualPod.Namespace).Get(ctx, virtualPod.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + if virtualPod.Labels == nil { + virtualPod.Labels = map[string]string{} + } + + virtualPod.Labels["k3k.io/test"] = "updated" + + virtualPod, err = virtualCluster.Client.CoreV1().Pods(virtualPod.Namespace).Update(ctx, virtualPod, metav1.UpdateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + Eventually(func(g Gomega) { + hostPodName := translator.NamespacedName(virtualPod) + + hPod, err := k8s.CoreV1().Pods(hostPodName.Namespace).Get(ctx, hostPodName.Name, metav1.GetOptions{}) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(hPod.Labels).To(HaveKeyWithValue("k3k.io/test", "updated")) + }). + WithPolling(time.Second). + WithTimeout(time.Minute). + Should(Succeed()) + + By("Recording the host and virtual Pod UIDs before the restart") + + hostPodName := translator.NamespacedName(virtualPod) + + hPod, err := k8s.CoreV1().Pods(hostPodName.Namespace).Get(ctx, hostPodName.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + hostPodUID = hPod.UID + Expect(hostPodUID).NotTo(BeEmpty()) + }) + + It("should not delete existing Pods on the host or virtual cluster", func() { + ctx := context.Background() + + By("Restarting every k3k-kubelet agent Pod (one per host node)") + + oldAgentPods := listAgentPods(ctx, virtualCluster) + Expect(oldAgentPods).NotTo(BeEmpty()) + + oldAgentUIDs := map[types.UID]bool{} + for _, agentPod := range oldAgentPods { + oldAgentUIDs[agentPod.UID] = true + } + + for _, agentPod := range oldAgentPods { + Expect(k8s.CoreV1().Pods(virtualCluster.Cluster.Namespace).Delete(ctx, agentPod.Name, metav1.DeleteOptions{})).To(Succeed()) + } + + By("Waiting for every k3k-kubelet agent Pod to be replaced and become Ready") + + Eventually(func(g Gomega) { + newAgentPods := listAgentPods(ctx, virtualCluster) + g.Expect(newAgentPods).To(HaveLen(len(oldAgentPods))) + + for _, agentPod := range newAgentPods { + g.Expect(oldAgentUIDs).NotTo(HaveKey(agentPod.UID), "Pod %q was not replaced", agentPod.Name) + + _, cond := pod.GetPodCondition(&agentPod.Status, corev1.PodReady) + g.Expect(cond).NotTo(BeNil()) + g.Expect(cond.Status).To(BeEquivalentTo(metav1.ConditionTrue)) + } + }). + WithPolling(time.Second). + WithTimeout(2 * time.Minute). + Should(Succeed()) + + // The dangling-pod cleanup inside the virtual-kubelet runs asynchronously right after + // startup, and the new agent Pod can report Ready before that pass has actually run + // or completed. A single check right after restart can pass simply because it ran + // too early. Poll repeatedly over a longer window, and compare against the UIDs + // captured before the restart, so a delete-and-recreate (same name, new UID) is + // caught even if it happens a bit later and even if something else recreated it. + + By("Checking the Pod is never deleted from the virtual cluster") + + Consistently(func(g Gomega) { + vPod, err := virtualCluster.Client.CoreV1().Pods(virtualPod.Namespace).Get(ctx, virtualPod.Name, metav1.GetOptions{}) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(vPod.UID).To(Equal(virtualPod.UID)) + g.Expect(vPod.Status.Phase).To(Equal(corev1.PodRunning)) + }). + WithPolling(5 * time.Second). + WithTimeout(time.Minute). + Should(Succeed()) + + By("Checking the Pod is never deleted from the host cluster") + + Consistently(func(g Gomega) { + hostPodName := translator.NamespacedName(virtualPod) + + hPod, err := k8s.CoreV1().Pods(hostPodName.Namespace).Get(ctx, hostPodName.Name, metav1.GetOptions{}) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(hPod.UID).To(Equal(hostPodUID)) + g.Expect(hPod.Status.Phase).To(Equal(corev1.PodRunning)) + }). + WithPolling(5 * time.Second). + WithTimeout(time.Minute). + Should(Succeed()) + }) + }) +})