Move e2e tests to a multi-node cluster, add kubelet restart test (#1010)

* Add E2E tests for multi-node setup and kubelet restart scenarios

* Update Kubernetes version options in conformance tests workflow

* Enhance VM provisioning action to support dynamic bridge CIDR prefix and pin package versions

.github/workflows/test-e2e.yaml

@@ -43,7 +43,38 @@ jobs:
 
     - name: Install k3s
       run: |
-        curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=${K3S_HOST_VERSION} INSTALL_K3S_EXEC="--write-kubeconfig-mode=777" sh -s - 
+        curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=${K3S_HOST_VERSION} INSTALL_K3S_EXEC="--write-kubeconfig-mode=777" sh -s -
+
+    - name: Read host k3s node token
+      id: k3s-token
+      run: |
+        TOKEN=$(sudo cat /var/lib/rancher/k3s/server/node-token)
+        echo "::add-mask::${TOKEN}"
+        echo "token=${TOKEN}" >> "$GITHUB_OUTPUT"
+
+    - name: Provision worker VMs and join them to the host cluster
+      uses: ./.github/actions/provision-k3s-vm-workers
+      env:
+        K3S_HOST_VERSION: ${{ env.K3S_HOST_VERSION }}
+      with:
+        worker-count: 2
+        k3s-url: https://192.168.100.1:6443
+        k3s-token: ${{ steps.k3s-token.outputs.token }}
+        k3s-version: ${{ env.K3S_HOST_VERSION }}
+
+    - name: Wait for all host nodes to be Ready
+      env:
+        KUBECONFIG: /etc/rancher/k3s/k3s.yaml
+      run: |
+        kubectl get nodes
+
+        timeout 180s bash -c '
+          until [ $(kubectl get nodes --no-headers 2>/dev/null | grep -c "Ready") -eq 3 ]; do
+            echo "Waiting for all 3 nodes to show Ready..."
+            kubectl get nodes || true
+            sleep 5
+          done
+        '
 
     - name: Build and package and push dev images
       env:
@@ -65,7 +96,7 @@ jobs:
 
     - name: Convert coverage data
       run: go tool covdata textfmt -i=${GOCOVERDIR} -o ${GOCOVERDIR}/cover.out
-    
+
     - name: Upload coverage reports to Codecov (controller)
       uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7
       with:
@@ -88,6 +119,12 @@ jobs:
         journalctl -u k3s -o cat --no-pager > /tmp/k3s.log
         kubectl logs -n k3k-system -l "app.kubernetes.io/name=k3k" --tail=-1 > /tmp/k3k.log
 
+        for i in 1 2; do
+          IP="192.168.100.1${i}"
+          ssh -i ./id_rsa -o StrictHostKeyChecking=no ubuntu@${IP} \
+            "sudo journalctl -u k3s-agent -o cat --no-pager" > /tmp/worker-${i}.log || true
+        done
+
     - name: Archive k3s logs
       uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
       if: always()
@@ -101,6 +138,14 @@ jobs:
       with:
         name: e2e-k3k-logs
         path: /tmp/k3k.log
+
+    - name: Archive worker VM logs
+      uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+      if: always()
+      with:
+        name: e2e-worker-logs
+        path: /tmp/worker-*.log
+
   tests-e2e-slow:
     runs-on: ubuntu-latest
 
@@ -130,7 +175,38 @@ jobs:
 
     - name: Install k3s
       run: |
-        curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=${K3S_HOST_VERSION} INSTALL_K3S_EXEC="--write-kubeconfig-mode=777" sh -s - 
+        curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=${K3S_HOST_VERSION} INSTALL_K3S_EXEC="--write-kubeconfig-mode=777" sh -s -
+
+    - name: Read host k3s node token
+      id: k3s-token
+      run: |
+        TOKEN=$(sudo cat /var/lib/rancher/k3s/server/node-token)
+        echo "::add-mask::${TOKEN}"
+        echo "token=${TOKEN}" >> "$GITHUB_OUTPUT"
+
+    - name: Provision worker VMs and join them to the host cluster
+      uses: ./.github/actions/provision-k3s-vm-workers
+      env:
+        K3S_HOST_VERSION: ${{ env.K3S_HOST_VERSION }}
+      with:
+        worker-count: 2
+        k3s-url: https://192.168.100.1:6443
+        k3s-token: ${{ steps.k3s-token.outputs.token }}
+        k3s-version: ${{ env.K3S_HOST_VERSION }}
+
+    - name: Wait for all host nodes to be Ready
+      env:
+        KUBECONFIG: /etc/rancher/k3s/k3s.yaml
+      run: |
+        kubectl get nodes
+
+        timeout 180s bash -c '
+          until [ $(kubectl get nodes --no-headers 2>/dev/null | grep -c "Ready") -eq 3 ]; do
+            echo "Waiting for all 3 nodes to show Ready..."
+            kubectl get nodes || true
+            sleep 5
+          done
+        '
 
     - name: Build and package and push dev images
       env:
@@ -152,7 +228,7 @@ jobs:
 
     - name: Convert coverage data
       run: go tool covdata textfmt -i=${GOCOVERDIR} -o ${GOCOVERDIR}/cover.out
-    
+
     - name: Upload coverage reports to Codecov (controller)
       uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7
       with:
@@ -175,6 +251,12 @@ jobs:
         journalctl -u k3s -o cat --no-pager > /tmp/k3s.log
         kubectl logs -n k3k-system -l "app.kubernetes.io/name=k3k" --tail=-1 > /tmp/k3k.log
 
+        for i in 1 2; do
+          IP="192.168.100.1${i}"
+          ssh -i ./id_rsa -o StrictHostKeyChecking=no ubuntu@${IP} \
+            "sudo journalctl -u k3s-agent -o cat --no-pager" > /tmp/worker-${i}.log || true
+        done
+
     - name: Archive k3s logs
       uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
       if: always()
@@ -187,4 +269,11 @@ jobs:
       if: always()
       with:
         name: e2e-slow-k3k-logs
-        path: /tmp/k3k.log
+        path: /tmp/k3k.log
+
+    - name: Archive worker VM logs
+      uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+      if: always()
+      with:
+        name: e2e-slow-worker-logs
+        path: /tmp/worker-*.log