k3k/.github/workflows/test-e2e.yaml

name: Tests E2E

on:
  push:
    branches: [main]
  pull_request:
    types: [opened, synchronize, reopened]
    paths-ignore:
    - 'docs/**'
    - '**/*.md'
  workflow_dispatch:

permissions:
  contents: read

env:
  KUBERNETES_VERSION: v1.36.2

jobs:
  tests-e2e:
    runs-on: ubuntu-latest

    strategy:
      fail-fast: false
      matrix:
        include:
        - name: update-config
          label-filter: "update && config"
        - name: update-version
          label-filter: "update && upgrade"
        - name: update-scaling
          label-filter: "update && scaling"
        - name: lifecycle
          label-filter: "lifecycle || kubelet"
        - name: persistence
          label-filter: "persistence"
        - name: sync
          label-filter: "pod || app || sync || policy || status"
        - name: features
          label-filter: "networking || certificates || registry || addons"

    name: tests-e2e (${{ matrix.name }})

    steps:
    - name: Checkout code
      uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
      with:
        fetch-depth: 0
        fetch-tags: true

    - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
      with:
        go-version-file: go.mod

    - name: Install Ginkgo
      run: go install github.com/onsi/ginkgo/v2/ginkgo@9ff1646a26f77a4c0d33ddba3e6368c42c0e8842 # v2.32.0

    - name: Setup environment
      run: |
        mkdir ${{ github.workspace }}/covdata

        echo "COVERAGE=true" >> $GITHUB_ENV
        echo "GOCOVERDIR=${{ github.workspace }}/covdata" >> $GITHUB_ENV
        echo "REPO=ttl.sh/$(uuidgen)" >> $GITHUB_ENV
        echo "VERSION=1h" >> $GITHUB_ENV
        echo "K3S_HOST_VERSION=${{ env.KUBERNETES_VERSION }}+k3s1" >> $GITHUB_ENV

    - name: Install k3s
      run: |
        curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=${K3S_HOST_VERSION} INSTALL_K3S_EXEC="--write-kubeconfig-mode=777" sh -s -

    - name: Read host k3s node token
      id: k3s-token
      run: |
        TOKEN=$(sudo cat /var/lib/rancher/k3s/server/node-token)
        echo "::add-mask::${TOKEN}"
        echo "token=${TOKEN}" >> "$GITHUB_OUTPUT"

    - name: Provision worker VMs and join them to the host cluster
      uses: ./.github/actions/provision-k3s-vm-workers
      env:
        K3S_HOST_VERSION: ${{ env.K3S_HOST_VERSION }}
      with:
        worker-count: 2
        k3s-url: https://192.168.100.1:6443
        k3s-token: ${{ steps.k3s-token.outputs.token }}
        k3s-version: ${{ env.K3S_HOST_VERSION }}

    - name: Wait for all host nodes to be Ready
      env:
        KUBECONFIG: /etc/rancher/k3s/k3s.yaml
      run: |
        kubectl get nodes

        timeout 180s bash -c '
          until [ $(kubectl get nodes --no-headers 2>/dev/null | grep -c "Ready") -eq 3 ]; do
            echo "Waiting for all 3 nodes to show Ready..."
            kubectl get nodes || true
            sleep 5
          done
        '

    - name: Build and package and push dev images
      env:
        KUBECONFIG: /etc/rancher/k3s/k3s.yaml
        REPO: ${{ env.REPO }}
        VERSION: ${{ env.VERSION }}
      run: |
        make build
        make package
        make push
        make install

    - name: Run e2e tests
      env:
        KUBECONFIG: /etc/rancher/k3s/k3s.yaml
        REPO: ${{ env.REPO }}
        VERSION: ${{ env.VERSION }}
      run: make E2E_LABEL_FILTER="${{ matrix.label-filter }}" test-e2e

    - name: Convert coverage data
      run: go tool covdata textfmt -i=${GOCOVERDIR} -o ${GOCOVERDIR}/cover.out

    - name: Upload coverage reports to Codecov (controller)
      uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7
      with:
        token: ${{ secrets.CODECOV_TOKEN }}
        files: ${GOCOVERDIR}/cover.out
        flags: controller

    - name: Upload coverage reports to Codecov (e2e)
      uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7
      with:
        token: ${{ secrets.CODECOV_TOKEN }}
        files: ./cover.out
        flags: e2e

    - name: Export logs
      if: always()
      env:
        KUBECONFIG: /etc/rancher/k3s/k3s.yaml
      run: |
        journalctl -u k3s -o cat --no-pager > /tmp/k3s.log
        kubectl logs -n k3k-system -l "app.kubernetes.io/name=k3k" --tail=-1 > /tmp/k3k.log

        for i in 1 2; do
          IP="192.168.100.1${i}"
          ssh -i ./id_rsa -o StrictHostKeyChecking=no ubuntu@${IP} \
            "sudo journalctl -u k3s-agent -o cat --no-pager" > /tmp/worker-${i}.log || true
        done

    - name: Archive k3s logs
      uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
      if: always()
      with:
        name: e2e-${{ matrix.name }}-k3s-logs
        path: /tmp/k3s.log

    - name: Archive k3k logs
      uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
      if: always()
      with:
        name: e2e-${{ matrix.name }}-k3k-logs
        path: /tmp/k3k.log

    - name: Archive worker VM logs
      uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
      if: always()
      with:
        name: e2e-${{ matrix.name }}-worker-logs
        path: /tmp/worker-*.log