k3k/.github/workflows/test.yaml

name: Tests

on:
  push:
    branches: [main]
  pull_request:
    types: [opened, synchronize, reopened]
  workflow_dispatch:

permissions:
    contents: read

env:
  KUBERNETES_VERSION: v1.35.3
  HELM_VERSION: v4.1.3
  HELM_CHECKSUM_AMD64: 02ce9722d541238f81459938b84cf47df2fdf1187493b4bfb2346754d82a4700

jobs:
  hcp-test:
    runs-on: ubuntu-latest

    steps:
    - name: Checkout code
      uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
      with:
        fetch-depth: 0
        fetch-tags: true

    - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6
      with:
        go-version-file: go.mod

    - name: Install helm
      env:
        FILENAME: helm.tar.gz
      run: |
        curl -sSfL -o ${{ env.FILENAME }} https://get.helm.sh/helm-${{ env.HELM_VERSION }}-linux-amd64.tar.gz
        echo "${{ env.HELM_CHECKSUM_AMD64 }}  ${{ env.FILENAME }}" | sha256sum --check
        tar -xvzf ${{ env.FILENAME }} linux-amd64/helm
        sudo install -m 755 linux-amd64/helm /usr/local/bin/helm

        rm -fr "${{ env.FILENAME }}" linux-amd64/helm

    - name: Install k3s
      env:
        K3S_HOST_VERSION: ${{ env.KUBERNETES_VERSION }}+k3s1
      run: |
        curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=${K3S_HOST_VERSION} INSTALL_K3S_EXEC="--write-kubeconfig-mode=777" sh -s -

        export KUBECONFIG=/etc/rancher/k3s/k3s.yaml
        echo "KUBECONFIG=${KUBECONFIG}" >> $GITHUB_ENV

        kubectl cluster-info
        kubectl get nodes

    - name: Setup K3k (from source)
      run: |
        export REPO=ttl.sh/$(uuidgen)
        export VERSION=1h

        make build
        make package
        make push
        make install

        # add k3kcli to $PATH
        echo "${{ github.workspace }}/bin" >> $GITHUB_PATH

    - name: Wait for K3k controller
      run: |
        echo "Wait for K3k controller deployment to be available"
        kubectl wait -n k3k-system deployment -l "app.kubernetes.io/name=k3k" --for=condition=Available --timeout=5m

    - name: Check k3kcli
      run: k3kcli -v

    - name: Create virtual cluster
      run: |
        kubectl create namespace k3k-mycluster

        cat <<EOF | kubectl apply -f -
        apiVersion: k3k.io/v1beta1
        kind: Cluster
        metadata:
          name: mycluster
          namespace: k3k-mycluster
        spec:
          mode: hcp
          tlsSANs:
            - "127.0.0.1"
            - "10.0.2.2"
          expose:
            nodePort:
              serverPort: 30001
        EOF

        echo "Wait for bootstrap secret to be available"
        kubectl wait -n k3k-mycluster --for=create secret k3k-mycluster-bootstrap --timeout=5m

        k3kcli kubeconfig generate --name mycluster

        export KUBECONFIG=${{ github.workspace }}/k3k-mycluster-mycluster-kubeconfig.yaml

        kubectl cluster-info
        kubectl get nodes
        kubectl get pods -A

    - name: Wait for cluster to be ready
      run: |
        echo "Waiting for cluster to reach Ready phase..."

        timeout 300s bash -c '
          until kubectl get cluster -n k3k-mycluster mycluster -o jsonpath="{.status.phase}" | grep -q "Ready"; do
            PHASE=$(kubectl get cluster -n k3k-mycluster mycluster -o jsonpath="{.status.phase}")
            echo "Current phase: ${PHASE}"
            sleep 5
          done
        '

        echo "✓ Cluster is ready!"

        echo "Verifying NodePort service is accessible from host..."

        if ! curl -k --max-time 5 https://127.0.0.1:30001/readyz; then
          echo "ERROR: Cannot reach NodePort service from host!"
          echo "Service details:"
          kubectl get svc -n k3k-mycluster -l cluster=mycluster,role=server -o yaml
          exit 1
        fi

        echo "✓ NodePort service is accessible"

    - name: Install Virtualization Dependencies
      run: |
        sudo apt-get update
        sudo apt-get install -y qemu-kvm qemu-utils cloud-image-utils wget
        sudo usermod -aG kvm $USER

        kvm-ok

    - name: Download Base Cloud Image
      run: |
        wget -q https://cloud-images.ubuntu.com/resolute/current/resolute-server-cloudimg-amd64.img
        
    - name: Generate SSH Key and Cloud-Init Seed
      run: |
        ssh-keygen -t rsa -b 4096 -f ./id_rsa -N ""
        
        # Define the cloud-init config to authorize our new key
        cat <<EOF > user-data
        #cloud-config
        users:
          - name: ubuntu
            ssh_authorized_keys:
              - $(cat ./id_rsa.pub)
            sudo: ['ALL=(ALL) NOPASSWD:ALL']
            shell: /bin/bash
        EOF
        
        # Bake the configuration into a reusable metadata ISO
        cloud-localds seed.img user-data

    # 4. Provision independent overlay disks for Worker
    - name: Create Worker Disks
      run: |
        qemu-img create -f qcow2 -b resolute-server-cloudimg-amd64.img -F qcow2 worker-1.qcow2 20G

    # 5. Boot both VMs headlessly in the background with unique MACs and forwarded SSH ports
    - name: Launch Worker VM
      run: |
        # Launch Worker 1 (SSH forwarded to Host Port 2222)
        sudo qemu-system-x86_64 \
          -m 2048 -smp 2 -cpu host -enable-kvm -nographic \
          -drive file=worker-1.qcow2,if=virtio \
          -drive file=seed.img,format=raw,if=virtio \
          -net nic,model=virtio,macaddr=52:54:00:12:34:56 \
          -net user,hostfwd=tcp::2222-:22 \
          &

    # 6. Block the script until both VMs are completely booted and listening on SSH
    - name: Wait for SSH Availability
      run: |
        echo "Waiting for Worker (Port 2222) to respond..."
        timeout 120s bash -c '
          until ssh -i ./id_rsa -p 2222 -o StrictHostKeyChecking=no -o ConnectTimeout=2 ubuntu@127.0.0.1 true 2>/dev/null; do sleep 3; done
        '

        echo "VM is completely up and accessible!"


    - name: Join Worker to K3k Control Plane
      run: |
        K3S_TOKEN=$(kubectl get secret -n k3k-mycluster k3k-mycluster-token -o jsonpath='{.data.token}' | base64 -d)

        echo "Testing connectivity from VM to K3k API server..."
        ssh -i ./id_rsa -p 2222 -o StrictHostKeyChecking=no ubuntu@127.0.0.1 \
          "curl -kv --max-time 10 https://10.0.2.2:30001/readyz || echo 'VM connectivity test failed'"

        echo "Registering Worker..."
        set +e  # Don't exit on error, we want to collect logs
        ssh -i ./id_rsa -p 2222 -o StrictHostKeyChecking=no ubuntu@127.0.0.1 bash -s <<EOSSH
          set -x

          sudo -i

          curl -u "node:${K3S_TOKEN}" https://10.0.2.2:30001/v1-k3s/config
          curl -k -u "node:${K3S_TOKEN}" https://10.0.2.2:30001/v1-k3s/config

          # Run the official registration command
          curl -sfL https://get.k3s.io | K3S_URL=https://10.0.2.2:30001 K3S_TOKEN=${K3S_TOKEN} sh -

          # Check if service started
          if ! sudo systemctl is-active k3s-agent; then
            echo "=== k3s-agent service failed to start ==="
            echo "Service status:"
            sudo systemctl status k3s-agent --no-pager -l || true

            echo ""
            echo "=== Last 100 lines of k3s-agent journal ==="
            sudo journalctl -u k3s-agent -n 100 --no-pager || true

            echo ""
            echo "=== Network connectivity from within VM ==="
            curl -kv --max-time 5 https://10.0.2.2:30001/readyz || echo "Cannot reach API server"

            echo ""
            echo "=== Checking k3s-agent service environment ==="
            sudo cat /etc/systemd/system/k3s-agent.service.env || true

            exit 1
          fi

          echo "✓ K3s agent service started successfully"
          sudo journalctl -u k3s-agent -n 50 --no-pager
        EOSSH
        JOIN_RESULT=$?
        set -e

        if [ $JOIN_RESULT -ne 0 ]; then
          echo "Worker join failed with exit code $JOIN_RESULT"
          exit 1
        fi

    # 8. Assert that both nodes successfully registered and transitioned to a Ready status
    - name: Verify Cluster Nodes
      env:
        KUBECONFIG: ${{ github.workspace }}/k3k-mycluster-mycluster-kubeconfig.yaml
      run: |
        echo "Monitoring K3k Virtual Cluster for Node registration..."

        kubectl get pod -A
        kubectl get nodes

        timeout 180s bash -c '
          until [ $(kubectl get nodes --no-headers 2>/dev/null | grep -c "Ready") -eq 1 ]; do
            echo "Waiting for both worker nodes to show Ready..."
            kubectl get nodes || true
            sleep 5
          done
        '

    - name: Collect logs
      if: always()
      run: |
        journalctl -u k3s -o cat --no-pager > /tmp/k3s.log
        kubectl logs -n k3k-system -l "app.kubernetes.io/name=k3k" --tail=-1 > /tmp/k3k.log

    - name: Archive K3s logs
      uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
      if: always()
      with:
        name: k3s-hcp-logs
        path: /tmp/k3s.log

    - name: Archive K3k logs
      uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
      if: always()
      with:
        name: k3k-hcp-logs
        path: /tmp/k3k.log

  tests:
    runs-on: ubuntu-latest

    steps:
    - name: Checkout code
      uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6

    - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6
      with:
        go-version-file: go.mod

    - name: Run unit tests
      run: make test-unit

    - name: Upload coverage reports to Codecov
      uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v6
      with:
        token: ${{ secrets.CODECOV_TOKEN }}
        files: ./cover.out
        flags: unit

  tests-cli:
    runs-on: ubuntu-latest

    steps:
    - name: Checkout code
      uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
      with:
        fetch-depth: 0
        fetch-tags: true

    - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6
      with:
        go-version-file: go.mod

    - name: Install Ginkgo
      run: go install github.com/onsi/ginkgo/v2/ginkgo@5d1d628ac86668c8f944c8c491c3d1ab86b3bed4 #v2.28.1
    
    - name: Setup environment
      run: |
        mkdir ${{ github.workspace }}/covdata
        
        echo "COVERAGE=true" >> $GITHUB_ENV
        echo "GOCOVERDIR=${{ github.workspace }}/covdata" >> $GITHUB_ENV
        echo "K3S_HOST_VERSION=${{ env.KUBERNETES_VERSION }}+k3s1" >> $GITHUB_ENV
    
    - name: Build and package
      run: |
        make build
        make package

        # add k3kcli to $PATH
        echo "${{ github.workspace }}/bin" >> $GITHUB_PATH

    - name: Check k3kcli
      run: k3kcli -v

    - name: Run cli tests
      env:
        K3K_DOCKER_INSTALL: "true"
        K3S_HOST_VERSION: "${{ env.K3S_HOST_VERSION }}"
      run: make test-cli

    - name: Convert coverage data
      run: go tool covdata textfmt -i=${{ github.workspace }}/covdata -o ${{ github.workspace }}/covdata/cover.out

    - name: Upload coverage reports to Codecov
      uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v6
      with:
        token: ${{ secrets.CODECOV_TOKEN }}
        files: ${{ github.workspace }}/covdata/cover.out
        flags: cli

    - name: Archive k3s logs
      uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
      if: always()
      with:
        name: cli-k3s-logs
        path: /tmp/k3s.log

    - name: Archive k3k logs
      uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
      if: always()
      with:
        name: cli-k3k-logs
        path: /tmp/k3k.log