k3k/.github/workflows/test.yaml

name: Tests

on:
  push:
    branches: [main]
  pull_request:
    types: [opened, synchronize, reopened]
  workflow_dispatch:

permissions:
    contents: read

env:
  KUBERNETES_VERSION: v1.35.3
  HELM_VERSION: v4.1.3
  HELM_CHECKSUM_AMD64: 02ce9722d541238f81459938b84cf47df2fdf1187493b4bfb2346754d82a4700
  K3D_VERSION: v5.8.3
  K3D_CHECKSUM_AMD64: dbaa79a76ace7f4ca230a1ff41dc7d8a5036a8ad0309e9c54f9bf3836dbe853e

jobs:
  hcp-test:
    runs-on: ubuntu-latest

    steps:
    - name: Checkout code
      uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
      with:
        fetch-depth: 0
        fetch-tags: true

    - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6
      with:
        go-version-file: go.mod

    - name: Install helm
      env:
        FILENAME: helm.tar.gz
      run: |
        curl -sSfL -o ${{ env.FILENAME }} https://get.helm.sh/helm-${{ env.HELM_VERSION }}-linux-amd64.tar.gz
        echo "${{ env.HELM_CHECKSUM_AMD64 }}  ${{ env.FILENAME }}" | sha256sum --check
        tar -xvzf ${{ env.FILENAME }} linux-amd64/helm
        sudo install -m 755 linux-amd64/helm /usr/local/bin/helm

        rm -fr "${{ env.FILENAME }}" linux-amd64/helm

    - name: Install hydrophone
      run: go install sigs.k8s.io/hydrophone@3de3e886a2f6f09635d8b981c195490af1584d97 #v0.7.0

    - name: Install k3d
      run: |
        curl -sSfL -o k3d "https://github.com/k3d-io/k3d/releases/download/${{ env.K3D_VERSION }}/k3d-linux-amd64"
        echo "${{ env.K3D_CHECKSUM_AMD64 }}  k3d" | sha256sum --check
        sudo install -m 755 k3d /usr/local/bin/k3d

        rm -f k3d

    - name: Install kubectl
      run: |
        curl -LO "https://dl.k8s.io/release/${{ env.KUBERNETES_VERSION }}/bin/linux/amd64/kubectl"
        curl -LO "https://dl.k8s.io/release/${{ env.KUBERNETES_VERSION }}/bin/linux/amd64/kubectl.sha256"
        echo "$(cat kubectl.sha256)  kubectl" | sha256sum --check

    - name: Setup Kubernetes (k3d)
      env:
        REPO_NAME: k3k-registry
        REPO_PORT: 12345
      run: |
        echo "127.0.0.1 ${REPO_NAME}" | sudo tee -a /etc/hosts

        k3d registry create ${REPO_NAME} --port ${REPO_PORT}
        
        k3d cluster create k3k --servers 2 \
          --image rancher/k3s:${{ env.KUBERNETES_VERSION }}-k3s1 \
          -p "30000-30010:30000-30010@server:0" \
          --registry-use k3d-${REPO_NAME}:${REPO_PORT}
        
        kubectl cluster-info
        kubectl get nodes

    - name: Setup K3k (from source)
      env:
        REPO: k3k-registry:12345
      run: |
        echo "127.0.0.1 k3k-registry" | sudo tee -a /etc/hosts

        make build
        make package
        make push

        # add k3kcli to $PATH
        echo "${{ github.workspace }}/bin" >> $GITHUB_PATH

        VERSION=$(make version)
        k3d image import ${REPO}/k3k:${VERSION} -c k3k --verbose
        k3d image import ${REPO}/k3k-kubelet:${VERSION} -c k3k --verbose

        make install

    - name: Wait for K3k controller
      run: |
        echo "Wait for K3k controller deployment to be available"
        kubectl wait -n k3k-system deployment -l "app.kubernetes.io/name=k3k" --for=condition=Available --timeout=5m

    - name: Check k3kcli
      run: k3kcli -v

    - name: Create virtual cluster
      run: |
        kubectl create namespace k3k-mycluster

        cat <<EOF | kubectl apply -f -
        apiVersion: k3k.io/v1beta1
        kind: Cluster
        metadata:
          name: mycluster
          namespace: k3k-mycluster
        spec:
          mode: hcp
          mirrorHostNodes: true
          tlsSANs:
            - "127.0.0.1"
            - "10.0.2.2"
          expose:
            nodePort:
              serverPort: 30001
        EOF

        echo "Wait for bootstrap secret to be available"
        kubectl wait -n k3k-mycluster --for=create secret k3k-mycluster-bootstrap --timeout=5m

        k3kcli kubeconfig generate --name mycluster

        export KUBECONFIG=${{ github.workspace }}/k3k-mycluster-mycluster-kubeconfig.yaml
        
        kubectl cluster-info
        kubectl get nodes
        kubectl get pods -A

    - name: Install Virtualization Dependencies
      run: |
        sudo apt-get update
        sudo apt-get install -y qemu-kvm qemu-utils cloud-image-utils wget
        sudo usermod -aG kvm $USER

        kvm-ok

    - name: Download Base Cloud Image
      run: |
        wget -q https://cloud-images.ubuntu.com/focal/current/focal-server-cloudimg-amd64.img
        
    - name: Generate SSH Key and Cloud-Init Seed
      run: |
        ssh-keygen -t rsa -b 4096 -f ./id_rsa -N ""
        
        # Define the cloud-init config to authorize our new key
        cat <<EOF > user-data
        #cloud-config
        users:
          - name: ubuntu
            ssh_authorized_keys:
              - $(cat ./id_rsa.pub)
            sudo: ['ALL=(ALL) NOPASSWD:ALL']
            shell: /bin/bash
        EOF
        
        # Bake the configuration into a reusable metadata ISO
        cloud-localds seed.img user-data

    # 4. Provision independent overlay disks for Worker
    - name: Create Worker Disks
      run: |
        qemu-img create -f qcow2 -b focal-server-cloudimg-amd64.img -F qcow2 worker-1.qcow2 20G

    # 5. Boot both VMs headlessly in the background with unique MACs and forwarded SSH ports
    - name: Launch Worker VM
      run: |
        # Launch Worker 1 (SSH forwarded to Host Port 2222)
        sudo qemu-system-x86_64 \
          -m 2048 -smp 2 -cpu host -enable-kvm -nographic \
          -drive file=worker-1.qcow2,if=virtio \
          -drive file=seed.img,format=raw,if=virtio \
          -net nic,model=virtio,macaddr=52:54:00:12:34:56 \
          -net user,hostfwd=tcp::2222-:22 \
          &

    # 6. Block the script until both VMs are completely booted and listening on SSH
    - name: Wait for SSH Availability
      run: |
        echo "Waiting for Worker (Port 2222) to respond..."
        timeout 120s bash -c '
          until ssh -i ./id_rsa -p 2222 -o StrictHostKeyChecking=no -o ConnectTimeout=2 ubuntu@127.0.0.1 true 2>/dev/null; do sleep 3; done
        '

        echo "VM is completely up and accessible!"


    - name: Join Worker to K3k Control Plane
      run: |
        echo "Getting cluster token"
        K3S_TOKEN=$(kubectl get secret -n k3k-mycluster k3k-mycluster-token -o jsonpath='{.data.token}' | base64 -d)
        
        echo "Registering Worker..."
        ssh -i ./id_rsa -p 2222 -o StrictHostKeyChecking=no ubuntu@127.0.0.1 \
          "curl -sfL https://get.k3s.io | K3S_URL='https://10.0.2.2:6443' K3S_TOKEN='${K3S_TOKEN}' sh -s - agent"

    # 8. Assert that both nodes successfully registered and transitioned to a Ready status
    - name: Verify Cluster Nodes
      run: |
        echo "Monitoring K3k Virtual Cluster for Node registration..."
        timeout 180s bash -c '
          until [ $(kubectl get nodes --no-headers 2>/dev/null | grep -c "Ready") -eq 2 ]; do
            echo "Waiting for both worker nodes to show Ready..."
            kubectl get nodes || true
            sleep 5
          done
        '
        echo "E2E Success: Both QEMU workers successfully attached to the Hosted Control Plane!"

  tests:
    runs-on: ubuntu-latest

    steps:
    - name: Checkout code
      uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6

    - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6
      with:
        go-version-file: go.mod

    - name: Run unit tests
      run: make test-unit

    - name: Upload coverage reports to Codecov
      uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v6
      with:
        token: ${{ secrets.CODECOV_TOKEN }}
        files: ./cover.out
        flags: unit

  tests-cli:
    runs-on: ubuntu-latest

    steps:
    - name: Checkout code
      uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
      with:
        fetch-depth: 0
        fetch-tags: true

    - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6
      with:
        go-version-file: go.mod

    - name: Install Ginkgo
      run: go install github.com/onsi/ginkgo/v2/ginkgo@5d1d628ac86668c8f944c8c491c3d1ab86b3bed4 #v2.28.1
    
    - name: Setup environment
      run: |
        mkdir ${{ github.workspace }}/covdata
        
        echo "COVERAGE=true" >> $GITHUB_ENV
        echo "GOCOVERDIR=${{ github.workspace }}/covdata" >> $GITHUB_ENV
        echo "K3S_HOST_VERSION=${{ env.KUBERNETES_VERSION }}+k3s1" >> $GITHUB_ENV
    
    - name: Build and package
      run: |
        make build
        make package

        # add k3kcli to $PATH
        echo "${{ github.workspace }}/bin" >> $GITHUB_PATH

    - name: Check k3kcli
      run: k3kcli -v

    - name: Run cli tests
      env:
        K3K_DOCKER_INSTALL: "true"
        K3S_HOST_VERSION: "${{ env.K3S_HOST_VERSION }}"
      run: make test-cli

    - name: Convert coverage data
      run: go tool covdata textfmt -i=${{ github.workspace }}/covdata -o ${{ github.workspace }}/covdata/cover.out

    - name: Upload coverage reports to Codecov
      uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v6
      with:
        token: ${{ secrets.CODECOV_TOKEN }}
        files: ${{ github.workspace }}/covdata/cover.out
        flags: cli

    - name: Archive k3s logs
      uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
      if: always()
      with:
        name: cli-k3s-logs
        path: /tmp/k3s.log

    - name: Archive k3k logs
      uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
      if: always()
      with:
        name: cli-k3k-logs
        path: /tmp/k3k.log