diff --git a/.github/workflows/test-conformance-hcp.yaml b/.github/workflows/test-conformance-hcp.yaml new file mode 100644 index 00000000..b2c8caf5 --- /dev/null +++ b/.github/workflows/test-conformance-hcp.yaml @@ -0,0 +1,342 @@ +name: Conformance Tests - HCP Mode + +on: + schedule: + - cron: "0 1 * * *" + workflow_dispatch: + inputs: + k3k_version: + description: 'K3k version to test (e.g. v1.1.0). Leave empty to build from source.' + required: false + type: string + k8s_version: + description: 'Kubernetes version to test' + required: false + type: choice + options: + - "" + - "v1.34.6" + - "v1.35.3" + +permissions: + contents: read + +env: + K8S_VERSIONS: "v1.34.6,v1.35.3" + HELM_VERSION: v4.1.3 + HELM_CHECKSUM_AMD64: 02ce9722d541238f81459938b84cf47df2fdf1187493b4bfb2346754d82a4700 + +jobs: + setup: + runs-on: ubuntu-latest + outputs: + k8s_versions: ${{ steps.set-matrix.outputs.k8s_versions }} + steps: + - id: set-matrix + run: | + if [[ -z "${{ inputs.k8s_version }}" ]]; then + JSON_ARRAY=$(jq -nc '"${{ env.K8S_VERSIONS }}" | split(",")') + echo "k8s_versions=${JSON_ARRAY}" >> "$GITHUB_OUTPUT" + else + echo "k8s_versions=[\"${{ inputs.k8s_version }}\"]" >> "$GITHUB_OUTPUT" + fi + + conformance: + needs: setup + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + k8s_version: ${{ fromJSON(needs.setup.outputs.k8s_versions) }} + + env: + KUBERNETES_VERSION: ${{ matrix.k8s_version }} + + steps: + - name: Checkout code + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + with: + fetch-depth: 0 + fetch-tags: true + + - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6 + with: + go-version-file: go.mod + + - name: Install helm + env: + FILENAME: helm.tar.gz + run: | + curl -sSfL -o ${{ env.FILENAME }} https://get.helm.sh/helm-${{ env.HELM_VERSION }}-linux-amd64.tar.gz + echo "${{ env.HELM_CHECKSUM_AMD64 }} ${{ env.FILENAME }}" | sha256sum --check + tar -xvzf ${{ env.FILENAME }} linux-amd64/helm + sudo install -m 755 linux-amd64/helm /usr/local/bin/helm + + rm -fr "${{ env.FILENAME }}" linux-amd64/helm + + - name: Install hydrophone + run: go install sigs.k8s.io/hydrophone@3de3e886a2f6f09635d8b981c195490af1584d97 #v0.7.0 + + - name: Install k3s + env: + KUBECONFIG: /etc/rancher/k3s/k3s.yaml + K3S_HOST_VERSION: ${{ env.KUBERNETES_VERSION }}+k3s1 + run: | + curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=${K3S_HOST_VERSION} INSTALL_K3S_EXEC="--write-kubeconfig-mode=777" sh -s - + + kubectl cluster-info + kubectl get nodes + + echo "KUBECONFIG=${KUBECONFIG}" >> $GITHUB_ENV + + - name: Setup K3k (from source) + if: inputs.k3k_version == '' + run: | + export REPO=ttl.sh/$(uuidgen) + export VERSION=1h + + make build + make package + make push + make install + + # add k3kcli to $PATH + echo "${{ github.workspace }}/bin" >> $GITHUB_PATH + + - name: Setup K3k (from release) + if: inputs.k3k_version != '' + run: | + K3K_VERSION="${{ inputs.k3k_version }}" + CHART_VERSION="${K3K_VERSION#v}" + + helm repo add k3k https://rancher.github.io/k3k + helm repo update + helm install --namespace k3k-system --create-namespace --version "${CHART_VERSION}" k3k k3k/k3k + + wget -qO k3kcli "https://github.com/rancher/k3k/releases/download/${{ inputs.k3k_version }}/k3kcli-linux-amd64" + sudo mv k3kcli /usr/local/bin/k3kcli + sudo chmod +x /usr/local/bin/k3kcli + + - name: Wait for K3k controller + run: | + echo "Wait for K3k controller deployment to be available" + kubectl wait -n k3k-system deployment -l "app.kubernetes.io/name=k3k" --for=condition=Available --timeout=5m + + - name: Check k3kcli + run: k3kcli -v + + - name: Create virtual cluster + run: | + kubectl create namespace k3k-mycluster + + cat < user-data + #cloud-config + users: + - name: ubuntu + ssh_authorized_keys: + - $(cat ./id_rsa.pub) + sudo: ['ALL=(ALL) NOPASSWD:ALL'] + shell: /bin/bash + EOF + + # Create separate seed images for each worker to avoid locking issues + cloud-localds seed-1.img user-data + cloud-localds seed-2.img user-data + + - name: Create Worker Disks + run: | + qemu-img create -f qcow2 -b resolute-server-cloudimg-amd64.img -F qcow2 worker-1.qcow2 20G + qemu-img create -f qcow2 -b resolute-server-cloudimg-amd64.img -F qcow2 worker-2.qcow2 20G + + - name: Launch Worker VMs + run: | + # Launch Worker 1 + sudo qemu-system-x86_64 \ + -m 2048 -smp 2 -cpu host -enable-kvm -nographic \ + -drive file=worker-1.qcow2,if=virtio \ + -drive file=seed-1.img,format=raw,if=virtio \ + -net nic,model=virtio,macaddr=52:54:00:12:34:56 \ + -net user,hostfwd=tcp::2222-:22 \ + & + + # Wait a moment before launching the second VM + sleep 5 + + # Launch Worker 2 + sudo qemu-system-x86_64 \ + -m 2048 -smp 2 -cpu host -enable-kvm -nographic \ + -drive file=worker-2.qcow2,if=virtio \ + -drive file=seed-2.img,format=raw,if=virtio \ + -net nic,model=virtio,macaddr=52:54:00:12:34:57 \ + -net user,hostfwd=tcp::2223-:22 \ + & + + - name: Wait for SSH Availability + run: | + echo "Waiting for Worker 1 (Port 2222) to respond..." + timeout 120s bash -c ' + until ssh -i ./id_rsa -p 2222 -o StrictHostKeyChecking=no -o ConnectTimeout=2 ubuntu@127.0.0.1 true 2>/dev/null; do sleep 3; done + ' + + echo "Waiting for Worker 2 (Port 2223) to respond..." + timeout 120s bash -c ' + until ssh -i ./id_rsa -p 2223 -o StrictHostKeyChecking=no -o ConnectTimeout=2 ubuntu@127.0.0.1 true 2>/dev/null; do sleep 3; done + ' + + echo "Both VMs are up and running!" + + echo "Testing connectivity from VMs to K3k API server..." + ssh -i ./id_rsa -p 2222 -o StrictHostKeyChecking=no ubuntu@127.0.0.1 \ + "curl -kv --max-time 10 https://10.0.2.2:30001/readyz || echo 'Worker 1 connectivity test failed'" + + ssh -i ./id_rsa -p 2223 -o StrictHostKeyChecking=no ubuntu@127.0.0.1 \ + "curl -kv --max-time 10 https://10.0.2.2:30001/readyz || echo 'Worker 2 connectivity test failed'" + + ###################### + + - name: Join Workers to K3k Control Plane + run: | + K3S_TOKEN=$(kubectl get secret -n k3k-mycluster k3k-mycluster-token -o jsonpath='{.data.token}' | base64 -d) + + echo "Registering Worker 1..." + ssh -i ./id_rsa -p 2222 -o StrictHostKeyChecking=no ubuntu@127.0.0.1 \ + "curl -sfL https://get.k3s.io | K3S_URL=https://10.0.2.2:30001 K3S_TOKEN=${K3S_TOKEN} sh - &" & + WORKER1_PID=$! + + echo "Registering Worker 2..." + ssh -i ./id_rsa -p 2223 -o StrictHostKeyChecking=no ubuntu@127.0.0.1 \ + "curl -sfL https://get.k3s.io | K3S_URL=https://10.0.2.2:30001 K3S_TOKEN=${K3S_TOKEN} sh - &" & + WORKER2_PID=$! + + echo "Waiting for both workers to complete registration..." + wait $WORKER1_PID + wait $WORKER2_PID + echo "Both workers registration initiated" + + - name: Verify Cluster Nodes + env: + KUBECONFIG: ${{ github.workspace }}/k3k-mycluster-mycluster-kubeconfig.yaml + run: | + echo "Monitoring K3k Virtual Cluster for Node registration..." + + kubectl get pod -A + kubectl get nodes + + timeout 180s bash -c ' + until [ $(kubectl get nodes --no-headers 2>/dev/null | grep -c "Ready") -eq 2 ]; do + echo "Waiting for both worker nodes to show Ready..." + kubectl get nodes || true + sleep 5 + done + ' + + ###################### + + - name: Run conformance tests + run: | + hydrophone --conformance --parallel 4 \ + --kubeconfig ${{ github.workspace }}/k3k-mycluster-mycluster-kubeconfig.yaml \ + --output-dir /tmp + + - name: Collect logs + if: always() + env: + KUBECONFIG: /etc/rancher/k3s/k3s.yaml + run: | + journalctl -u k3s -o cat --no-pager > /tmp/k3s.log + kubectl logs -n k3k-system -l "app.kubernetes.io/name=k3k" --tail=-1 > /tmp/k3k.log + + - name: Archive K3s logs + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + if: always() + with: + name: k3s-${{ matrix.k8s_version }}-logs + path: /tmp/k3s.log + + - name: Archive K3k logs + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + if: always() + with: + name: k3k-${{ matrix.k8s_version }}-logs + path: /tmp/k3k.log + + - name: Archive conformance logs + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + if: always() + with: + name: conformance-${{ matrix.k8s_version }}-logs + path: /tmp/e2e.log + + - name: Archive results + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + if: always() + with: + name: conformance-${{ matrix.k8s_version }}-results + path: /tmp/junit_01.xml + + - name: Job Summary + if: always() + run: | + echo '## 📊 Conformance Tests Results (${{ matrix.k8s_version }})' >> $GITHUB_STEP_SUMMARY + echo '| Passed | Failed | Pending | Skipped |' >> $GITHUB_STEP_SUMMARY + echo '|---|---|---|---|' >> $GITHUB_STEP_SUMMARY + + RESULTS=$(tail -10 /tmp/e2e.log | grep -E "Passed .* Failed .* Pending .* Skipped" | cut -d '-' -f 3) + RESULTS=$(echo $RESULTS | grep -oE '[0-9]+' | xargs | sed 's/ / | /g') + echo "| $RESULTS |" >> $GITHUB_STEP_SUMMARY + + # only include failed tests section if there are any + if grep -q '\[FAIL\]' /tmp/e2e.log; then + echo '' >> $GITHUB_STEP_SUMMARY + echo '### Failed Tests' >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + grep '\[FAIL\]' /tmp/e2e.log >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + fi diff --git a/pkg/controller/cluster/cluster.go b/pkg/controller/cluster/cluster.go index 91cd7ce5..9a4601fd 100644 --- a/pkg/controller/cluster/cluster.go +++ b/pkg/controller/cluster/cluster.go @@ -454,13 +454,17 @@ func (c *ClusterReconciler) reconcile(ctx context.Context, cluster *v1beta1.Clus // reconciler is disabled for HCP) so external-node pods can reach the // in-cluster apiserver ClusterIP. if cluster.Spec.Mode == v1beta1.HCPClusterMode { - if err := c.ensureHCPRegistration(ctx, cluster, token); err != nil { + if err := c.ensureHCPRegistration(ctx, cluster); err != nil { return err } if err := c.ensureHCPKubernetesEndpointSlice(ctx, cluster); err != nil { return err } + + if err := c.ensureHCPKubernetesEndpoints(ctx, cluster); err != nil { + return err + } } // Important: if you need to call the Server API of the Virtual Cluster diff --git a/pkg/controller/cluster/hcp.go b/pkg/controller/cluster/hcp.go index 0c5ce04b..127c5e0c 100644 --- a/pkg/controller/cluster/hcp.go +++ b/pkg/controller/cluster/hcp.go @@ -26,7 +26,7 @@ import ( // LoadBalancer or Ingress configured) the command cannot be built; in that // case the Ready condition is set to False with reason HCPNoExternalEndpoint // so the operator surfaces the problem without failing the reconciliation. -func (c *ClusterReconciler) ensureHCPRegistration(ctx context.Context, cluster *v1beta1.Cluster, token string) error { +func (c *ClusterReconciler) ensureHCPRegistration(ctx context.Context, cluster *v1beta1.Cluster) error { log := ctrl.LoggerFrom(ctx) _, external, err := server.ServerURL(ctx, c.Client, cluster, selectNonLoopbackSAN(cluster), 0) @@ -143,11 +143,11 @@ func (c *ClusterReconciler) ensureHCPKubernetesEndpointSlice(ctx context.Context } _, err = controllerutil.CreateOrUpdate(ctx, virtClient, endpointSlice, func() error { - // Ensure the service-name label is set if endpointSlice.Labels == nil { endpointSlice.Labels = make(map[string]string) } + // Ensure the service-name label is set endpointSlice.Labels[discoveryv1.LabelServiceName] = "kubernetes" endpointSlice.AddressType = addressType @@ -182,6 +182,77 @@ func (c *ClusterReconciler) ensureHCPKubernetesEndpointSlice(ctx context.Context return nil } +func (c *ClusterReconciler) ensureHCPKubernetesEndpoints(ctx context.Context, cluster *v1beta1.Cluster) error { + log := ctrl.LoggerFrom(ctx) + + rawURL, external, err := server.ServerURL(ctx, c.Client, cluster, selectNonLoopbackSAN(cluster), 0) + if err != nil { + return err + } + + if !external { + // ensureHCPRegistration already surfaces this via Ready=False; + // nothing for us to do here. + return nil + } + + host, port, err := parseHCPHostPort(rawURL) + if err != nil { + return fmt.Errorf("parsing HCP server URL %q: %w", rawURL, err) + } + + addr, err := hcpEndpointAddress(host) + if err != nil { + return err + } + + virtClient, err := newVirtualClient(ctx, c.Client, cluster.Name, cluster.Namespace) + if err != nil { + return fmt.Errorf("creating virtual cluster client: %w", err) + } + + //nolint:staticcheck // SA1019 corev1.Endpoints is deprecated in v1.33+, but needed in the Conformance tests + // We are already using the discoveryv1.EndpointSlice + endpoints := &corev1.Endpoints{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kubernetes", + Namespace: metav1.NamespaceDefault, + }, + } + + _, err = controllerutil.CreateOrUpdate(ctx, virtClient, endpoints, func() error { + if endpoints.Labels == nil { + endpoints.Labels = make(map[string]string) + } + + // Ensure the skip-mirror label is set + endpoints.Labels[discoveryv1.LabelSkipMirror] = "true" + + //nolint:staticcheck // SA1019 corev1.EndpointSubset is deprecated in v1.33+, but needed in the Conformance tests + endpoints.Subsets = []corev1.EndpointSubset{ + { + Addresses: []corev1.EndpointAddress{addr}, + Ports: []corev1.EndpointPort{ + { + Name: "https", + Port: port, + Protocol: corev1.ProtocolTCP, + }, + }, + }, + } + + return nil + }) + if err != nil { + return fmt.Errorf("upserting default/kubernetes endpoints in virtual cluster: %w", err) + } + + log.V(1).Info("HCP kubernetes endpoints reconciled", "address", addr.IP, "hostname", addr.Hostname, "port", port) + + return nil +} + // parseHCPHostPort extracts the host and port from a server URL produced by // server.ServerURL. The port defaults to 443 when omitted. func parseHCPHostPort(rawURL string) (string, int32, error) { diff --git a/pkg/controller/cluster/hcp_test.go b/pkg/controller/cluster/hcp_test.go index da1b5f57..c85ef986 100644 --- a/pkg/controller/cluster/hcp_test.go +++ b/pkg/controller/cluster/hcp_test.go @@ -54,7 +54,7 @@ func Test_ensureHCPRegistration(t *testing.T) { r := &ClusterReconciler{Client: fakeClient} c := cluster.DeepCopy() - require.NoError(t, r.ensureHCPRegistration(context.Background(), c, "ignored")) + require.NoError(t, r.ensureHCPRegistration(context.Background(), c)) cond := meta.FindStatusCondition(c.Status.Conditions, ConditionReady) require.NotNil(t, cond)