github/k3k
1
0
Fork 0
mirror of https://github.com/rancher/k3k.git synced 2026-08-19 04:16:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

added HCP conformance tests

Browse Source
This commit is contained in:
Enrico Candino
2026-06-12 17:36:50 +02:00
parent 9c06d44835
commit 51a7ca0107
4 changed files with 421 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/test-conformance-hcp.yaml
+342
View File
@@ -0,0 +1,342 @@
name: Conformance Tests - HCP Mode
on:
schedule:
- cron: "0 1 * * *"
workflow_dispatch:
inputs:
k3k_version:
description: 'K3k version to test (e.g. v1.1.0). Leave empty to build from source.'
required: false
type: string
k8s_version:
description: 'Kubernetes version to test'
required: false
type: choice
options:
- ""
- "v1.34.6"
- "v1.35.3"
permissions:
contents: read
env:
K8S_VERSIONS: "v1.34.6,v1.35.3"
HELM_VERSION: v4.1.3
HELM_CHECKSUM_AMD64: 02ce9722d541238f81459938b84cf47df2fdf1187493b4bfb2346754d82a4700
jobs:
setup:
runs-on: ubuntu-latest
outputs:
k8s_versions: ${{ steps.set-matrix.outputs.k8s_versions }}
steps:
- id: set-matrix
run: |
if [[ -z "${{ inputs.k8s_version }}" ]]; then
JSON_ARRAY=$(jq -nc '"${{ env.K8S_VERSIONS }}" | split(",")')
echo "k8s_versions=${JSON_ARRAY}" >> "$GITHUB_OUTPUT"
else
echo "k8s_versions=[\"${{ inputs.k8s_version }}\"]" >> "$GITHUB_OUTPUT"
fi
conformance:
needs: setup
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
k8s_version: ${{ fromJSON(needs.setup.outputs.k8s_versions) }}
env:
KUBERNETES_VERSION: ${{ matrix.k8s_version }}
steps:
- name: Checkout code
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
with:
fetch-depth: 0
fetch-tags: true
- uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6
with:
go-version-file: go.mod
- name: Install helm
env:
FILENAME: helm.tar.gz
run: |
curl -sSfL -o ${{ env.FILENAME }} https://get.helm.sh/helm-${{ env.HELM_VERSION }}-linux-amd64.tar.gz
echo "${{ env.HELM_CHECKSUM_AMD64 }} ${{ env.FILENAME }}" | sha256sum --check
tar -xvzf ${{ env.FILENAME }} linux-amd64/helm
sudo install -m 755 linux-amd64/helm /usr/local/bin/helm
rm -fr "${{ env.FILENAME }}" linux-amd64/helm
- name: Install hydrophone
run: go install sigs.k8s.io/hydrophone@3de3e886a2f6f09635d8b981c195490af1584d97 #v0.7.0
- name: Install k3s
env:
KUBECONFIG: /etc/rancher/k3s/k3s.yaml
K3S_HOST_VERSION: ${{ env.KUBERNETES_VERSION }}+k3s1
run: |
curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=${K3S_HOST_VERSION} INSTALL_K3S_EXEC="--write-kubeconfig-mode=777" sh -s -
kubectl cluster-info
kubectl get nodes
echo "KUBECONFIG=${KUBECONFIG}" >> $GITHUB_ENV
- name: Setup K3k (from source)
if: inputs.k3k_version == ''
run: |
export REPO=ttl.sh/$(uuidgen)
export VERSION=1h
make build
make package
make push
make install
# add k3kcli to $PATH
echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
- name: Setup K3k (from release)
if: inputs.k3k_version != ''
run: |
K3K_VERSION="${{ inputs.k3k_version }}"
CHART_VERSION="${K3K_VERSION#v}"
helm repo add k3k https://rancher.github.io/k3k
helm repo update
helm install --namespace k3k-system --create-namespace --version "${CHART_VERSION}" k3k k3k/k3k
wget -qO k3kcli "https://github.com/rancher/k3k/releases/download/${{ inputs.k3k_version }}/k3kcli-linux-amd64"
sudo mv k3kcli /usr/local/bin/k3kcli
sudo chmod +x /usr/local/bin/k3kcli
- name: Wait for K3k controller
run: |
echo "Wait for K3k controller deployment to be available"
kubectl wait -n k3k-system deployment -l "app.kubernetes.io/name=k3k" --for=condition=Available --timeout=5m
- name: Check k3kcli
run: k3kcli -v
- name: Create virtual cluster
run: |
kubectl create namespace k3k-mycluster
cat <<EOF | kubectl apply -f -
apiVersion: k3k.io/v1beta1
kind: Cluster
metadata:
name: mycluster
namespace: k3k-mycluster
spec:
mode: hcp
tlsSANs:
- "127.0.0.1"
- "10.0.2.2"
expose:
nodePort:
serverPort: 30001
EOF
echo "Wait for bootstrap secret to be available"
kubectl wait -n k3k-mycluster --for=create secret k3k-mycluster-bootstrap --timeout=5m
k3kcli kubeconfig generate --name mycluster
export KUBECONFIG=${{ github.workspace }}/k3k-mycluster-mycluster-kubeconfig.yaml
kubectl cluster-info
kubectl get nodes
kubectl get pods -A
###################
- name: Install Virtualization Dependencies
run: |
sudo apt-get update
sudo apt-get install -y qemu-kvm qemu-utils cloud-image-utils wget
sudo usermod -aG kvm $USER
kvm-ok
- name: Download Base Cloud Image
run: |
wget -q https://cloud-images.ubuntu.com/resolute/current/resolute-server-cloudimg-amd64.img
- name: Generate SSH Key and Cloud-Init Seed
run: |
ssh-keygen -t rsa -b 4096 -f ./id_rsa -N ""
# Define the cloud-init config to authorize our new key
cat <<EOF > user-data
#cloud-config
users:
- name: ubuntu
ssh_authorized_keys:
- $(cat ./id_rsa.pub)
sudo: ['ALL=(ALL) NOPASSWD:ALL']
shell: /bin/bash
EOF
# Create separate seed images for each worker to avoid locking issues
cloud-localds seed-1.img user-data
cloud-localds seed-2.img user-data
- name: Create Worker Disks
run: |
qemu-img create -f qcow2 -b resolute-server-cloudimg-amd64.img -F qcow2 worker-1.qcow2 20G
qemu-img create -f qcow2 -b resolute-server-cloudimg-amd64.img -F qcow2 worker-2.qcow2 20G
- name: Launch Worker VMs
run: |
# Launch Worker 1
sudo qemu-system-x86_64 \
-m 2048 -smp 2 -cpu host -enable-kvm -nographic \
-drive file=worker-1.qcow2,if=virtio \
-drive file=seed-1.img,format=raw,if=virtio \
-net nic,model=virtio,macaddr=52:54:00:12:34:56 \
-net user,hostfwd=tcp::2222-:22 \
&
# Wait a moment before launching the second VM
sleep 5
# Launch Worker 2
sudo qemu-system-x86_64 \
-m 2048 -smp 2 -cpu host -enable-kvm -nographic \
-drive file=worker-2.qcow2,if=virtio \
-drive file=seed-2.img,format=raw,if=virtio \
-net nic,model=virtio,macaddr=52:54:00:12:34:57 \
-net user,hostfwd=tcp::2223-:22 \
&
- name: Wait for SSH Availability
run: |
echo "Waiting for Worker 1 (Port 2222) to respond..."
timeout 120s bash -c '
until ssh -i ./id_rsa -p 2222 -o StrictHostKeyChecking=no -o ConnectTimeout=2 ubuntu@127.0.0.1 true 2>/dev/null; do sleep 3; done
'
echo "Waiting for Worker 2 (Port 2223) to respond..."
timeout 120s bash -c '
until ssh -i ./id_rsa -p 2223 -o StrictHostKeyChecking=no -o ConnectTimeout=2 ubuntu@127.0.0.1 true 2>/dev/null; do sleep 3; done
'
echo "Both VMs are up and running!"
echo "Testing connectivity from VMs to K3k API server..."
ssh -i ./id_rsa -p 2222 -o StrictHostKeyChecking=no ubuntu@127.0.0.1 \
"curl -kv --max-time 10 https://10.0.2.2:30001/readyz || echo 'Worker 1 connectivity test failed'"
ssh -i ./id_rsa -p 2223 -o StrictHostKeyChecking=no ubuntu@127.0.0.1 \
"curl -kv --max-time 10 https://10.0.2.2:30001/readyz || echo 'Worker 2 connectivity test failed'"
######################
- name: Join Workers to K3k Control Plane
run: |
K3S_TOKEN=$(kubectl get secret -n k3k-mycluster k3k-mycluster-token -o jsonpath='{.data.token}' | base64 -d)
echo "Registering Worker 1..."
ssh -i ./id_rsa -p 2222 -o StrictHostKeyChecking=no ubuntu@127.0.0.1 \
"curl -sfL https://get.k3s.io | K3S_URL=https://10.0.2.2:30001 K3S_TOKEN=${K3S_TOKEN} sh - &" &
WORKER1_PID=$!
echo "Registering Worker 2..."
ssh -i ./id_rsa -p 2223 -o StrictHostKeyChecking=no ubuntu@127.0.0.1 \
"curl -sfL https://get.k3s.io | K3S_URL=https://10.0.2.2:30001 K3S_TOKEN=${K3S_TOKEN} sh - &" &
WORKER2_PID=$!
echo "Waiting for both workers to complete registration..."
wait $WORKER1_PID
wait $WORKER2_PID
echo "Both workers registration initiated"
- name: Verify Cluster Nodes
env:
KUBECONFIG: ${{ github.workspace }}/k3k-mycluster-mycluster-kubeconfig.yaml
run: |
echo "Monitoring K3k Virtual Cluster for Node registration..."
kubectl get pod -A
kubectl get nodes
timeout 180s bash -c '
until [ $(kubectl get nodes --no-headers 2>/dev/null | grep -c "Ready") -eq 2 ]; do
echo "Waiting for both worker nodes to show Ready..."
kubectl get nodes || true
sleep 5
done
'
######################
- name: Run conformance tests
run: |
hydrophone --conformance --parallel 4 \
--kubeconfig ${{ github.workspace }}/k3k-mycluster-mycluster-kubeconfig.yaml \
--output-dir /tmp
- name: Collect logs
if: always()
env:
KUBECONFIG: /etc/rancher/k3s/k3s.yaml
run: |
journalctl -u k3s -o cat --no-pager > /tmp/k3s.log
kubectl logs -n k3k-system -l "app.kubernetes.io/name=k3k" --tail=-1 > /tmp/k3k.log
- name: Archive K3s logs
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
if: always()
with:
name: k3s-${{ matrix.k8s_version }}-logs
path: /tmp/k3s.log
- name: Archive K3k logs
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
if: always()
with:
name: k3k-${{ matrix.k8s_version }}-logs
path: /tmp/k3k.log
- name: Archive conformance logs
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
if: always()
with:
name: conformance-${{ matrix.k8s_version }}-logs
path: /tmp/e2e.log
- name: Archive results
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
if: always()
with:
name: conformance-${{ matrix.k8s_version }}-results
path: /tmp/junit_01.xml
- name: Job Summary
if: always()
run: |
echo '## 📊 Conformance Tests Results (${{ matrix.k8s_version }})' >> $GITHUB_STEP_SUMMARY
echo '| Passed | Failed | Pending | Skipped |' >> $GITHUB_STEP_SUMMARY
echo '|---|---|---|---|' >> $GITHUB_STEP_SUMMARY
RESULTS=$(tail -10 /tmp/e2e.log | grep -E "Passed .* Failed .* Pending .* Skipped" | cut -d '-' -f 3)
RESULTS=$(echo $RESULTS | grep -oE '[0-9]+' | xargs | sed 's/ / | /g')
echo "| $RESULTS |" >> $GITHUB_STEP_SUMMARY
# only include failed tests section if there are any
if grep -q '\[FAIL\]' /tmp/e2e.log; then
echo '' >> $GITHUB_STEP_SUMMARY
echo '### Failed Tests' >> $GITHUB_STEP_SUMMARY
echo '```' >> $GITHUB_STEP_SUMMARY
grep '\[FAIL\]' /tmp/e2e.log >> $GITHUB_STEP_SUMMARY
echo '```' >> $GITHUB_STEP_SUMMARY
fi
pkg/controller/cluster/cluster.go
+5 -1
View File
@@ -454,13 +454,17 @@ func (c *ClusterReconciler) reconcile(ctx context.Context, cluster *v1beta1.Clus
// reconciler is disabled for HCP) so external-node pods can reach the
// in-cluster apiserver ClusterIP.
if cluster.Spec.Mode == v1beta1.HCPClusterMode {
if err := c.ensureHCPRegistration(ctx, cluster, token); err != nil {
if err := c.ensureHCPRegistration(ctx, cluster); err != nil {
return err
}
if err := c.ensureHCPKubernetesEndpointSlice(ctx, cluster); err != nil {
return err
}
if err := c.ensureHCPKubernetesEndpoints(ctx, cluster); err != nil {
return err
}
}
// Important: if you need to call the Server API of the Virtual Cluster
pkg/controller/cluster/hcp.go
+73 -2
View File
@@ -26,7 +26,7 @@ import (
// LoadBalancer or Ingress configured) the command cannot be built; in that
// case the Ready condition is set to False with reason HCPNoExternalEndpoint
// so the operator surfaces the problem without failing the reconciliation.
func (c *ClusterReconciler) ensureHCPRegistration(ctx context.Context, cluster *v1beta1.Cluster, token string) error {
func (c *ClusterReconciler) ensureHCPRegistration(ctx context.Context, cluster *v1beta1.Cluster) error {
log := ctrl.LoggerFrom(ctx)
_, external, err := server.ServerURL(ctx, c.Client, cluster, selectNonLoopbackSAN(cluster), 0)
@@ -143,11 +143,11 @@ func (c *ClusterReconciler) ensureHCPKubernetesEndpointSlice(ctx context.Context
}
_, err = controllerutil.CreateOrUpdate(ctx, virtClient, endpointSlice, func() error {
// Ensure the service-name label is set
if endpointSlice.Labels == nil {
endpointSlice.Labels = make(map[string]string)
}
// Ensure the service-name label is set
endpointSlice.Labels[discoveryv1.LabelServiceName] = "kubernetes"
endpointSlice.AddressType = addressType
@@ -182,6 +182,77 @@ func (c *ClusterReconciler) ensureHCPKubernetesEndpointSlice(ctx context.Context
return nil
}
func (c *ClusterReconciler) ensureHCPKubernetesEndpoints(ctx context.Context, cluster *v1beta1.Cluster) error {
log := ctrl.LoggerFrom(ctx)
rawURL, external, err := server.ServerURL(ctx, c.Client, cluster, selectNonLoopbackSAN(cluster), 0)
if err != nil {
return err
}
if !external {
// ensureHCPRegistration already surfaces this via Ready=False;
// nothing for us to do here.
return nil
}
host, port, err := parseHCPHostPort(rawURL)
if err != nil {
return fmt.Errorf("parsing HCP server URL %q: %w", rawURL, err)
}
addr, err := hcpEndpointAddress(host)
if err != nil {
return err
}
virtClient, err := newVirtualClient(ctx, c.Client, cluster.Name, cluster.Namespace)
if err != nil {
return fmt.Errorf("creating virtual cluster client: %w", err)
}
//nolint:staticcheck // SA1019 corev1.Endpoints is deprecated in v1.33+, but needed in the Conformance tests
// We are already using the discoveryv1.EndpointSlice
endpoints := &corev1.Endpoints{
ObjectMeta: metav1.ObjectMeta{
Name: "kubernetes",
Namespace: metav1.NamespaceDefault,
},
}
_, err = controllerutil.CreateOrUpdate(ctx, virtClient, endpoints, func() error {
if endpoints.Labels == nil {
endpoints.Labels = make(map[string]string)
}
// Ensure the skip-mirror label is set
endpoints.Labels[discoveryv1.LabelSkipMirror] = "true"
//nolint:staticcheck // SA1019 corev1.EndpointSubset is deprecated in v1.33+, but needed in the Conformance tests
endpoints.Subsets = []corev1.EndpointSubset{
{
Addresses: []corev1.EndpointAddress{addr},
Ports: []corev1.EndpointPort{
{
Name: "https",
Port: port,
Protocol: corev1.ProtocolTCP,
},
},
},
}
return nil
})
if err != nil {
return fmt.Errorf("upserting default/kubernetes endpoints in virtual cluster: %w", err)
}
log.V(1).Info("HCP kubernetes endpoints reconciled", "address", addr.IP, "hostname", addr.Hostname, "port", port)
return nil
}
// parseHCPHostPort extracts the host and port from a server URL produced by
// server.ServerURL. The port defaults to 443 when omitted.
func parseHCPHostPort(rawURL string) (string, int32, error) {
pkg/controller/cluster/hcp_test.go
+1 -1
View File
@@ -54,7 +54,7 @@ func Test_ensureHCPRegistration(t *testing.T) {
r := &ClusterReconciler{Client: fakeClient}
c := cluster.DeepCopy()
require.NoError(t, r.ensureHCPRegistration(context.Background(), c, "ignored"))
require.NoError(t, r.ensureHCPRegistration(context.Background(), c))
cond := meta.FindStatusCondition(c.Status.Conditions, ConditionReady)
require.NotNil(t, cond)