diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 1846e638..b9a288b9 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -14,8 +14,6 @@ env: KUBERNETES_VERSION: v1.35.3 HELM_VERSION: v4.1.3 HELM_CHECKSUM_AMD64: 02ce9722d541238f81459938b84cf47df2fdf1187493b4bfb2346754d82a4700 - K3D_VERSION: v5.8.3 - K3D_CHECKSUM_AMD64: dbaa79a76ace7f4ca230a1ff41dc7d8a5036a8ad0309e9c54f9bf3836dbe853e jobs: hcp-test: @@ -43,59 +41,31 @@ jobs: rm -fr "${{ env.FILENAME }}" linux-amd64/helm - - name: Install hydrophone - run: go install sigs.k8s.io/hydrophone@3de3e886a2f6f09635d8b981c195490af1584d97 #v0.7.0 - - - name: Install k3d - run: | - curl -sSfL -o k3d "https://github.com/k3d-io/k3d/releases/download/${{ env.K3D_VERSION }}/k3d-linux-amd64" - echo "${{ env.K3D_CHECKSUM_AMD64 }} k3d" | sha256sum --check - sudo install -m 755 k3d /usr/local/bin/k3d - - rm -f k3d - - - name: Install kubectl - run: | - curl -LO "https://dl.k8s.io/release/${{ env.KUBERNETES_VERSION }}/bin/linux/amd64/kubectl" - curl -LO "https://dl.k8s.io/release/${{ env.KUBERNETES_VERSION }}/bin/linux/amd64/kubectl.sha256" - echo "$(cat kubectl.sha256) kubectl" | sha256sum --check - - - name: Setup Kubernetes (k3d) + - name: Install k3s env: - REPO_NAME: k3k-registry - REPO_PORT: 12345 + K3S_HOST_VERSION: ${{ env.KUBERNETES_VERSION }}+k3s1 run: | - echo "127.0.0.1 ${REPO_NAME}" | sudo tee -a /etc/hosts + curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=${K3S_HOST_VERSION} INSTALL_K3S_EXEC="--write-kubeconfig-mode=777" sh -s - + + export KUBECONFIG=/etc/rancher/k3s/k3s.yaml + echo "KUBECONFIG=${KUBECONFIG}" >> $GITHUB_ENV - k3d registry create ${REPO_NAME} --port ${REPO_PORT} - - k3d cluster create k3k --servers 2 \ - --image rancher/k3s:${{ env.KUBERNETES_VERSION }}-k3s1 \ - -p "30000-30010:30000-30010@server:0" \ - --registry-use k3d-${REPO_NAME}:${REPO_PORT} - kubectl cluster-info kubectl get nodes - name: Setup K3k (from source) - env: - REPO: k3k-registry:12345 run: | - echo "127.0.0.1 k3k-registry" | sudo tee -a /etc/hosts + export REPO=ttl.sh/$(uuidgen) + export VERSION=1h make build make package make push + make install # add k3kcli to $PATH echo "${{ github.workspace }}/bin" >> $GITHUB_PATH - VERSION=$(make version) - k3d image import ${REPO}/k3k:${VERSION} -c k3k --verbose - k3d image import ${REPO}/k3k-kubelet:${VERSION} -c k3k --verbose - - make install - - name: Wait for K3k controller run: | echo "Wait for K3k controller deployment to be available" @@ -116,7 +86,6 @@ jobs: namespace: k3k-mycluster spec: mode: hcp - mirrorHostNodes: true tlsSANs: - "127.0.0.1" - "10.0.2.2" @@ -131,11 +100,36 @@ jobs: k3kcli kubeconfig generate --name mycluster export KUBECONFIG=${{ github.workspace }}/k3k-mycluster-mycluster-kubeconfig.yaml - + kubectl cluster-info kubectl get nodes kubectl get pods -A + - name: Wait for cluster to be ready + run: | + echo "Waiting for cluster to reach Ready phase..." + + timeout 300s bash -c ' + until kubectl get cluster -n k3k-mycluster mycluster -o jsonpath="{.status.phase}" | grep -q "Ready"; do + PHASE=$(kubectl get cluster -n k3k-mycluster mycluster -o jsonpath="{.status.phase}") + echo "Current phase: ${PHASE}" + sleep 5 + done + ' + + echo "✓ Cluster is ready!" + + echo "Verifying NodePort service is accessible from host..." + + if ! curl -k --max-time 5 https://127.0.0.1:30001/readyz; then + echo "ERROR: Cannot reach NodePort service from host!" + echo "Service details:" + kubectl get svc -n k3k-mycluster -l cluster=mycluster,role=server -o yaml + exit 1 + fi + + echo "✓ NodePort service is accessible" + - name: Install Virtualization Dependencies run: | sudo apt-get update @@ -146,7 +140,7 @@ jobs: - name: Download Base Cloud Image run: | - wget -q https://cloud-images.ubuntu.com/focal/current/focal-server-cloudimg-amd64.img + wget -q https://cloud-images.ubuntu.com/resolute/current/resolute-server-cloudimg-amd64.img - name: Generate SSH Key and Cloud-Init Seed run: | @@ -169,7 +163,7 @@ jobs: # 4. Provision independent overlay disks for Worker - name: Create Worker Disks run: | - qemu-img create -f qcow2 -b focal-server-cloudimg-amd64.img -F qcow2 worker-1.qcow2 20G + qemu-img create -f qcow2 -b resolute-server-cloudimg-amd64.img -F qcow2 worker-1.qcow2 20G # 5. Boot both VMs headlessly in the background with unique MACs and forwarded SSH ports - name: Launch Worker VM @@ -196,25 +190,94 @@ jobs: - name: Join Worker to K3k Control Plane run: | - echo "Getting cluster token" K3S_TOKEN=$(kubectl get secret -n k3k-mycluster k3k-mycluster-token -o jsonpath='{.data.token}' | base64 -d) - - echo "Registering Worker..." + + echo "Testing connectivity from VM to K3k API server..." ssh -i ./id_rsa -p 2222 -o StrictHostKeyChecking=no ubuntu@127.0.0.1 \ - "curl -sfL https://get.k3s.io | K3S_URL='https://10.0.2.2:6443' K3S_TOKEN='${K3S_TOKEN}' sh -s - agent" + "curl -kv --max-time 10 https://10.0.2.2:30001/readyz || echo 'VM connectivity test failed'" + + echo "Registering Worker..." + set +e # Don't exit on error, we want to collect logs + ssh -i ./id_rsa -p 2222 -o StrictHostKeyChecking=no ubuntu@127.0.0.1 bash -s </dev/null | grep -c "Ready") -eq 2 ]; do + until [ $(kubectl get nodes --no-headers 2>/dev/null | grep -c "Ready") -eq 1 ]; do echo "Waiting for both worker nodes to show Ready..." kubectl get nodes || true sleep 5 done ' - echo "E2E Success: Both QEMU workers successfully attached to the Hosted Control Plane!" + + - name: Collect logs + if: always() + run: | + journalctl -u k3s -o cat --no-pager > /tmp/k3s.log + kubectl logs -n k3k-system -l "app.kubernetes.io/name=k3k" --tail=-1 > /tmp/k3k.log + + - name: Archive K3s logs + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + if: always() + with: + name: k3s-hcp-logs + path: /tmp/k3s.log + + - name: Archive K3k logs + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + if: always() + with: + name: k3k-hcp-logs + path: /tmp/k3k.log tests: runs-on: ubuntu-latest diff --git a/pkg/controller/cluster/cluster.go b/pkg/controller/cluster/cluster.go index dad6447e..91cd7ce5 100644 --- a/pkg/controller/cluster/cluster.go +++ b/pkg/controller/cluster/cluster.go @@ -458,7 +458,7 @@ func (c *ClusterReconciler) reconcile(ctx context.Context, cluster *v1beta1.Clus return err } - if err := c.ensureHCPKubernetesEndpoints(ctx, cluster); err != nil { + if err := c.ensureHCPKubernetesEndpointSlice(ctx, cluster); err != nil { return err } } diff --git a/pkg/controller/cluster/hcp.go b/pkg/controller/cluster/hcp.go index 8d82cd8b..dff7fab8 100644 --- a/pkg/controller/cluster/hcp.go +++ b/pkg/controller/cluster/hcp.go @@ -7,23 +7,18 @@ import ( "net/url" "strconv" - corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + corev1 "k8s.io/api/core/v1" + discoveryv1 "k8s.io/api/discovery/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1" "github.com/rancher/k3k/pkg/controller/cluster/server" ) -// endpointSliceSkipMirrorLabel is the upstream label that opts an Endpoints -// object out of the kube-controller-manager EndpointSlice mirroring controller. -// The kube-apiserver normally sets it on default/kubernetes (because it -// manages EndpointSlices itself); in HCP mode we want the mirror controller -// to handle slices, so we strip the label. -const endpointSliceSkipMirrorLabel = "endpointslice.kubernetes.io/skip-mirror" - // ensureHCPRegistration computes the K3s installer command external nodes can // run to join an HCP-mode cluster and stores it on cluster.Status.HCPRegistration. // @@ -34,7 +29,7 @@ const endpointSliceSkipMirrorLabel = "endpointslice.kubernetes.io/skip-mirror" func (c *ClusterReconciler) ensureHCPRegistration(ctx context.Context, cluster *v1beta1.Cluster, token string) error { log := ctrl.LoggerFrom(ctx) - url, external, err := server.ServerURL(ctx, c.Client, cluster, "", 0) + url, external, err := server.ServerURL(ctx, c.Client, cluster, selectNonLoopbackSAN(cluster), 0) if err != nil { return err } @@ -76,22 +71,55 @@ func hcpRegistrationCommand(version, serverURL, token string) string { version, serverURL, token) } -// ensureHCPKubernetesEndpoints maintains the default/kubernetes Service -// Endpoints inside the virtual cluster, pointing them at the externally +// selectNonLoopbackSAN returns the first non-loopback address from the +// cluster's TLS SANs, preferring spec.TLSSANs then falling back to status.TLSSANs. +// Returns empty string if no non-loopback address is found. +func selectNonLoopbackSAN(cluster *v1beta1.Cluster) string { + // Try spec.TLSSANs first (user-provided values) + for _, san := range cluster.Spec.TLSSANs { + if ip := net.ParseIP(san); ip != nil && ip.IsLoopback() { + continue + } + + if san == "localhost" { + continue + } + + return san // Found a non-loopback address + } + + // Fall back to status.TLSSANs (computed values) + for _, san := range cluster.Status.TLSSANs { + if ip := net.ParseIP(san); ip != nil && ip.IsLoopback() { + continue + } + + if san == "localhost" { + continue + } + + return san + } + + return "" +} + +// ensureHCPKubernetesEndpointSlice maintains the default/kubernetes Service +// EndpointSlice inside the virtual cluster, pointing it at the externally // reachable host:port (NodePort / LoadBalancer / Ingress) so that pods // scheduled on external worker nodes can reach the in-cluster apiserver // ClusterIP. // // Background: the kube-apiserver normally reconciles default/kubernetes -// Endpoints to its own --advertise-address:--secure-port (the host-cluster +// EndpointSlice to its own --advertise-address:--secure-port (the host-cluster // pod IP and 6443). External worker nodes have no route to the host-cluster // pod CIDR, so kube-proxy DNAT to that endpoint fails. We disable the // apiserver reconciler in HCP mode (see serverOptions) and own this -// Endpoints object instead. -func (c *ClusterReconciler) ensureHCPKubernetesEndpoints(ctx context.Context, cluster *v1beta1.Cluster) error { +// EndpointSlice object instead. +func (c *ClusterReconciler) ensureHCPKubernetesEndpointSlice(ctx context.Context, cluster *v1beta1.Cluster) error { log := ctrl.LoggerFrom(ctx) - rawURL, external, err := server.ServerURL(ctx, c.Client, cluster, "", 0) + rawURL, external, err := server.ServerURL(ctx, c.Client, cluster, selectNonLoopbackSAN(cluster), 0) if err != nil { return err } @@ -112,45 +140,65 @@ func (c *ClusterReconciler) ensureHCPKubernetesEndpoints(ctx context.Context, cl return err } + var addressType discoveryv1.AddressType + + if ip := net.ParseIP(addr.IP); ip != nil { + if ip.To4() != nil { + addressType = discoveryv1.AddressTypeIPv4 + } else { + addressType = discoveryv1.AddressTypeIPv6 + } + } else { + return fmt.Errorf("invalid IP address %q", addr.IP) + } + virtClient, err := newVirtualClient(ctx, c.Client, cluster.Name, cluster.Namespace) if err != nil { return fmt.Errorf("creating virtual cluster client: %w", err) } - endpoints := &corev1.Endpoints{ + endpointSlice := &discoveryv1.EndpointSlice{ ObjectMeta: metav1.ObjectMeta{ Name: "kubernetes", Namespace: metav1.NamespaceDefault, }, } - _, err = controllerutil.CreateOrUpdate(ctx, virtClient, endpoints, func() error { - // Allow EndpointSlice mirroring; the apiserver may have set - // skip-mirror=true before we disabled its endpoint reconciler. - if endpoints.Labels != nil { - delete(endpoints.Labels, endpointSliceSkipMirrorLabel) + _, err = controllerutil.CreateOrUpdate(ctx, virtClient, endpointSlice, func() error { + // Ensure the service-name label is set + if endpointSlice.Labels == nil { + endpointSlice.Labels = make(map[string]string) } - endpoints.Subsets = []corev1.EndpointSubset{ + endpointSlice.Labels[discoveryv1.LabelServiceName] = "kubernetes" + endpointSlice.AddressType = addressType + + endpoint := discoveryv1.Endpoint{ + Addresses: []string{addr.IP}, + } + + if addr.Hostname != "" { + endpoint.Hostname = &addr.Hostname + } + + endpointSlice.Endpoints = []discoveryv1.Endpoint{endpoint} + portName := "https" + + endpointSlice.Ports = []discoveryv1.EndpointPort{ { - Addresses: []corev1.EndpointAddress{addr}, - Ports: []corev1.EndpointPort{ - { - Name: "https", - Port: port, - Protocol: corev1.ProtocolTCP, - }, - }, + Name: &portName, + Port: &port, + Protocol: new(corev1.ProtocolTCP), }, } return nil }) if err != nil { - return fmt.Errorf("upserting default/kubernetes endpoints in virtual cluster: %w", err) + return fmt.Errorf("upserting default/kubernetes endpointslice in virtual cluster: %w", err) } - log.V(1).Info("HCP kubernetes endpoints reconciled", + log.V(1).Info("HCP kubernetes endpointslice reconciled", "address", addr.IP, "hostname", addr.Hostname, "port", port) return nil @@ -172,6 +220,7 @@ func parseHCPHostPort(rawURL string) (string, int32, error) { portStr := u.Port() var port int32 = 443 + if portStr != "" { p, err := strconv.Atoi(portStr) if err != nil { @@ -194,6 +243,10 @@ func parseHCPHostPort(rawURL string) (string, int32, error) { // human-readable. func hcpEndpointAddress(host string) (corev1.EndpointAddress, error) { if ip := net.ParseIP(host); ip != nil { + if ip.IsLoopback() { + return corev1.EndpointAddress{}, fmt.Errorf("HCP endpoint host %q is a loopback address and cannot be used", host) + } + return corev1.EndpointAddress{IP: host}, nil } @@ -202,15 +255,21 @@ func hcpEndpointAddress(host string) (corev1.EndpointAddress, error) { return corev1.EndpointAddress{}, fmt.Errorf("HCP endpoint host %q is not an IP and does not resolve: %w", host, err) } + var filteredIPs []net.IP + for _, ip := range ips { - if v4 := ip.To4(); v4 != nil { - return corev1.EndpointAddress{IP: v4.String(), Hostname: host}, nil + if !ip.IsLoopback() { + filteredIPs = append(filteredIPs, ip) } } - if len(ips) == 0 { - return corev1.EndpointAddress{}, fmt.Errorf("HCP endpoint host %q resolved to no IPs", host) + if len(filteredIPs) == 0 { + return corev1.EndpointAddress{}, fmt.Errorf("HCP endpoint host %q resolved to no non-loopback IPs", host) } - return corev1.EndpointAddress{IP: ips[0].String(), Hostname: host}, nil + if v4 := filteredIPs[0].To4(); v4 != nil { + return corev1.EndpointAddress{IP: v4.String(), Hostname: host}, nil + } + + return corev1.EndpointAddress{IP: filteredIPs[0].String(), Hostname: host}, nil } diff --git a/pkg/controller/cluster/hcp_test.go b/pkg/controller/cluster/hcp_test.go index a7edb8a9..6587b0aa 100644 --- a/pkg/controller/cluster/hcp_test.go +++ b/pkg/controller/cluster/hcp_test.go @@ -7,13 +7,12 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client/fake" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/controller-runtime/pkg/client/fake" "github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1" "github.com/rancher/k3k/pkg/controller" @@ -180,18 +179,142 @@ func Test_parseHCPHostPort(t *testing.T) { } } -func Test_hcpEndpointAddress(t *testing.T) { - t.Run("ipv4 literal is passed through", func(t *testing.T) { - got, err := hcpEndpointAddress("10.144.101.195") - require.NoError(t, err) - assert.Equal(t, "10.144.101.195", got.IP) - assert.Empty(t, got.Hostname) - }) +func Test_selectNonLoopbackSAN(t *testing.T) { + tests := []struct { + name string + specSANs []string + statusSANs []string + want string + }{ + { + name: "spec with loopback first, external second", + specSANs: []string{"127.0.0.1", "10.0.0.100"}, + statusSANs: []string{}, + want: "10.0.0.100", + }, + { + name: "spec with external first", + specSANs: []string{"10.0.0.100", "127.0.0.1"}, + statusSANs: []string{}, + want: "10.0.0.100", + }, + { + name: "spec empty, status with loopback first, external second", + specSANs: []string{}, + statusSANs: []string{"127.0.0.1", "10.43.0.50"}, + want: "10.43.0.50", + }, + { + name: "spec with only loopback", + specSANs: []string{"127.0.0.1", "::1"}, + statusSANs: []string{}, + want: "", + }, + { + name: "spec with localhost hostname", + specSANs: []string{"localhost", "example.com"}, + statusSANs: []string{}, + want: "example.com", + }, + { + name: "spec with external hostname", + specSANs: []string{"hcp.example.com"}, + statusSANs: []string{}, + want: "hcp.example.com", + }, + { + name: "empty spec and status", + specSANs: []string{}, + statusSANs: []string{}, + want: "", + }, + { + name: "ipv6 loopback filtered", + specSANs: []string{"::1", "2001:db8::1"}, + statusSANs: []string{}, + want: "2001:db8::1", + }, + } - t.Run("unresolvable hostname errors", func(t *testing.T) { - _, err := hcpEndpointAddress("definitely-not-a-real-host.invalid") - require.Error(t, err) - }) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cluster := &v1beta1.Cluster{ + Spec: v1beta1.ClusterSpec{ + TLSSANs: tt.specSANs, + }, + Status: v1beta1.ClusterStatus{ + TLSSANs: tt.statusSANs, + }, + } + got := selectNonLoopbackSAN(cluster) + assert.Equal(t, tt.want, got) + }) + } +} + +func Test_hcpEndpointAddress(t *testing.T) { + tests := []struct { + name string + input string + wantIP string + wantHostname string + wantErr bool + }{ + { + name: "ipv4 literal is passed through", + input: "10.144.101.195", + wantIP: "10.144.101.195", + wantHostname: "", + wantErr: false, + }, + { + name: "unresolvable hostname errors", + input: "definitely-not-a-real-host.invalid", + wantErr: true, + }, + { + name: "ipv4 loopback literal is rejected", + input: "127.0.0.1", + wantErr: true, + }, + { + name: "ipv6 loopback literal is rejected", + input: "::1", + wantErr: true, + }, + { + name: "ipv4 loopback in range is rejected", + input: "127.0.0.100", + wantErr: true, + }, + { + name: "valid ipv6 literal passes through", + input: "2001:db8::1", + wantIP: "2001:db8::1", + wantHostname: "", + wantErr: false, + }, + { + name: "localhost hostname filters loopbacks", + input: "localhost", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := hcpEndpointAddress(tt.input) + if tt.wantErr { + require.Error(t, err) + return + } + + require.NoError(t, err) + + assert.Equal(t, tt.wantIP, got.IP) + assert.Equal(t, tt.wantHostname, got.Hostname) + }) + } } // Compile-time assertion: every reused exported name from the controller diff --git a/pkg/controller/cluster/server/endpoint.go b/pkg/controller/cluster/server/endpoint.go index 13066304..23962630 100644 --- a/pkg/controller/cluster/server/endpoint.go +++ b/pkg/controller/cluster/server/endpoint.go @@ -6,11 +6,12 @@ import ( "slices" "github.com/sirupsen/logrus" - corev1 "k8s.io/api/core/v1" - networkingv1 "k8s.io/api/networking/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" + corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + "github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1" ) diff --git a/pkg/controller/kubeconfig/kubeconfig.go b/pkg/controller/kubeconfig/kubeconfig.go index 86ecf9e3..9bb4cf8b 100644 --- a/pkg/controller/kubeconfig/kubeconfig.go +++ b/pkg/controller/kubeconfig/kubeconfig.go @@ -86,4 +86,3 @@ func NewConfig(url string, serverCA, clientCert, clientKey []byte) *clientcmdapi return config } -