github/k3k
1
0
Fork 0
mirror of https://github.com/rancher/k3k.git synced 2026-08-18 20:07:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

removed k3d

Browse Source
This commit is contained in:
Enrico Candino
2026-06-12 17:36:49 +02:00
parent aa46e5140b
commit 898b2f34f0
6 changed files with 349 additions and 104 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/test.yaml
+112 -49
View File
@@ -14,8 +14,6 @@ env:
KUBERNETES_VERSION: v1.35.3
HELM_VERSION: v4.1.3
HELM_CHECKSUM_AMD64: 02ce9722d541238f81459938b84cf47df2fdf1187493b4bfb2346754d82a4700
K3D_VERSION: v5.8.3
K3D_CHECKSUM_AMD64: dbaa79a76ace7f4ca230a1ff41dc7d8a5036a8ad0309e9c54f9bf3836dbe853e
jobs:
hcp-test:
@@ -43,59 +41,31 @@ jobs:
rm -fr "${{ env.FILENAME }}" linux-amd64/helm
- name: Install hydrophone
run: go install sigs.k8s.io/hydrophone@3de3e886a2f6f09635d8b981c195490af1584d97 #v0.7.0
- name: Install k3d
run: |
curl -sSfL -o k3d "https://github.com/k3d-io/k3d/releases/download/${{ env.K3D_VERSION }}/k3d-linux-amd64"
echo "${{ env.K3D_CHECKSUM_AMD64 }} k3d" | sha256sum --check
sudo install -m 755 k3d /usr/local/bin/k3d
rm -f k3d
- name: Install kubectl
run: |
curl -LO "https://dl.k8s.io/release/${{ env.KUBERNETES_VERSION }}/bin/linux/amd64/kubectl"
curl -LO "https://dl.k8s.io/release/${{ env.KUBERNETES_VERSION }}/bin/linux/amd64/kubectl.sha256"
echo "$(cat kubectl.sha256) kubectl" | sha256sum --check
- name: Setup Kubernetes (k3d)
- name: Install k3s
env:
REPO_NAME: k3k-registry
REPO_PORT: 12345
K3S_HOST_VERSION: ${{ env.KUBERNETES_VERSION }}+k3s1
run: |
echo "127.0.0.1 ${REPO_NAME}" | sudo tee -a /etc/hosts
curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=${K3S_HOST_VERSION} INSTALL_K3S_EXEC="--write-kubeconfig-mode=777" sh -s -
export KUBECONFIG=/etc/rancher/k3s/k3s.yaml
echo "KUBECONFIG=${KUBECONFIG}" >> $GITHUB_ENV
k3d registry create ${REPO_NAME} --port ${REPO_PORT}
k3d cluster create k3k --servers 2 \
--image rancher/k3s:${{ env.KUBERNETES_VERSION }}-k3s1 \
-p "30000-30010:30000-30010@server:0" \
--registry-use k3d-${REPO_NAME}:${REPO_PORT}
kubectl cluster-info
kubectl get nodes
- name: Setup K3k (from source)
env:
REPO: k3k-registry:12345
run: |
echo "127.0.0.1 k3k-registry" | sudo tee -a /etc/hosts
export REPO=ttl.sh/$(uuidgen)
export VERSION=1h
make build
make package
make push
make install
# add k3kcli to $PATH
echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
VERSION=$(make version)
k3d image import ${REPO}/k3k:${VERSION} -c k3k --verbose
k3d image import ${REPO}/k3k-kubelet:${VERSION} -c k3k --verbose
make install
- name: Wait for K3k controller
run: |
echo "Wait for K3k controller deployment to be available"
@@ -116,7 +86,6 @@ jobs:
namespace: k3k-mycluster
spec:
mode: hcp
mirrorHostNodes: true
tlsSANs:
- "127.0.0.1"
- "10.0.2.2"
@@ -131,11 +100,36 @@ jobs:
k3kcli kubeconfig generate --name mycluster
export KUBECONFIG=${{ github.workspace }}/k3k-mycluster-mycluster-kubeconfig.yaml
kubectl cluster-info
kubectl get nodes
kubectl get pods -A
- name: Wait for cluster to be ready
run: |
echo "Waiting for cluster to reach Ready phase..."
timeout 300s bash -c '
until kubectl get cluster -n k3k-mycluster mycluster -o jsonpath="{.status.phase}" | grep -q "Ready"; do
PHASE=$(kubectl get cluster -n k3k-mycluster mycluster -o jsonpath="{.status.phase}")
echo "Current phase: ${PHASE}"
sleep 5
done
'
echo "✓ Cluster is ready!"
echo "Verifying NodePort service is accessible from host..."
if ! curl -k --max-time 5 https://127.0.0.1:30001/readyz; then
echo "ERROR: Cannot reach NodePort service from host!"
echo "Service details:"
kubectl get svc -n k3k-mycluster -l cluster=mycluster,role=server -o yaml
exit 1
fi
echo "✓ NodePort service is accessible"
- name: Install Virtualization Dependencies
run: |
sudo apt-get update
@@ -146,7 +140,7 @@ jobs:
- name: Download Base Cloud Image
run: |
wget -q https://cloud-images.ubuntu.com/focal/current/focal-server-cloudimg-amd64.img
wget -q https://cloud-images.ubuntu.com/resolute/current/resolute-server-cloudimg-amd64.img
- name: Generate SSH Key and Cloud-Init Seed
run: |
@@ -169,7 +163,7 @@ jobs:
# 4. Provision independent overlay disks for Worker
- name: Create Worker Disks
run: |
qemu-img create -f qcow2 -b focal-server-cloudimg-amd64.img -F qcow2 worker-1.qcow2 20G
qemu-img create -f qcow2 -b resolute-server-cloudimg-amd64.img -F qcow2 worker-1.qcow2 20G
# 5. Boot both VMs headlessly in the background with unique MACs and forwarded SSH ports
- name: Launch Worker VM
@@ -196,25 +190,94 @@ jobs:
- name: Join Worker to K3k Control Plane
run: |
echo "Getting cluster token"
K3S_TOKEN=$(kubectl get secret -n k3k-mycluster k3k-mycluster-token -o jsonpath='{.data.token}' | base64 -d)
echo "Registering Worker..."
echo "Testing connectivity from VM to K3k API server..."
ssh -i ./id_rsa -p 2222 -o StrictHostKeyChecking=no ubuntu@127.0.0.1 \
"curl -sfL https://get.k3s.io | K3S_URL='https://10.0.2.2:6443' K3S_TOKEN='${K3S_TOKEN}' sh -s - agent"
"curl -kv --max-time 10 https://10.0.2.2:30001/readyz || echo 'VM connectivity test failed'"
echo "Registering Worker..."
set +e # Don't exit on error, we want to collect logs
ssh -i ./id_rsa -p 2222 -o StrictHostKeyChecking=no ubuntu@127.0.0.1 bash -s <<EOSSH
set -x
sudo -i
curl -u "node:${K3S_TOKEN}" https://10.0.2.2:30001/v1-k3s/config
curl -k -u "node:${K3S_TOKEN}" https://10.0.2.2:30001/v1-k3s/config
# Run the official registration command
curl -sfL https://get.k3s.io | K3S_URL=https://10.0.2.2:30001 K3S_TOKEN=${K3S_TOKEN} sh -
# Check if service started
if ! sudo systemctl is-active k3s-agent; then
echo "=== k3s-agent service failed to start ==="
echo "Service status:"
sudo systemctl status k3s-agent --no-pager -l || true
echo ""
echo "=== Last 100 lines of k3s-agent journal ==="
sudo journalctl -u k3s-agent -n 100 --no-pager || true
echo ""
echo "=== Network connectivity from within VM ==="
curl -kv --max-time 5 https://10.0.2.2:30001/readyz || echo "Cannot reach API server"
echo ""
echo "=== Checking k3s-agent service environment ==="
sudo cat /etc/systemd/system/k3s-agent.service.env || true
exit 1
fi
echo "✓ K3s agent service started successfully"
sudo journalctl -u k3s-agent -n 50 --no-pager
EOSSH
JOIN_RESULT=$?
set -e
if [ $JOIN_RESULT -ne 0 ]; then
echo "Worker join failed with exit code $JOIN_RESULT"
exit 1
fi
# 8. Assert that both nodes successfully registered and transitioned to a Ready status
- name: Verify Cluster Nodes
env:
KUBECONFIG: ${{ github.workspace }}/k3k-mycluster-mycluster-kubeconfig.yaml
run: |
echo "Monitoring K3k Virtual Cluster for Node registration..."
kubectl get pod -A
kubectl get nodes
timeout 180s bash -c '
until [ $(kubectl get nodes --no-headers 2>/dev/null | grep -c "Ready") -eq 2 ]; do
until [ $(kubectl get nodes --no-headers 2>/dev/null | grep -c "Ready") -eq 1 ]; do
echo "Waiting for both worker nodes to show Ready..."
kubectl get nodes || true
sleep 5
done
'
echo "E2E Success: Both QEMU workers successfully attached to the Hosted Control Plane!"
- name: Collect logs
if: always()
run: |
journalctl -u k3s -o cat --no-pager > /tmp/k3s.log
kubectl logs -n k3k-system -l "app.kubernetes.io/name=k3k" --tail=-1 > /tmp/k3k.log
- name: Archive K3s logs
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
if: always()
with:
name: k3s-hcp-logs
path: /tmp/k3s.log
- name: Archive K3k logs
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
if: always()
with:
name: k3k-hcp-logs
path: /tmp/k3k.log
tests:
runs-on: ubuntu-latest
pkg/controller/cluster/cluster.go
+1 -1
View File
@@ -458,7 +458,7 @@ func (c *ClusterReconciler) reconcile(ctx context.Context, cluster *v1beta1.Clus
return err
}
if err := c.ensureHCPKubernetesEndpoints(ctx, cluster); err != nil {
if err := c.ensureHCPKubernetesEndpointSlice(ctx, cluster); err != nil {
return err
}
}
pkg/controller/cluster/hcp.go
+97 -38
View File
@@ -7,23 +7,18 @@ import (
"net/url"
"strconv"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/meta"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
corev1 "k8s.io/api/core/v1"
discoveryv1 "k8s.io/api/discovery/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
"github.com/rancher/k3k/pkg/controller/cluster/server"
)
// endpointSliceSkipMirrorLabel is the upstream label that opts an Endpoints
// object out of the kube-controller-manager EndpointSlice mirroring controller.
// The kube-apiserver normally sets it on default/kubernetes (because it
// manages EndpointSlices itself); in HCP mode we want the mirror controller
// to handle slices, so we strip the label.
const endpointSliceSkipMirrorLabel = "endpointslice.kubernetes.io/skip-mirror"
// ensureHCPRegistration computes the K3s installer command external nodes can
// run to join an HCP-mode cluster and stores it on cluster.Status.HCPRegistration.
//
@@ -34,7 +29,7 @@ const endpointSliceSkipMirrorLabel = "endpointslice.kubernetes.io/skip-mirror"
func (c *ClusterReconciler) ensureHCPRegistration(ctx context.Context, cluster *v1beta1.Cluster, token string) error {
log := ctrl.LoggerFrom(ctx)
url, external, err := server.ServerURL(ctx, c.Client, cluster, "", 0)
url, external, err := server.ServerURL(ctx, c.Client, cluster, selectNonLoopbackSAN(cluster), 0)
if err != nil {
return err
}
@@ -76,22 +71,55 @@ func hcpRegistrationCommand(version, serverURL, token string) string {
version, serverURL, token)
}
// ensureHCPKubernetesEndpoints maintains the default/kubernetes Service
// Endpoints inside the virtual cluster, pointing them at the externally
// selectNonLoopbackSAN returns the first non-loopback address from the
// cluster's TLS SANs, preferring spec.TLSSANs then falling back to status.TLSSANs.
// Returns empty string if no non-loopback address is found.
func selectNonLoopbackSAN(cluster *v1beta1.Cluster) string {
// Try spec.TLSSANs first (user-provided values)
for _, san := range cluster.Spec.TLSSANs {
if ip := net.ParseIP(san); ip != nil && ip.IsLoopback() {
continue
}
if san == "localhost" {
continue
}
return san // Found a non-loopback address
}
// Fall back to status.TLSSANs (computed values)
for _, san := range cluster.Status.TLSSANs {
if ip := net.ParseIP(san); ip != nil && ip.IsLoopback() {
continue
}
if san == "localhost" {
continue
}
return san
}
return ""
}
// ensureHCPKubernetesEndpointSlice maintains the default/kubernetes Service
// EndpointSlice inside the virtual cluster, pointing it at the externally
// reachable host:port (NodePort / LoadBalancer / Ingress) so that pods
// scheduled on external worker nodes can reach the in-cluster apiserver
// ClusterIP.
//
// Background: the kube-apiserver normally reconciles default/kubernetes
// Endpoints to its own --advertise-address:--secure-port (the host-cluster
// EndpointSlice to its own --advertise-address:--secure-port (the host-cluster
// pod IP and 6443). External worker nodes have no route to the host-cluster
// pod CIDR, so kube-proxy DNAT to that endpoint fails. We disable the
// apiserver reconciler in HCP mode (see serverOptions) and own this
// Endpoints object instead.
func (c *ClusterReconciler) ensureHCPKubernetesEndpoints(ctx context.Context, cluster *v1beta1.Cluster) error {
// EndpointSlice object instead.
func (c *ClusterReconciler) ensureHCPKubernetesEndpointSlice(ctx context.Context, cluster *v1beta1.Cluster) error {
log := ctrl.LoggerFrom(ctx)
rawURL, external, err := server.ServerURL(ctx, c.Client, cluster, "", 0)
rawURL, external, err := server.ServerURL(ctx, c.Client, cluster, selectNonLoopbackSAN(cluster), 0)
if err != nil {
return err
}
@@ -112,45 +140,65 @@ func (c *ClusterReconciler) ensureHCPKubernetesEndpoints(ctx context.Context, cl
return err
}
var addressType discoveryv1.AddressType
if ip := net.ParseIP(addr.IP); ip != nil {
if ip.To4() != nil {
addressType = discoveryv1.AddressTypeIPv4
} else {
addressType = discoveryv1.AddressTypeIPv6
}
} else {
return fmt.Errorf("invalid IP address %q", addr.IP)
}
virtClient, err := newVirtualClient(ctx, c.Client, cluster.Name, cluster.Namespace)
if err != nil {
return fmt.Errorf("creating virtual cluster client: %w", err)
}
endpoints := &corev1.Endpoints{
endpointSlice := &discoveryv1.EndpointSlice{
ObjectMeta: metav1.ObjectMeta{
Name: "kubernetes",
Namespace: metav1.NamespaceDefault,
},
}
_, err = controllerutil.CreateOrUpdate(ctx, virtClient, endpoints, func() error {
// Allow EndpointSlice mirroring; the apiserver may have set
// skip-mirror=true before we disabled its endpoint reconciler.
if endpoints.Labels != nil {
delete(endpoints.Labels, endpointSliceSkipMirrorLabel)
_, err = controllerutil.CreateOrUpdate(ctx, virtClient, endpointSlice, func() error {
// Ensure the service-name label is set
if endpointSlice.Labels == nil {
endpointSlice.Labels = make(map[string]string)
}
endpoints.Subsets = []corev1.EndpointSubset{
endpointSlice.Labels[discoveryv1.LabelServiceName] = "kubernetes"
endpointSlice.AddressType = addressType
endpoint := discoveryv1.Endpoint{
Addresses: []string{addr.IP},
}
if addr.Hostname != "" {
endpoint.Hostname = &addr.Hostname
}
endpointSlice.Endpoints = []discoveryv1.Endpoint{endpoint}
portName := "https"
endpointSlice.Ports = []discoveryv1.EndpointPort{
{
Addresses: []corev1.EndpointAddress{addr},
Ports: []corev1.EndpointPort{
{
Name: "https",
Port: port,
Protocol: corev1.ProtocolTCP,
},
},
Name: &portName,
Port: &port,
Protocol: new(corev1.ProtocolTCP),
},
}
return nil
})
if err != nil {
return fmt.Errorf("upserting default/kubernetes endpoints in virtual cluster: %w", err)
return fmt.Errorf("upserting default/kubernetes endpointslice in virtual cluster: %w", err)
}
log.V(1).Info("HCP kubernetes endpoints reconciled",
log.V(1).Info("HCP kubernetes endpointslice reconciled",
"address", addr.IP, "hostname", addr.Hostname, "port", port)
return nil
@@ -172,6 +220,7 @@ func parseHCPHostPort(rawURL string) (string, int32, error) {
portStr := u.Port()
var port int32 = 443
if portStr != "" {
p, err := strconv.Atoi(portStr)
if err != nil {
@@ -194,6 +243,10 @@ func parseHCPHostPort(rawURL string) (string, int32, error) {
// human-readable.
func hcpEndpointAddress(host string) (corev1.EndpointAddress, error) {
if ip := net.ParseIP(host); ip != nil {
if ip.IsLoopback() {
return corev1.EndpointAddress{}, fmt.Errorf("HCP endpoint host %q is a loopback address and cannot be used", host)
}
return corev1.EndpointAddress{IP: host}, nil
}
@@ -202,15 +255,21 @@ func hcpEndpointAddress(host string) (corev1.EndpointAddress, error) {
return corev1.EndpointAddress{}, fmt.Errorf("HCP endpoint host %q is not an IP and does not resolve: %w", host, err)
}
var filteredIPs []net.IP
for _, ip := range ips {
if v4 := ip.To4(); v4 != nil {
return corev1.EndpointAddress{IP: v4.String(), Hostname: host}, nil
if !ip.IsLoopback() {
filteredIPs = append(filteredIPs, ip)
}
}
if len(ips) == 0 {
return corev1.EndpointAddress{}, fmt.Errorf("HCP endpoint host %q resolved to no IPs", host)
if len(filteredIPs) == 0 {
return corev1.EndpointAddress{}, fmt.Errorf("HCP endpoint host %q resolved to no non-loopback IPs", host)
}
return corev1.EndpointAddress{IP: ips[0].String(), Hostname: host}, nil
if v4 := filteredIPs[0].To4(); v4 != nil {
return corev1.EndpointAddress{IP: v4.String(), Hostname: host}, nil
}
return corev1.EndpointAddress{IP: filteredIPs[0].String(), Hostname: host}, nil
}
pkg/controller/cluster/hcp_test.go
+136 -13
View File
@@ -7,13 +7,12 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apimachinery/pkg/runtime"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
"github.com/rancher/k3k/pkg/controller"
@@ -180,18 +179,142 @@ func Test_parseHCPHostPort(t *testing.T) {
}
}
func Test_hcpEndpointAddress(t *testing.T) {
t.Run("ipv4 literal is passed through", func(t *testing.T) {
got, err := hcpEndpointAddress("10.144.101.195")
require.NoError(t, err)
assert.Equal(t, "10.144.101.195", got.IP)
assert.Empty(t, got.Hostname)
})
func Test_selectNonLoopbackSAN(t *testing.T) {
tests := []struct {
name string
specSANs []string
statusSANs []string
want string
}{
{
name: "spec with loopback first, external second",
specSANs: []string{"127.0.0.1", "10.0.0.100"},
statusSANs: []string{},
want: "10.0.0.100",
},
{
name: "spec with external first",
specSANs: []string{"10.0.0.100", "127.0.0.1"},
statusSANs: []string{},
want: "10.0.0.100",
},
{
name: "spec empty, status with loopback first, external second",
specSANs: []string{},
statusSANs: []string{"127.0.0.1", "10.43.0.50"},
want: "10.43.0.50",
},
{
name: "spec with only loopback",
specSANs: []string{"127.0.0.1", "::1"},
statusSANs: []string{},
want: "",
},
{
name: "spec with localhost hostname",
specSANs: []string{"localhost", "example.com"},
statusSANs: []string{},
want: "example.com",
},
{
name: "spec with external hostname",
specSANs: []string{"hcp.example.com"},
statusSANs: []string{},
want: "hcp.example.com",
},
{
name: "empty spec and status",
specSANs: []string{},
statusSANs: []string{},
want: "",
},
{
name: "ipv6 loopback filtered",
specSANs: []string{"::1", "2001:db8::1"},
statusSANs: []string{},
want: "2001:db8::1",
},
}
t.Run("unresolvable hostname errors", func(t *testing.T) {
_, err := hcpEndpointAddress("definitely-not-a-real-host.invalid")
require.Error(t, err)
})
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
cluster := &v1beta1.Cluster{
Spec: v1beta1.ClusterSpec{
TLSSANs: tt.specSANs,
},
Status: v1beta1.ClusterStatus{
TLSSANs: tt.statusSANs,
},
}
got := selectNonLoopbackSAN(cluster)
assert.Equal(t, tt.want, got)
})
}
}
func Test_hcpEndpointAddress(t *testing.T) {
tests := []struct {
name string
input string
wantIP string
wantHostname string
wantErr bool
}{
{
name: "ipv4 literal is passed through",
input: "10.144.101.195",
wantIP: "10.144.101.195",
wantHostname: "",
wantErr: false,
},
{
name: "unresolvable hostname errors",
input: "definitely-not-a-real-host.invalid",
wantErr: true,
},
{
name: "ipv4 loopback literal is rejected",
input: "127.0.0.1",
wantErr: true,
},
{
name: "ipv6 loopback literal is rejected",
input: "::1",
wantErr: true,
},
{
name: "ipv4 loopback in range is rejected",
input: "127.0.0.100",
wantErr: true,
},
{
name: "valid ipv6 literal passes through",
input: "2001:db8::1",
wantIP: "2001:db8::1",
wantHostname: "",
wantErr: false,
},
{
name: "localhost hostname filters loopbacks",
input: "localhost",
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := hcpEndpointAddress(tt.input)
if tt.wantErr {
require.Error(t, err)
return
}
require.NoError(t, err)
assert.Equal(t, tt.wantIP, got.IP)
assert.Equal(t, tt.wantHostname, got.Hostname)
})
}
}
// Compile-time assertion: every reused exported name from the controller
pkg/controller/cluster/server/endpoint.go
+3 -2
View File
@@ -6,11 +6,12 @@ import (
"slices"
"github.com/sirupsen/logrus"
corev1 "k8s.io/api/core/v1"
networkingv1 "k8s.io/api/networking/v1"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/client"
corev1 "k8s.io/api/core/v1"
networkingv1 "k8s.io/api/networking/v1"
"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
)
pkg/controller/kubeconfig/kubeconfig.go
-1
View File
@@ -86,4 +86,3 @@ func NewConfig(url string, serverCA, clientCert, clientKey []byte) *clientcmdapi
return config
}