mirror of
https://github.com/rancher/k3k.git
synced 2026-08-19 20:36:17 +00:00
* Add HCP (Hosted Control Plane) support Introduce hosted control plane mode for k3k virtual clusters, including API types, controller logic, server endpoint handling, CLI flags, CRD updates, kubeconfig generation, and examples. Co-Authored-By: RuFlo <ruv@ruv.net> * removed hcpRegitration command added HCP conformance tests warning for hcp fix multi-VM HCP conformance test networking Both QEMU workers booted with `-net user` and ended up registering the same InternalIP (10.0.2.15) because each VM gets its own isolated NAT slirp. Flannel propagated this to `public-ip` on both nodes, so VXLAN could not tunnel between workers and any cross-node pod traffic broke (89 failed / 335 passed of 424 conformance specs). Replace user-mode networking with a Linux bridge (k3kbr0, 192.168.100.0/24) and one TAP device per VM, so the two workers share an L2 segment with unique routable IPs. NAT outbound from the bridge keeps internet access working for image pulls. Also set unique hostnames via cloud-init (worker-1/worker-2) and drop the `--node-name` flag from INSTALL_K3S_EXEC, since k3s now picks the correct node name from the OS hostname on its own. Bump hydrophone back to `--parallel 4` to match the single-VM job (parallelism was reduced earlier when the failure was thought to be resource-related). added HCP print command updated crds adding e2e tests Refactor selectNonLoopbackSAN function to accept SANs directly and update related logic in ensureHCPRegistration * Update agent flag validation and enhance ingress host check with a warning log Refactor descriptions for cluster provisioning mode and role in CRDs and documentation Refactor logging in ServerURL function to use controller-runtime logger Rename selectNonLoopbackSAN to findNonLoopbackSAN for clarity and update references Refactor ServerURL function and related code to remove unused parameters and improve clarity Remove unused imports from kubeconfig.go to improve code clarity * suggested changes * fix comment * fix test --------- Co-authored-by: jpgouin <jeanphilippe.gouin@suse.com> Co-authored-by: RuFlo <ruv@ruv.net>
139 lines
4.1 KiB
Go
139 lines
4.1 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"net"
|
|
"net/url"
|
|
"slices"
|
|
"strconv"
|
|
|
|
"k8s.io/apimachinery/pkg/types"
|
|
"sigs.k8s.io/controller-runtime/pkg/client"
|
|
|
|
corev1 "k8s.io/api/core/v1"
|
|
networkingv1 "k8s.io/api/networking/v1"
|
|
ctrl "sigs.k8s.io/controller-runtime"
|
|
|
|
"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
|
|
)
|
|
|
|
// ServerURL generates the API server URL for the kubeconfig based on the service configuration.
|
|
//
|
|
// It handles internal vs external access patterns:
|
|
// - Internal access (hostServerIP == service.ClusterIP): uses the ClusterIP for direct pod-to-pod communication
|
|
// - External access (hostServerIP != service.ClusterIP): uses the appropriate external endpoint based on the service type
|
|
//
|
|
// Service type handling:
|
|
// - ClusterIP: uses service.Spec.ClusterIP (internal-only)
|
|
// - NodePort: uses hostServerIP:NodePort for external access, ClusterIP:Port for internal access
|
|
// - LoadBalancer: uses the LoadBalancer ingress IP, falling back to its hostname
|
|
// - Ingress (if configured): takes precedence over the service-based URL
|
|
//
|
|
// The hostServerIP parameter determines the access pattern:
|
|
// - Controller reconciliation: passes service.Spec.ClusterIP → internal access
|
|
// - CLI kubeconfig export: passes the external host → external access
|
|
func ServerURL(ctx context.Context, c client.Client, cluster *v1beta1.Cluster, hostServerIP string) (*url.URL, error) {
|
|
log := ctrl.LoggerFrom(ctx)
|
|
|
|
key := types.NamespacedName{
|
|
Name: ServiceName(cluster.Name),
|
|
Namespace: cluster.Namespace,
|
|
}
|
|
|
|
// Check if ingress is configured
|
|
if cluster.Spec.Expose != nil && cluster.Spec.Expose.Ingress != nil {
|
|
key := types.NamespacedName{
|
|
Name: IngressName(cluster.Name),
|
|
Namespace: cluster.Namespace,
|
|
}
|
|
|
|
var k3kIngress networkingv1.Ingress
|
|
if err := c.Get(ctx, key, &k3kIngress); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if len(k3kIngress.Spec.Rules) > 0 && k3kIngress.Spec.Rules[0].Host != "" {
|
|
return &url.URL{
|
|
Scheme: "https",
|
|
Host: k3kIngress.Spec.Rules[0].Host,
|
|
}, nil
|
|
}
|
|
|
|
log.V(1).Info("Ingress has no rule with a host set, falling back to the service URL.")
|
|
}
|
|
|
|
// Fall back to Service-based URL
|
|
var k3kService corev1.Service
|
|
if err := c.Get(ctx, key, &k3kService); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// init to hostServerIP and 443 port
|
|
host := hostServerIP
|
|
port := int32(443)
|
|
|
|
// Use service port as default if available
|
|
if len(k3kService.Spec.Ports) > 0 {
|
|
port = k3kService.Spec.Ports[0].Port
|
|
}
|
|
|
|
// Handle each service type separately
|
|
switch k3kService.Spec.Type {
|
|
case corev1.ServiceTypeClusterIP:
|
|
host = k3kService.Spec.ClusterIP
|
|
|
|
case corev1.ServiceTypeNodePort:
|
|
// Only use NodePort if hostServerIP is NOT the ClusterIP
|
|
// If hostServerIP == ClusterIP, this is an internal connection, use ClusterIP
|
|
if hostServerIP != k3kService.Spec.ClusterIP {
|
|
if len(k3kService.Spec.Ports) > 0 {
|
|
port = k3kService.Spec.Ports[0].NodePort
|
|
}
|
|
} else {
|
|
// Internal connection: use ClusterIP
|
|
host = k3kService.Spec.ClusterIP
|
|
}
|
|
|
|
case corev1.ServiceTypeLoadBalancer:
|
|
if len(k3kService.Status.LoadBalancer.Ingress) > 0 {
|
|
ingress := k3kService.Status.LoadBalancer.Ingress[0]
|
|
|
|
switch {
|
|
case ingress.IP != "":
|
|
host = ingress.IP
|
|
case ingress.Hostname != "":
|
|
host = ingress.Hostname
|
|
default:
|
|
log.V(1).Info("No usable ingress address found in LoadBalancer service.")
|
|
}
|
|
}
|
|
}
|
|
|
|
if !slices.Contains(cluster.Status.TLSSANs, host) {
|
|
log.V(1).Info(fmt.Sprintf("IP %s not in tlsSANs.", host))
|
|
|
|
if len(cluster.Spec.TLSSANs) > 0 {
|
|
log.V(1).Info("Using the first TLS SAN in the spec as a fallback: " + cluster.Spec.TLSSANs[0])
|
|
|
|
host = cluster.Spec.TLSSANs[0]
|
|
} else if len(cluster.Status.TLSSANs) > 0 {
|
|
log.V(1).Info("No explicit tlsSANs specified. Trying to use the first TLS SAN in the status: " + cluster.Status.TLSSANs[0])
|
|
|
|
host = cluster.Status.TLSSANs[0]
|
|
} else {
|
|
log.V(1).Info("IP not found in tlsSANs. This could cause issue with the certificate validation.")
|
|
}
|
|
}
|
|
|
|
// Build URL with port only if not the default HTTPS port
|
|
if port != int32(443) {
|
|
host = net.JoinHostPort(host, strconv.Itoa(int(port)))
|
|
}
|
|
|
|
return &url.URL{
|
|
Scheme: "https",
|
|
Host: host,
|
|
}, nil
|
|
}
|