mirror of
https://github.com/rancher/k3k.git
synced 2026-08-19 12:26:17 +00:00
* Add HCP (Hosted Control Plane) support Introduce hosted control plane mode for k3k virtual clusters, including API types, controller logic, server endpoint handling, CLI flags, CRD updates, kubeconfig generation, and examples. Co-Authored-By: RuFlo <ruv@ruv.net> * removed hcpRegitration command added HCP conformance tests warning for hcp fix multi-VM HCP conformance test networking Both QEMU workers booted with `-net user` and ended up registering the same InternalIP (10.0.2.15) because each VM gets its own isolated NAT slirp. Flannel propagated this to `public-ip` on both nodes, so VXLAN could not tunnel between workers and any cross-node pod traffic broke (89 failed / 335 passed of 424 conformance specs). Replace user-mode networking with a Linux bridge (k3kbr0, 192.168.100.0/24) and one TAP device per VM, so the two workers share an L2 segment with unique routable IPs. NAT outbound from the bridge keeps internet access working for image pulls. Also set unique hostnames via cloud-init (worker-1/worker-2) and drop the `--node-name` flag from INSTALL_K3S_EXEC, since k3s now picks the correct node name from the OS hostname on its own. Bump hydrophone back to `--parallel 4` to match the single-VM job (parallelism was reduced earlier when the failure was thought to be resource-related). added HCP print command updated crds adding e2e tests Refactor selectNonLoopbackSAN function to accept SANs directly and update related logic in ensureHCPRegistration * Update agent flag validation and enhance ingress host check with a warning log Refactor descriptions for cluster provisioning mode and role in CRDs and documentation Refactor logging in ServerURL function to use controller-runtime logger Rename selectNonLoopbackSAN to findNonLoopbackSAN for clarity and update references Refactor ServerURL function and related code to remove unused parameters and improve clarity Remove unused imports from kubeconfig.go to improve code clarity * suggested changes * fix comment * fix test --------- Co-authored-by: jpgouin <jeanphilippe.gouin@suse.com> Co-authored-by: RuFlo <ruv@ruv.net>
142 lines
3.5 KiB
Go
142 lines
3.5 KiB
Go
package client
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"net"
|
|
"net/url"
|
|
"os"
|
|
|
|
"github.com/go-logr/zapr"
|
|
"go.uber.org/zap"
|
|
"k8s.io/apimachinery/pkg/runtime"
|
|
"k8s.io/client-go/kubernetes"
|
|
"k8s.io/client-go/rest"
|
|
"k8s.io/client-go/tools/clientcmd"
|
|
"sigs.k8s.io/controller-runtime/pkg/client"
|
|
"sigs.k8s.io/controller-runtime/pkg/log"
|
|
)
|
|
|
|
// Config holds the Kubernetes client configuration and clients.
|
|
type Config struct {
|
|
RestConfig *rest.Config
|
|
Clientset *kubernetes.Clientset
|
|
Client client.Client
|
|
HostIP string
|
|
}
|
|
|
|
// InitFromKubeconfig initializes Kubernetes clients from the KUBECONFIG environment variable.
|
|
// It sets up logging, reads the kubeconfig file, creates REST config and clients.
|
|
// The scheme parameter should be created using the scheme package.
|
|
func InitFromKubeconfig(ctx context.Context, scheme *runtime.Scheme) (*Config, error) {
|
|
// Setup logger
|
|
logger, err := zap.NewDevelopment()
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to create logger: %w", err)
|
|
}
|
|
|
|
log.SetLogger(zapr.NewLogger(logger))
|
|
|
|
// Get kubeconfig path from environment
|
|
kubeconfigPath := os.Getenv("KUBECONFIG")
|
|
if kubeconfigPath == "" {
|
|
return nil, fmt.Errorf("KUBECONFIG environment variable is not set")
|
|
}
|
|
|
|
// Read kubeconfig file
|
|
kubeconfig, err := os.ReadFile(kubeconfigPath)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to read kubeconfig from %s: %w", kubeconfigPath, err)
|
|
}
|
|
|
|
return InitFromBytes(ctx, kubeconfig, scheme)
|
|
}
|
|
|
|
// InitFromBytes initializes Kubernetes clients from kubeconfig bytes.
|
|
// The scheme parameter should be created using the scheme package.
|
|
func InitFromBytes(ctx context.Context, kubeconfig []byte, scheme *runtime.Scheme) (*Config, error) {
|
|
// Create REST config from kubeconfig
|
|
restConfig, err := clientcmd.RESTConfigFromKubeConfig(kubeconfig)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to create REST config: %w", err)
|
|
}
|
|
|
|
// Extract host IP from REST config
|
|
hostIP, err := getServerIP(restConfig)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to get server IP: %w", err)
|
|
}
|
|
|
|
// Create Kubernetes clientset
|
|
clientset, err := kubernetes.NewForConfig(restConfig)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to create Kubernetes clientset: %w", err)
|
|
}
|
|
|
|
// Create controller-runtime client
|
|
runtimeClient, err := client.New(restConfig, client.Options{Scheme: scheme})
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to create controller-runtime client: %w", err)
|
|
}
|
|
|
|
return &Config{
|
|
RestConfig: restConfig,
|
|
Clientset: clientset,
|
|
Client: runtimeClient,
|
|
HostIP: hostIP,
|
|
}, nil
|
|
}
|
|
|
|
// getServerIP extracts the server IP by parsing the hostname from the REST config host.
|
|
func getServerIP(cfg *rest.Config) (string, error) {
|
|
u, err := url.Parse(cfg.Host)
|
|
if err != nil {
|
|
return "", fmt.Errorf("failed to parse REST config host: %w", err)
|
|
}
|
|
|
|
host := u.Hostname()
|
|
|
|
if isLoopbackHost(host) {
|
|
if ip, ok := firstNonLoopbackIPv4(); ok {
|
|
return ip, nil
|
|
}
|
|
}
|
|
|
|
return host, nil
|
|
}
|
|
|
|
func isLoopbackHost(host string) bool {
|
|
if host == "" || host == "localhost" {
|
|
return true
|
|
}
|
|
|
|
if ip := net.ParseIP(host); ip != nil && ip.IsLoopback() {
|
|
return true
|
|
}
|
|
|
|
return false
|
|
}
|
|
|
|
func firstNonLoopbackIPv4() (string, bool) {
|
|
addrs, err := net.InterfaceAddrs()
|
|
if err != nil {
|
|
return "", false
|
|
}
|
|
|
|
for _, addr := range addrs {
|
|
ipNet, ok := addr.(*net.IPNet)
|
|
if !ok {
|
|
continue
|
|
}
|
|
|
|
ip := ipNet.IP.To4()
|
|
if ip == nil || ip.IsLoopback() || !ip.IsGlobalUnicast() {
|
|
continue
|
|
}
|
|
|
|
return ip.String(), true
|
|
}
|
|
|
|
return "", false
|
|
}
|