From 5bd779cdaada3bb21878cd507fcce5c18dec95ed Mon Sep 17 00:00:00 2001 From: Hussein Galal Date: Wed, 3 Jun 2026 19:40:31 +0300 Subject: [PATCH] Refactor bootstrap data management (#869) * Refactor bootstrap data management * Change errors from errors.New to fmt.Errorf Signed-off-by: galal-hussein --- k3k-kubelet/kubelet.go | 108 +++-------- k3k-kubelet/node.go | 71 +++---- pkg/controller/cluster/cluster.go | 26 +-- .../cluster/server/bootstrap/bootstrap.go | 180 +++++------------- pkg/controller/cluster/server/server.go | 89 +++++---- pkg/controller/cluster/statefulset.go | 38 +--- pkg/controller/cluster/status.go | 4 +- pkg/controller/kubeconfig/kubeconfig.go | 2 +- pkg/k3s/bootstrap.go | 89 +++++++++ pkg/k3s/certs.go | 22 +++ pkg/k3s/client.go | 124 ++++++++++++ 11 files changed, 390 insertions(+), 363 deletions(-) create mode 100644 pkg/k3s/bootstrap.go create mode 100644 pkg/k3s/certs.go create mode 100644 pkg/k3s/client.go diff --git a/k3k-kubelet/kubelet.go b/k3k-kubelet/kubelet.go index 83da76b0..473a72c2 100644 --- a/k3k-kubelet/kubelet.go +++ b/k3k-kubelet/kubelet.go @@ -2,8 +2,7 @@ package main import ( "context" - "crypto/x509" - "errors" + "fmt" "path" "time" @@ -14,7 +13,6 @@ import ( "github.com/virtual-kubelet/virtual-kubelet/node/nodeutil" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" - "k8s.io/apiserver/pkg/authentication/user" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" @@ -27,7 +25,6 @@ import ( corev1 "k8s.io/api/core/v1" clientgoscheme "k8s.io/client-go/kubernetes/scheme" corev1client "k8s.io/client-go/kubernetes/typed/core/v1" - clientcmdapi "k8s.io/client-go/tools/clientcmd/api" ctrl "sigs.k8s.io/controller-runtime" ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" ctrlserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" @@ -36,9 +33,6 @@ import ( "github.com/rancher/k3k/k3k-kubelet/provider" "github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1" "github.com/rancher/k3k/pkg/controller" - "github.com/rancher/k3k/pkg/controller/certs" - "github.com/rancher/k3k/pkg/controller/cluster/server" - "github.com/rancher/k3k/pkg/controller/cluster/server/bootstrap" ) var baseScheme = runtime.NewScheme() @@ -82,7 +76,7 @@ func newKubelet(ctx context.Context, c *config) (*kubelet, error) { return nil, err } - virtConfig, err := virtRestConfig(ctx, c.VirtKubeconfig, hostClient, c.ClusterName, c.ClusterNamespace, c.Token) + virtConfig, err := virtRestConfig(ctx, c.VirtKubeconfig, hostClient, c.ClusterName, c.ClusterNamespace) if err != nil { return nil, err } @@ -115,13 +109,13 @@ func newKubelet(ctx context.Context, c *config) (*kubelet, error) { }, }) if err != nil { - return nil, errors.New("unable to create controller-runtime mgr for host cluster: " + err.Error()) + return nil, fmt.Errorf("unable to create controller-runtime mgr for host cluster: %w", err) } // virtual client will only use core types (for now), no need to add anything other than the basics virtualScheme := runtime.NewScheme() if err := clientgoscheme.AddToScheme(virtualScheme); err != nil { - return nil, errors.New("unable to add client go types to virtual cluster scheme: " + err.Error()) + return nil, fmt.Errorf("unable to add client go types to virtual cluster scheme: %w", err) } virtualMgr, err := ctrl.NewManager(virtConfig, manager.Options{ @@ -134,7 +128,7 @@ func newKubelet(ctx context.Context, c *config) (*kubelet, error) { }, }) if err != nil { - return nil, errors.New("unable to create controller-runtime mgr for virtual cluster: " + err.Error()) + return nil, fmt.Errorf("unable to create controller-runtime mgr for virtual cluster: %w", err) } controllerName := c.AgentHostname @@ -144,12 +138,12 @@ func newKubelet(ctx context.Context, c *config) (*kubelet, error) { virtEventRecorder := eb.NewRecorder(virtualScheme, corev1.EventSource{Component: path.Join(controllerName, "pod-controller")}) if err := addControllers(ctx, hostMgr, virtualMgr, c, hostClient, virtEventRecorder); err != nil { - return nil, errors.New("failed to add controller: " + err.Error()) + return nil, fmt.Errorf("failed to add controller: %w", err) } clusterIP, err := clusterIP(ctx, c.ServiceName, c.ClusterNamespace, hostClient) if err != nil { - return nil, errors.New("failed to extract the clusterIP for the server service: " + err.Error()) + return nil, fmt.Errorf("failed to extract the clusterIP for the server service: %w", err) } // get the cluster's DNS IP to be injected to pods @@ -157,12 +151,12 @@ func newKubelet(ctx context.Context, c *config) (*kubelet, error) { dnsName := controller.SafeConcatNameWithPrefix(c.ClusterName, "kube-dns") if err := hostClient.Get(ctx, types.NamespacedName{Name: dnsName, Namespace: c.ClusterNamespace}, &dnsService); err != nil { - return nil, errors.New("failed to get the DNS service for the cluster: " + err.Error()) + return nil, fmt.Errorf("failed to get the DNS service for the cluster: %w", err) } var virtualCluster v1beta1.Cluster if err := hostClient.Get(ctx, types.NamespacedName{Name: c.ClusterName, Namespace: c.ClusterNamespace}, &virtualCluster); err != nil { - return nil, errors.New("failed to get virtualCluster spec: " + err.Error()) + return nil, fmt.Errorf("failed to get virtualCluster spec: %w", err) } return &kubelet{ @@ -245,7 +239,7 @@ func (k *kubelet) newProviderFunc(cfg config) nodeutil.NewProviderFunc { return func(pc nodeutil.ProviderConfig) (nodeutil.Provider, node.NodeProvider, error) { utilProvider, err := provider.New(*k.hostConfig, k.hostMgr, k.virtualMgr, k.logger, cfg.ClusterNamespace, cfg.ClusterName, cfg.ServerIP, k.dnsIP, cfg.AgentHostname) if err != nil { - return nil, nil, errors.New("unable to make nodeutil provider: " + err.Error()) + return nil, nil, fmt.Errorf("unable to make nodeutil provider: %w", err) } err = provider.ConfigureNode( @@ -265,76 +259,32 @@ func (k *kubelet) newProviderFunc(cfg config) nodeutil.NewProviderFunc { } } -func virtRestConfig(ctx context.Context, virtualConfigPath string, hostClient ctrlruntimeclient.Client, clusterName, clusterNamespace, token string) (*rest.Config, error) { +func virtRestConfig(ctx context.Context, virtualConfigPath string, hostClient ctrlruntimeclient.Client, clusterName, clusterNamespace string) (*rest.Config, error) { if virtualConfigPath != "" { return clientcmd.BuildConfigFromFlags("", virtualConfigPath) } - // virtual kubeconfig file is empty, trying to fetch the k3k cluster kubeconfig - var cluster v1beta1.Cluster - if err := hostClient.Get(ctx, types.NamespacedName{Namespace: clusterNamespace, Name: clusterName}, &cluster); err != nil { - return nil, err + + var clusterKubeConfig corev1.Secret + + kubeconfigSecretName := types.NamespacedName{ + Name: controller.SafeConcatNameWithPrefix(clusterName, "kubeconfig"), + Namespace: clusterNamespace, } - endpoint := server.ServiceName(cluster.Name) + "." + cluster.Namespace - - var b *bootstrap.ControlRuntimeBootstrap - if err := retry.OnError(controller.Backoff, func(err error) bool { return err != nil }, func() error { - var err error - - b, err = bootstrap.DecodedBootstrap(token, endpoint) - logger.Error(err, "decoded bootstrap") - - return err + return hostClient.Get(ctx, kubeconfigSecretName, &clusterKubeConfig) }); err != nil { - return nil, errors.New("unable to decode bootstrap: " + err.Error()) + return nil, fmt.Errorf("unable to decode bootstrap: %w", err) } - adminCert, adminKey, err := certs.CreateClientCertKey( - controller.AdminCommonName, - []string{user.SystemPrivilegedGroup}, - nil, []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, - time.Hour*24*time.Duration(356), - b.ClientCA.Content, - b.ClientCAKey.Content, - ) + restConfig, err := clientcmd.RESTConfigFromKubeConfig(clusterKubeConfig.Data["kubeconfig.yaml"]) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to create config from kubeconfig file: %w", err) } - url := "https://" + server.ServiceName(cluster.Name) - - kubeconfigData, err := kubeconfigBytes(url, []byte(b.ServerCA.Content), adminCert, adminKey) - if err != nil { - return nil, err - } - - return clientcmd.RESTConfigFromKubeConfig(kubeconfigData) -} - -func kubeconfigBytes(url string, serverCA, clientCert, clientKey []byte) ([]byte, error) { - config := clientcmdapi.NewConfig() - - cluster := clientcmdapi.NewCluster() - cluster.CertificateAuthorityData = serverCA - cluster.Server = url - - authInfo := clientcmdapi.NewAuthInfo() - authInfo.ClientCertificateData = clientCert - authInfo.ClientKeyData = clientKey - - context := clientcmdapi.NewContext() - context.AuthInfo = "default" - context.Cluster = "default" - - config.Clusters["default"] = cluster - config.AuthInfos["default"] = authInfo - config.Contexts["default"] = context - config.CurrentContext = "default" - - return clientcmd.Write(*config) + return restConfig, nil } func addControllers(ctx context.Context, hostMgr, virtualMgr manager.Manager, c *config, hostClient ctrlruntimeclient.Client, virtEventRecorder record.EventRecorder) error { @@ -350,39 +300,39 @@ func addControllers(ctx context.Context, hostMgr, virtualMgr manager.Manager, c } if err := syncer.AddConfigMapSyncer(ctx, virtualMgr, hostMgr, c.ClusterName, c.ClusterNamespace); err != nil { - return errors.New("failed to add configmap global syncer: " + err.Error()) + return fmt.Errorf("failed to add configmap global syncer: %w", err) } if err := syncer.AddSecretSyncer(ctx, virtualMgr, hostMgr, c.ClusterName, c.ClusterNamespace); err != nil { - return errors.New("failed to add secret global syncer: " + err.Error()) + return fmt.Errorf("failed to add secret global syncer: %w", err) } logger.Info("adding service syncer controller") if err := syncer.AddServiceSyncer(ctx, virtualMgr, hostMgr, c.ClusterName, c.ClusterNamespace); err != nil { - return errors.New("failed to add service syncer controller: " + err.Error()) + return fmt.Errorf("failed to add service syncer controller: %w", err) } logger.Info("adding ingress syncer controller") if err := syncer.AddIngressSyncer(ctx, virtualMgr, hostMgr, c.ClusterName, c.ClusterNamespace); err != nil { - return errors.New("failed to add ingress syncer controller: " + err.Error()) + return fmt.Errorf("failed to add ingress syncer controller: %w", err) } logger.Info("adding pvc syncer controller") if err := syncer.AddPVCSyncer(ctx, virtualMgr, hostMgr, c.ClusterName, c.ClusterNamespace); err != nil { - return errors.New("failed to add pvc syncer controller: " + err.Error()) + return fmt.Errorf("failed to add pvc syncer controller: %w", err) } logger.Info("adding priorityclass controller") if err := syncer.AddPriorityClassSyncer(ctx, virtualMgr, hostMgr, c.ClusterName, c.ClusterNamespace); err != nil { - return errors.New("failed to add priorityclass controller: " + err.Error()) + return fmt.Errorf("failed to add priorityclass controller: %w", err) } if err := syncer.AddEventSyncer(ctx, virtualMgr, hostMgr, c.ClusterName, c.ClusterNamespace, virtEventRecorder); err != nil { - return errors.New("failed to add event syncer controller: " + err.Error()) + return fmt.Errorf("failed to add event syncer controller: %w", err) } return nil diff --git a/k3k-kubelet/node.go b/k3k-kubelet/node.go index 52c2534e..26f9a438 100644 --- a/k3k-kubelet/node.go +++ b/k3k-kubelet/node.go @@ -2,27 +2,21 @@ package main import ( "crypto/tls" - "crypto/x509" - "errors" "fmt" - "net" "net/http" "github.com/virtual-kubelet/virtual-kubelet/node/nodeutil" "k8s.io/client-go/util/retry" - certutil "github.com/rancher/dynamiclistener/cert" - "github.com/rancher/k3k/pkg/controller" - "github.com/rancher/k3k/pkg/controller/certs" "github.com/rancher/k3k/pkg/controller/cluster/server" - "github.com/rancher/k3k/pkg/controller/cluster/server/bootstrap" + "github.com/rancher/k3k/pkg/k3s" ) func (k *kubelet) registerNode(agentIP, podIP string, cfg config) error { - tlsConfig, err := loadTLSConfig(cfg, k.name, k.token, agentIP, podIP) + tlsConfig, err := loadTLSConfig(cfg, k.token, agentIP, podIP) if err != nil { - return errors.New("unable to get tls config: " + err.Error()) + return fmt.Errorf("unable to get tls config: %w", err) } mux := http.NewServeMux() @@ -39,7 +33,7 @@ func (k *kubelet) registerNode(agentIP, podIP string, cfg config) error { }, ) if err != nil { - return errors.New("unable to start kubelet: " + err.Error()) + return fmt.Errorf("unable to start kubelet: %w", err) } k.node = node @@ -63,56 +57,35 @@ func nodeOpt(mux *http.ServeMux, tlsConfig *tls.Config, port int) nodeutil.NodeO } } -func loadTLSConfig(cfg config, nodeName, token, agentIP, podIP string) (*tls.Config, error) { - var b *bootstrap.ControlRuntimeBootstrap +// loadTLSConfig function will request kubelet serving crt from k3s server and will use it to +// register a new node to the server, note that we use serving cert to allow adding IPSans to +// the certificate request +func loadTLSConfig(cfg config, token, agentIP, podIP string) (*tls.Config, error) { + serviceName := fmt.Sprintf("%s.%s", server.ServiceName(cfg.ClusterName), cfg.ClusterNamespace) - endpoint := fmt.Sprintf("%s.%s", server.ServiceName(cfg.ClusterName), cfg.ClusterNamespace) + client := k3s.New(k3s.ClientConfig{ + ServerIP: serviceName, + Token: token, + AgentIP: agentIP, + PodIP: podIP, + NodeName: controller.SafeConcatName(cfg.ClusterName, "server-0"), + }) + + var tlsCrt *tls.Certificate if err := retry.OnError(controller.Backoff, func(err error) bool { - return err != nil + return err == k3s.ErrServerNotReady }, func() error { var err error - b, err = bootstrap.DecodedBootstrap(token, endpoint) + tlsCrt, err = k3s.GetServingKubeletCrt(client) return err }); err != nil { - return nil, errors.New("unable to decode bootstrap: " + err.Error()) + return nil, fmt.Errorf("unable to request serving kubelet certificate: %w", err) } - altNames := certutil.AltNames{ - DNSNames: []string{cfg.AgentHostname}, - IPs: []net.IP{ - net.ParseIP(agentIP), - net.ParseIP(podIP), - }, - } - - cert, key, err := certs.CreateClientCertKey(nodeName, nil, &altNames, []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, 0, b.ServerCA.Content, b.ServerCAKey.Content) - if err != nil { - return nil, errors.New("unable to get cert and key: " + err.Error()) - } - - clientCert, err := tls.X509KeyPair(cert, key) - if err != nil { - return nil, errors.New("unable to get key pair: " + err.Error()) - } - - // create rootCA CertPool - certs, err := certutil.ParseCertsPEM([]byte(b.ServerCA.Content)) - if err != nil { - return nil, errors.New("unable to create ca certs: " + err.Error()) - } - - if len(certs) < 1 { - return nil, errors.New("ca cert is not parsed correctly") - } - - pool := x509.NewCertPool() - pool.AddCert(certs[0]) - return &tls.Config{ - RootCAs: pool, - Certificates: []tls.Certificate{clientCert}, + Certificates: []tls.Certificate{*tlsCrt}, }, nil } diff --git a/pkg/controller/cluster/cluster.go b/pkg/controller/cluster/cluster.go index 8e09f80c..38df4510 100644 --- a/pkg/controller/cluster/cluster.go +++ b/pkg/controller/cluster/cluster.go @@ -42,6 +42,7 @@ import ( "github.com/rancher/k3k/pkg/controller/cluster/server/bootstrap" "github.com/rancher/k3k/pkg/controller/kubeconfig" "github.com/rancher/k3k/pkg/controller/policy" + "github.com/rancher/k3k/pkg/k3s" ) const ( @@ -293,7 +294,7 @@ func (c *ClusterReconciler) Reconcile(ctx context.Context, req reconcile.Request // if there was an error during the reconciliation, return if reconcilerErr != nil { - if errors.Is(reconcilerErr, bootstrap.ErrServerNotReady) { + if errors.Is(reconcilerErr, k3s.ErrServerNotReady) { log.V(1).Info("Server not ready, requeueing") return reconcile.Result{RequeueAfter: time.Second * 10}, nil } @@ -449,31 +450,12 @@ func (c *ClusterReconciler) ensureBootstrapSecret(ctx context.Context, cluster * log := ctrl.LoggerFrom(ctx) log.V(1).Info("Ensuring bootstrap secret") - bootstrapData, err := bootstrap.GenerateBootstrapData(ctx, cluster, serviceIP, token) + data, err := bootstrap.Fetch(ctx, serviceIP, token) if err != nil { return err } - bootstrapSecret := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: controller.SafeConcatNameWithPrefix(cluster.Name, "bootstrap"), - Namespace: cluster.Namespace, - }, - } - - _, err = controllerutil.CreateOrUpdate(ctx, c.Client, bootstrapSecret, func() error { - if err := controllerutil.SetControllerReference(cluster, bootstrapSecret, c.Scheme); err != nil { - return err - } - - bootstrapSecret.Data = map[string][]byte{ - "bootstrap": bootstrapData, - } - - return nil - }) - - return err + return bootstrap.SaveToSecret(ctx, c.Client, c.Scheme, cluster, data) } // ensureKubeconfigSecret will create or update the Secret containing the kubeconfig data from the k3s server diff --git a/pkg/controller/cluster/server/bootstrap/bootstrap.go b/pkg/controller/cluster/server/bootstrap/bootstrap.go index 2da85db9..19aa84e8 100644 --- a/pkg/controller/cluster/server/bootstrap/bootstrap.go +++ b/pkg/controller/cluster/server/bootstrap/bootstrap.go @@ -2,167 +2,68 @@ package bootstrap import ( "context" - "crypto/tls" - "encoding/base64" "encoding/json" "errors" - "fmt" - "net/http" - "syscall" - "time" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrl "sigs.k8s.io/controller-runtime" "github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1" "github.com/rancher/k3k/pkg/controller" + "github.com/rancher/k3k/pkg/k3s" ) -var ErrServerNotReady = errors.New("server not ready") +const ( + TLSDir = "/var/lib/rancher/k3s/server/tls/" +) -type ControlRuntimeBootstrap struct { - ServerCA content `json:"serverCA"` - ServerCAKey content `json:"serverCAKey"` - ClientCA content `json:"clientCA"` - ClientCAKey content `json:"clientCAKey"` - ETCDServerCA content `json:"etcdServerCA"` - ETCDServerCAKey content `json:"etcdServerCAKey"` +// Fetch requests bootstrap data from k3s using the token and decodes it, +// to avoid double encoding when stored as secret. +func Fetch(ctx context.Context, ip, token string) (*k3s.BootstrapData, error) { + log := ctrl.LoggerFrom(ctx) + log.V(1).Info("Fetching bootstrap data from K3s API") + + return fetchFromK3sServer(ip, token) } -type content struct { - Timestamp string - Content string -} - -// Generate generates the bootstrap for the cluster: -// 1- use the server token to get the bootstrap data from k3s -// 2- save the bootstrap data as a secret -func GenerateBootstrapData(ctx context.Context, cluster *v1beta1.Cluster, ip, token string) ([]byte, error) { - bootstrap, err := requestBootstrap(token, ip) +// SaveToSecret marshals the bootstrap data and stores it in a Secret owned by the cluster, +// creating the Secret if it does not exist or updating it otherwise. +func SaveToSecret(ctx context.Context, c client.Client, scheme *runtime.Scheme, cluster *v1beta1.Cluster, data *k3s.BootstrapData) error { + bootstrapData, err := json.Marshal(data) if err != nil { - return nil, fmt.Errorf("failed to request bootstrap secret: %w", err) + return err } - if err := decodeBootstrap(bootstrap); err != nil { - return nil, fmt.Errorf("failed to decode bootstrap secret: %w", err) - } - - return json.Marshal(bootstrap) -} - -func requestBootstrap(token, serverIP string) (*ControlRuntimeBootstrap, error) { - url := "https://" + serverIP + "/v1-k3s/server-bootstrap" - - client := http.Client{ - Transport: &http.Transport{ - TLSClientConfig: &tls.Config{ - InsecureSkipVerify: true, - }, + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: controller.SafeConcatNameWithPrefix(cluster.Name, "bootstrap"), + Namespace: cluster.Namespace, }, - Timeout: 5 * time.Second, } - req, err := http.NewRequest(http.MethodGet, url, nil) - if err != nil { - return nil, err - } - - req.Header.Add("Authorization", "Basic "+basicAuth("server", token)) - - resp, err := client.Do(req) - if err != nil { - if errors.Is(err, syscall.ECONNREFUSED) { - return nil, ErrServerNotReady + _, err = controllerutil.CreateOrUpdate(ctx, c, secret, func() error { + if err := controllerutil.SetControllerReference(cluster, secret, scheme); err != nil { + return err } - return nil, err - } + secret.Data = map[string][]byte{ + "bootstrap": bootstrapData, + } - defer func() { - _ = resp.Body.Close() - }() + return nil + }) - var runtimeBootstrap ControlRuntimeBootstrap - if err := json.NewDecoder(resp.Body).Decode(&runtimeBootstrap); err != nil { - return nil, err - } - - return &runtimeBootstrap, nil + return err } -func basicAuth(username, password string) string { - auth := username + ":" + password - return base64.StdEncoding.EncodeToString([]byte(auth)) -} - -func decodeBootstrap(bootstrap *ControlRuntimeBootstrap) error { - // client-ca - decoded, err := base64.StdEncoding.DecodeString(bootstrap.ClientCA.Content) - if err != nil { - return err - } - - bootstrap.ClientCA.Content = string(decoded) - - // client-ca-key - decoded, err = base64.StdEncoding.DecodeString(bootstrap.ClientCAKey.Content) - if err != nil { - return err - } - - bootstrap.ClientCAKey.Content = string(decoded) - - // server-ca - decoded, err = base64.StdEncoding.DecodeString(bootstrap.ServerCA.Content) - if err != nil { - return err - } - - bootstrap.ServerCA.Content = string(decoded) - - // server-ca-key - decoded, err = base64.StdEncoding.DecodeString(bootstrap.ServerCAKey.Content) - if err != nil { - return err - } - - bootstrap.ServerCAKey.Content = string(decoded) - - // etcd-ca - decoded, err = base64.StdEncoding.DecodeString(bootstrap.ETCDServerCA.Content) - if err != nil { - return err - } - - bootstrap.ETCDServerCA.Content = string(decoded) - - // etcd-ca-key - decoded, err = base64.StdEncoding.DecodeString(bootstrap.ETCDServerCAKey.Content) - if err != nil { - return err - } - - bootstrap.ETCDServerCAKey.Content = string(decoded) - - return nil -} - -func DecodedBootstrap(token, ip string) (*ControlRuntimeBootstrap, error) { - bootstrap, err := requestBootstrap(token, ip) - if err != nil { - return nil, err - } - - if err := decodeBootstrap(bootstrap); err != nil { - return nil, err - } - - return bootstrap, nil -} - -func GetFromSecret(ctx context.Context, client client.Client, cluster *v1beta1.Cluster) (*ControlRuntimeBootstrap, error) { +// LoadFromSecret reads the bootstrap data of a certain cluster and returns the decoded content. +func LoadFromSecret(ctx context.Context, client client.Client, cluster *v1beta1.Cluster) (*k3s.BootstrapData, error) { key := types.NamespacedName{ Name: controller.SafeConcatNameWithPrefix(cluster.Name, "bootstrap"), Namespace: cluster.Namespace, @@ -178,9 +79,18 @@ func GetFromSecret(ctx context.Context, client client.Client, cluster *v1beta1.C return nil, errors.New("empty bootstrap") } - var bootstrap ControlRuntimeBootstrap + var bootstrap k3s.BootstrapData err := json.Unmarshal(bootstrapData, &bootstrap) return &bootstrap, err } + +func fetchFromK3sServer(serviceIP, token string) (*k3s.BootstrapData, error) { + client := k3s.New(k3s.ClientConfig{ + ServerIP: serviceIP, + Token: token, + }) + + return k3s.GetServerBootstrap(client) +} diff --git a/pkg/controller/cluster/server/server.go b/pkg/controller/cluster/server/server.go index c80f47e7..526e6949 100644 --- a/pkg/controller/cluster/server/server.go +++ b/pkg/controller/cluster/server/server.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "fmt" + "path/filepath" "sort" "strings" "text/template" @@ -26,12 +27,22 @@ import ( ) const ( - serverName = "server" - configName = "server-config" - initConfigName = "init-server-config" + initConfigName = "init-server-config" + configName = "server-config" + serverName = "server" + k3sInitConfigDir = "/opt/rancher/k3s/init" + k3sConfigDir = "/opt/rancher/k3s/server" + k3sRunDir = "/run" + k3sCNIDir = "/var/lib/cni" + k3sKubeletDir = "/var/lib/kubelet" + k3sDataDir = "/var/lib/rancher/k3s" + k3sETCDDataDir = "/var/lib/rancher/k3s/server/db/etcd" + k3sManifestDir = "/var/lib/rancher/k3s/server/manifests" + k3sTLSDir = "/var/lib/rancher/k3s/server/tls" + k3sLogDir = "/var/log" + k3sVarRunDir = "/var/run" ) -// Server type Server struct { cluster *v1beta1.Cluster client client.Client @@ -70,7 +81,7 @@ func (s *Server) podSpec(ctx context.Context, image, name string, persistent boo PriorityClassName: s.cluster.Spec.PriorityClass, Volumes: []corev1.Volume{ { - Name: "initconfig", + Name: "init-config", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: configSecretName(s.cluster.Name, true), @@ -104,25 +115,25 @@ func (s *Server) podSpec(ctx context.Context, image, name string, persistent boo }, }, { - Name: "varrun", + Name: "var-run", VolumeSource: corev1.VolumeSource{ EmptyDir: &corev1.EmptyDirVolumeSource{}, }, }, { - Name: "varlibcni", + Name: "var-lib-cni", VolumeSource: corev1.VolumeSource{ EmptyDir: &corev1.EmptyDirVolumeSource{}, }, }, { - Name: "varlog", + Name: "var-log", VolumeSource: corev1.VolumeSource{ EmptyDir: &corev1.EmptyDirVolumeSource{}, }, }, { - Name: "varlibkubelet", + Name: "var-lib-kubelet", VolumeSource: corev1.VolumeSource{ EmptyDir: &corev1.EmptyDirVolumeSource{}, }, @@ -154,42 +165,42 @@ func (s *Server) podSpec(ctx context.Context, image, name string, persistent boo VolumeMounts: []corev1.VolumeMount{ { Name: "config", - MountPath: "/opt/rancher/k3s/server", + MountPath: k3sConfigDir, ReadOnly: false, }, { - Name: "initconfig", - MountPath: "/opt/rancher/k3s/init", + Name: "init-config", + MountPath: k3sInitConfigDir, ReadOnly: false, }, { Name: "run", - MountPath: "/run", + MountPath: k3sRunDir, ReadOnly: false, }, { - Name: "varrun", - MountPath: "/var/run", + Name: "var-run", + MountPath: k3sVarRunDir, ReadOnly: false, }, { - Name: "varlibcni", - MountPath: "/var/lib/cni", + Name: "var-lib-cni", + MountPath: k3sCNIDir, ReadOnly: false, }, { - Name: "varlibkubelet", - MountPath: "/var/lib/kubelet", + Name: "var-lib-kubelet", + MountPath: k3sKubeletDir, ReadOnly: false, }, { - Name: "varlibrancherk3s", - MountPath: "/var/lib/rancher/k3s", + Name: "var-lib-rancher-k3s", + MountPath: k3sDataDir, ReadOnly: false, }, { - Name: "varlog", - MountPath: "/var/log", + Name: "var-log", + MountPath: k3sLogDir, ReadOnly: false, }, }, @@ -206,7 +217,7 @@ func (s *Server) podSpec(ctx context.Context, image, name string, persistent boo podSpec.Containers[0].Command = cmd if !persistent { podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{ - Name: "varlibrancherk3s", + Name: "var-lib-rancher-k3s", VolumeSource: corev1.VolumeSource{ EmptyDir: &corev1.EmptyDirVolumeSource{}, }, @@ -405,7 +416,7 @@ func (s *Server) setupDynamicPersistence() corev1.PersistentVolumeClaim { APIVersion: "v1", }, ObjectMeta: metav1.ObjectMeta{ - Name: "varlibrancherk3s", + Name: "var-lib-rancher-k3s", Namespace: s.cluster.Namespace, }, Spec: corev1.PersistentVolumeClaimSpec{ @@ -441,9 +452,9 @@ func (s *Server) setupStartCommand() (string, error) { } if err := tmplCmd.Execute(&output, map[string]string{ - "ETCD_DIR": "/var/lib/rancher/k3s/server/db/etcd", - "INIT_CONFIG": "/opt/rancher/k3s/init/config.yaml", - "SERVER_CONFIG": "/opt/rancher/k3s/server/config.yaml", + "ETCD_DIR": k3sETCDDataDir, + "INIT_CONFIG": filepath.Join(k3sInitConfigDir, "config.yaml"), + "SERVER_CONFIG": filepath.Join(k3sConfigDir, "config.yaml"), "CLUSTER_MODE": mode, "K3K_MODE": string(s.cluster.Spec.Mode), "EXTRA_ARGS": strings.Join(s.cluster.Spec.ServerArgs, " "), @@ -511,7 +522,6 @@ func (s *Server) mountCACert(volumeName, certName, secretName string, subPathMou ) // avoid re-adding secretName in case of combined secret - volume = &corev1.Volume{ Name: volumeName, VolumeSource: corev1.VolumeSource{ @@ -528,22 +538,19 @@ func (s *Server) mountCACert(volumeName, certName, secretName string, subPathMou mountFile = strings.TrimPrefix(certName, "etcd-") } - // add the mount for the cert except for the service account token - if certName != "service" { + for _, crtOrKey := range []string{"crt", "key"} { + // skip adding cert mount for service account token + if certName == "service" && crtOrKey == "crt" { + continue + } + mounts = append(mounts, corev1.VolumeMount{ Name: volumeName, - MountPath: fmt.Sprintf("/var/lib/rancher/k3s/server/tls%s/%s.crt", etcdPrefix, mountFile), - SubPath: subPathMount + ".crt", + MountPath: filepath.Join(k3sTLSDir, etcdPrefix, mountFile) + "." + crtOrKey, + SubPath: subPathMount + "." + crtOrKey, }) } - // add the mount for the key - mounts = append(mounts, corev1.VolumeMount{ - Name: volumeName, - MountPath: fmt.Sprintf("/var/lib/rancher/k3s/server/tls%s/%s.key", etcdPrefix, mountFile), - SubPath: subPathMount + ".key", - }) - return volume, mounts } @@ -603,7 +610,7 @@ func (s *Server) buildAddonsVolumes(ctx context.Context) ([]corev1.Volume, []cor volumeMount := corev1.VolumeMount{ Name: name, - MountPath: "/var/lib/rancher/k3s/server/manifests/" + addon.SecretRef, + MountPath: filepath.Join(k3sManifestDir, addon.SecretRef), ReadOnly: true, } mounts = append(mounts, volumeMount) diff --git a/pkg/controller/cluster/statefulset.go b/pkg/controller/cluster/statefulset.go index c0230e6c..5f3df307 100644 --- a/pkg/controller/cluster/statefulset.go +++ b/pkg/controller/cluster/statefulset.go @@ -11,7 +11,6 @@ import ( "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/util/retry" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" @@ -32,6 +31,7 @@ import ( "github.com/rancher/k3k/pkg/controller/certs" "github.com/rancher/k3k/pkg/controller/cluster/server" "github.com/rancher/k3k/pkg/controller/cluster/server/bootstrap" + "github.com/rancher/k3k/pkg/k3s" ) const ( @@ -44,7 +44,7 @@ type StatefulSetReconciler struct { Scheme *runtime.Scheme } -// Add adds a new controller to the manager +// AddStatefulSetController adds a new statefulset controller to the manager func AddStatefulSetController(ctx context.Context, mgr manager.Manager, maxConcurrentReconciles int) error { // initialize a new Reconciler reconciler := StatefulSetReconciler{ @@ -180,21 +180,14 @@ func (p *StatefulSetReconciler) getETCDTLS(ctx context.Context, cluster *v1beta1 log := ctrl.LoggerFrom(ctx) log.V(1).Info("Generating ETCD TLS client certificate", "cluster", cluster) - token, err := p.clusterToken(ctx, cluster) - if err != nil { - return nil, err - } - - endpoint := server.ServiceName(cluster.Name) + "." + cluster.Namespace - - var b *bootstrap.ControlRuntimeBootstrap + var b *k3s.BootstrapData if err := retry.OnError(k3kcontroller.Backoff, func(err error) bool { return true }, func() error { var err error - b, err = bootstrap.DecodedBootstrap(token, endpoint) + b, err = bootstrap.LoadFromSecret(ctx, p.Client, cluster) return err }); err != nil { @@ -265,29 +258,6 @@ func removePeer(ctx context.Context, client *clientv3.Client, name, address stri return nil } -func (p *StatefulSetReconciler) clusterToken(ctx context.Context, cluster *v1beta1.Cluster) (string, error) { - var tokenSecret corev1.Secret - - nn := types.NamespacedName{ - Name: TokenSecretName(cluster.Name), - Namespace: cluster.Namespace, - } - - if cluster.Spec.TokenSecretRef != nil { - nn.Name = TokenSecretName(cluster.Name) - } - - if err := p.Client.Get(ctx, nn, &tokenSecret); err != nil { - return "", err - } - - if _, ok := tokenSecret.Data["token"]; !ok { - return "", fmt.Errorf("no token field in secret %s/%s", nn.Namespace, nn.Name) - } - - return string(tokenSecret.Data["token"]), nil -} - func (p *StatefulSetReconciler) handleDeletion(ctx context.Context, sts *appsv1.StatefulSet) (ctrl.Result, error) { log := ctrl.LoggerFrom(ctx) diff --git a/pkg/controller/cluster/status.go b/pkg/controller/cluster/status.go index 7553e55a..0a229f77 100644 --- a/pkg/controller/cluster/status.go +++ b/pkg/controller/cluster/status.go @@ -11,7 +11,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1" - "github.com/rancher/k3k/pkg/controller/cluster/server/bootstrap" + "github.com/rancher/k3k/pkg/k3s" ) const ( @@ -57,7 +57,7 @@ func (c *ClusterReconciler) updateStatus(ctx context.Context, cluster *v1beta1.C return } - if errors.Is(reconcileErr, bootstrap.ErrServerNotReady) { + if errors.Is(reconcileErr, k3s.ErrServerNotReady) { cluster.Status.Phase = v1beta1.ClusterProvisioning meta.SetStatusCondition(&cluster.Status.Conditions, metav1.Condition{ Type: ConditionReady, diff --git a/pkg/controller/kubeconfig/kubeconfig.go b/pkg/controller/kubeconfig/kubeconfig.go index ea20a6a9..44860314 100644 --- a/pkg/controller/kubeconfig/kubeconfig.go +++ b/pkg/controller/kubeconfig/kubeconfig.go @@ -40,7 +40,7 @@ func New() *KubeConfig { } func (k *KubeConfig) Generate(ctx context.Context, client client.Client, cluster *v1beta1.Cluster, hostServerIP string, port int) (*clientcmdapi.Config, error) { - bootstrapData, err := bootstrap.GetFromSecret(ctx, client, cluster) + bootstrapData, err := bootstrap.LoadFromSecret(ctx, client, cluster) if err != nil { return nil, err } diff --git a/pkg/k3s/bootstrap.go b/pkg/k3s/bootstrap.go new file mode 100644 index 00000000..10399d49 --- /dev/null +++ b/pkg/k3s/bootstrap.go @@ -0,0 +1,89 @@ +package k3s + +import ( + "encoding/base64" + "fmt" + "net/http" +) + +type BootstrapData struct { + ServerCA cert `json:"serverCA"` + ServerCAKey cert `json:"serverCAKey"` + ClientCA cert `json:"clientCA"` + ClientCAKey cert `json:"clientCAKey"` + ETCDServerCA cert `json:"etcdServerCA"` + ETCDServerCAKey cert `json:"etcdServerCAKey"` +} + +type cert struct { + Timestamp string + Content string +} + +func GetServerBootstrap(c *Client) (*BootstrapData, error) { + endpoint := "/v1-k3s/server-bootstrap" + + bootstrap, err := do[BootstrapData](c, endpoint, "server", http.MethodGet) + if err != nil { + return nil, err + } + + // we still need to decode each certs since the bootstrap data endpoint base64 encode each cert + if err := decode(&bootstrap); err != nil { + return nil, fmt.Errorf("failed to decode bootstrap secret: %w", err) + } + + return &bootstrap, nil +} + +func decode(data *BootstrapData) error { + // client-ca + decoded, err := base64.StdEncoding.DecodeString(data.ClientCA.Content) + if err != nil { + return err + } + + data.ClientCA.Content = string(decoded) + + // client-ca-key + decoded, err = base64.StdEncoding.DecodeString(data.ClientCAKey.Content) + if err != nil { + return err + } + + data.ClientCAKey.Content = string(decoded) + + // server-ca + decoded, err = base64.StdEncoding.DecodeString(data.ServerCA.Content) + if err != nil { + return err + } + + data.ServerCA.Content = string(decoded) + + // server-ca-key + decoded, err = base64.StdEncoding.DecodeString(data.ServerCAKey.Content) + if err != nil { + return err + } + + data.ServerCAKey.Content = string(decoded) + + // etcd-ca + decoded, err = base64.StdEncoding.DecodeString(data.ETCDServerCA.Content) + if err != nil { + return err + } + + data.ETCDServerCA.Content = string(decoded) + + // etcd-ca-key + decoded, err = base64.StdEncoding.DecodeString(data.ETCDServerCAKey.Content) + if err != nil { + return err + } + + data.ETCDServerCAKey.Content = string(decoded) + + return nil +} diff --git a/pkg/k3s/certs.go b/pkg/k3s/certs.go new file mode 100644 index 00000000..072b43aa --- /dev/null +++ b/pkg/k3s/certs.go @@ -0,0 +1,22 @@ +package k3s + +import ( + "crypto/tls" + "net/http" +) + +func GetServingKubeletCrt(c *Client) (*tls.Certificate, error) { + endpoint := "/v1-k3s/serving-kubelet.crt" + + tlsCrtData, err := c.do(endpoint, "node", http.MethodGet) + if err != nil { + return nil, err + } + + tlsCrt, err := tls.X509KeyPair(tlsCrtData, tlsCrtData) + if err != nil { + return nil, err + } + + return &tlsCrt, nil +} diff --git a/pkg/k3s/client.go b/pkg/k3s/client.go new file mode 100644 index 00000000..ae706ad4 --- /dev/null +++ b/pkg/k3s/client.go @@ -0,0 +1,124 @@ +package k3s + +import ( + "crypto/tls" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "strings" + "syscall" + "time" +) + +type ClientConfig struct { + AgentIP string + NodeName string + PodIP string + ServerIP string + Token string +} +type Client struct { + config ClientConfig + httpClient *http.Client + staticHeaders http.Header +} + +var ErrServerNotReady = errors.New("server not ready") + +func New(config ClientConfig) *Client { + httpClient := &http.Client{ + Transport: http.DefaultTransport, + Timeout: 5 * time.Second, + } + + // skip TLS verify for k3s server + if transport, ok := httpClient.Transport.(*http.Transport); ok { + transport.TLSClientConfig = &tls.Config{ + // This is insecure because the K3s CA hasn't been setup yet. + InsecureSkipVerify: true, + } + } + + headers := http.Header{} + + if config.Token != "" { + headers.Set("k3s-Node-Password", config.Token) + } + + if config.NodeName != "" { + headers.Set("k3s-Node-Name", config.NodeName) + } + + var nodeIPs []string + if config.AgentIP != "" { + nodeIPs = append(nodeIPs, config.AgentIP) + } + + if config.PodIP != "" { + nodeIPs = append(nodeIPs, config.PodIP) + } + + if len(nodeIPs) > 0 { + headers.Set("k3s-Node-IP", strings.Join(nodeIPs, ",")) + } + + return &Client{ + httpClient: httpClient, + config: config, + staticHeaders: headers, + } +} + +func do[T any](c *Client, endpoint, user, method string) (T, error) { + var response T + + respBody, err := c.do(endpoint, user, method) + if err != nil { + return response, err + } + + // unmarshal the json data to the generic struct + if err := json.Unmarshal(respBody, &response); err != nil { + return response, err + } + + return response, nil +} + +func (c *Client) do(endpoint, user, method string) ([]byte, error) { + url := "https://" + c.config.ServerIP + endpoint + + req, err := http.NewRequest(method, url, nil) + if err != nil { + return nil, err + } + + req.SetBasicAuth(user, c.config.Token) + + for headerName, headerValues := range c.staticHeaders { + for _, headerValue := range headerValues { + req.Header.Add(headerName, headerValue) + } + } + + resp, err := c.httpClient.Do(req) + if err != nil { + if errors.Is(err, syscall.ECONNREFUSED) { + return nil, ErrServerNotReady + } + + return nil, err + } + + if resp.StatusCode >= http.StatusBadRequest { + return nil, fmt.Errorf("failed executing '%s' request to k3s server: status code: %s", endpoint, resp.Status) + } + + defer func() { + _ = resp.Body.Close() + }() + + return io.ReadAll(resp.Body) +}