Refactor bootstrap data management (#869)

* Refactor bootstrap data management

* Change errors from errors.New to fmt.Errorf

Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>

k3k-kubelet/kubelet.go

@@ -2,8 +2,7 @@ package main
 
 import (
 	"context"
-	"crypto/x509"
-	"errors"
+	"fmt"
 	"path"
 	"time"
 
@@ -14,7 +13,6 @@ import (
 	"github.com/virtual-kubelet/virtual-kubelet/node/nodeutil"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
-	"k8s.io/apiserver/pkg/authentication/user"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
@@ -27,7 +25,6 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
-	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
 	ctrl "sigs.k8s.io/controller-runtime"
 	ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client"
 	ctrlserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
@@ -36,9 +33,6 @@ import (
 	"github.com/rancher/k3k/k3k-kubelet/provider"
 	"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
 	"github.com/rancher/k3k/pkg/controller"
-	"github.com/rancher/k3k/pkg/controller/certs"
-	"github.com/rancher/k3k/pkg/controller/cluster/server"
-	"github.com/rancher/k3k/pkg/controller/cluster/server/bootstrap"
 )
 
 var baseScheme = runtime.NewScheme()
@@ -82,7 +76,7 @@ func newKubelet(ctx context.Context, c *config) (*kubelet, error) {
 		return nil, err
 	}
 
-	virtConfig, err := virtRestConfig(ctx, c.VirtKubeconfig, hostClient, c.ClusterName, c.ClusterNamespace, c.Token)
+	virtConfig, err := virtRestConfig(ctx, c.VirtKubeconfig, hostClient, c.ClusterName, c.ClusterNamespace)
 	if err != nil {
 		return nil, err
 	}
@@ -115,13 +109,13 @@ func newKubelet(ctx context.Context, c *config) (*kubelet, error) {
 		},
 	})
 	if err != nil {
-		return nil, errors.New("unable to create controller-runtime mgr for host cluster: " + err.Error())
+		return nil, fmt.Errorf("unable to create controller-runtime mgr for host cluster: %w", err)
 	}
 
 	// virtual client will only use core types (for now), no need to add anything other than the basics
 	virtualScheme := runtime.NewScheme()
 	if err := clientgoscheme.AddToScheme(virtualScheme); err != nil {
-		return nil, errors.New("unable to add client go types to virtual cluster scheme: " + err.Error())
+		return nil, fmt.Errorf("unable to add client go types to virtual cluster scheme: %w", err)
 	}
 
 	virtualMgr, err := ctrl.NewManager(virtConfig, manager.Options{
@@ -134,7 +128,7 @@ func newKubelet(ctx context.Context, c *config) (*kubelet, error) {
 		},
 	})
 	if err != nil {
-		return nil, errors.New("unable to create controller-runtime mgr for virtual cluster: " + err.Error())
+		return nil, fmt.Errorf("unable to create controller-runtime mgr for virtual cluster: %w", err)
 	}
 
 	controllerName := c.AgentHostname
@@ -144,12 +138,12 @@ func newKubelet(ctx context.Context, c *config) (*kubelet, error) {
 	virtEventRecorder := eb.NewRecorder(virtualScheme, corev1.EventSource{Component: path.Join(controllerName, "pod-controller")})
 
 	if err := addControllers(ctx, hostMgr, virtualMgr, c, hostClient, virtEventRecorder); err != nil {
-		return nil, errors.New("failed to add controller: " + err.Error())
+		return nil, fmt.Errorf("failed to add controller: %w", err)
 	}
 
 	clusterIP, err := clusterIP(ctx, c.ServiceName, c.ClusterNamespace, hostClient)
 	if err != nil {
-		return nil, errors.New("failed to extract the clusterIP for the server service: " + err.Error())
+		return nil, fmt.Errorf("failed to extract the clusterIP for the server service: %w", err)
 	}
 
 	// get the cluster's DNS IP to be injected to pods
@@ -157,12 +151,12 @@ func newKubelet(ctx context.Context, c *config) (*kubelet, error) {
 
 	dnsName := controller.SafeConcatNameWithPrefix(c.ClusterName, "kube-dns")
 	if err := hostClient.Get(ctx, types.NamespacedName{Name: dnsName, Namespace: c.ClusterNamespace}, &dnsService); err != nil {
-		return nil, errors.New("failed to get the DNS service for the cluster: " + err.Error())
+		return nil, fmt.Errorf("failed to get the DNS service for the cluster: %w", err)
 	}
 
 	var virtualCluster v1beta1.Cluster
 	if err := hostClient.Get(ctx, types.NamespacedName{Name: c.ClusterName, Namespace: c.ClusterNamespace}, &virtualCluster); err != nil {
-		return nil, errors.New("failed to get virtualCluster spec: " + err.Error())
+		return nil, fmt.Errorf("failed to get virtualCluster spec: %w", err)
 	}
 
 	return &kubelet{
@@ -245,7 +239,7 @@ func (k *kubelet) newProviderFunc(cfg config) nodeutil.NewProviderFunc {
 	return func(pc nodeutil.ProviderConfig) (nodeutil.Provider, node.NodeProvider, error) {
 		utilProvider, err := provider.New(*k.hostConfig, k.hostMgr, k.virtualMgr, k.logger, cfg.ClusterNamespace, cfg.ClusterName, cfg.ServerIP, k.dnsIP, cfg.AgentHostname)
 		if err != nil {
-			return nil, nil, errors.New("unable to make nodeutil provider: " + err.Error())
+			return nil, nil, fmt.Errorf("unable to make nodeutil provider: %w", err)
 		}
 
 		err = provider.ConfigureNode(
@@ -265,76 +259,32 @@ func (k *kubelet) newProviderFunc(cfg config) nodeutil.NewProviderFunc {
 	}
 }
 
-func virtRestConfig(ctx context.Context, virtualConfigPath string, hostClient ctrlruntimeclient.Client, clusterName, clusterNamespace, token string) (*rest.Config, error) {
+func virtRestConfig(ctx context.Context, virtualConfigPath string, hostClient ctrlruntimeclient.Client, clusterName, clusterNamespace string) (*rest.Config, error) {
 	if virtualConfigPath != "" {
 		return clientcmd.BuildConfigFromFlags("", virtualConfigPath)
 	}
-	// virtual kubeconfig file is empty, trying to fetch the k3k cluster kubeconfig
-	var cluster v1beta1.Cluster
-	if err := hostClient.Get(ctx, types.NamespacedName{Namespace: clusterNamespace, Name: clusterName}, &cluster); err != nil {
-		return nil, err
+
+	var clusterKubeConfig corev1.Secret
+
+	kubeconfigSecretName := types.NamespacedName{
+		Name:      controller.SafeConcatNameWithPrefix(clusterName, "kubeconfig"),
+		Namespace: clusterNamespace,
 	}
 
-	endpoint := server.ServiceName(cluster.Name) + "." + cluster.Namespace
-
-	var b *bootstrap.ControlRuntimeBootstrap
-
 	if err := retry.OnError(controller.Backoff, func(err error) bool {
 		return err != nil
 	}, func() error {
-		var err error
-
-		b, err = bootstrap.DecodedBootstrap(token, endpoint)
-		logger.Error(err, "decoded bootstrap")
-
-		return err
+		return hostClient.Get(ctx, kubeconfigSecretName, &clusterKubeConfig)
 	}); err != nil {
-		return nil, errors.New("unable to decode bootstrap: " + err.Error())
+		return nil, fmt.Errorf("unable to decode bootstrap: %w", err)
 	}
 
-	adminCert, adminKey, err := certs.CreateClientCertKey(
-		controller.AdminCommonName,
-		[]string{user.SystemPrivilegedGroup},
-		nil, []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
-		time.Hour*24*time.Duration(356),
-		b.ClientCA.Content,
-		b.ClientCAKey.Content,
-	)
+	restConfig, err := clientcmd.RESTConfigFromKubeConfig(clusterKubeConfig.Data["kubeconfig.yaml"])
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("failed to create config from kubeconfig file: %w", err)
 	}
 
-	url := "https://" + server.ServiceName(cluster.Name)
-
-	kubeconfigData, err := kubeconfigBytes(url, []byte(b.ServerCA.Content), adminCert, adminKey)
-	if err != nil {
-		return nil, err
-	}
-
-	return clientcmd.RESTConfigFromKubeConfig(kubeconfigData)
-}
-
-func kubeconfigBytes(url string, serverCA, clientCert, clientKey []byte) ([]byte, error) {
-	config := clientcmdapi.NewConfig()
-
-	cluster := clientcmdapi.NewCluster()
-	cluster.CertificateAuthorityData = serverCA
-	cluster.Server = url
-
-	authInfo := clientcmdapi.NewAuthInfo()
-	authInfo.ClientCertificateData = clientCert
-	authInfo.ClientKeyData = clientKey
-
-	context := clientcmdapi.NewContext()
-	context.AuthInfo = "default"
-	context.Cluster = "default"
-
-	config.Clusters["default"] = cluster
-	config.AuthInfos["default"] = authInfo
-	config.Contexts["default"] = context
-	config.CurrentContext = "default"
-
-	return clientcmd.Write(*config)
+	return restConfig, nil
 }
 
 func addControllers(ctx context.Context, hostMgr, virtualMgr manager.Manager, c *config, hostClient ctrlruntimeclient.Client, virtEventRecorder record.EventRecorder) error {
@@ -350,39 +300,39 @@ func addControllers(ctx context.Context, hostMgr, virtualMgr manager.Manager, c
 	}
 
 	if err := syncer.AddConfigMapSyncer(ctx, virtualMgr, hostMgr, c.ClusterName, c.ClusterNamespace); err != nil {
-		return errors.New("failed to add configmap global syncer: " + err.Error())
+		return fmt.Errorf("failed to add configmap global syncer: %w", err)
 	}
 
 	if err := syncer.AddSecretSyncer(ctx, virtualMgr, hostMgr, c.ClusterName, c.ClusterNamespace); err != nil {
-		return errors.New("failed to add secret global syncer: " + err.Error())
+		return fmt.Errorf("failed to add secret global syncer: %w", err)
 	}
 
 	logger.Info("adding service syncer controller")
 
 	if err := syncer.AddServiceSyncer(ctx, virtualMgr, hostMgr, c.ClusterName, c.ClusterNamespace); err != nil {
-		return errors.New("failed to add service syncer controller: " + err.Error())
+		return fmt.Errorf("failed to add service syncer controller: %w", err)
 	}
 
 	logger.Info("adding ingress syncer controller")
 
 	if err := syncer.AddIngressSyncer(ctx, virtualMgr, hostMgr, c.ClusterName, c.ClusterNamespace); err != nil {
-		return errors.New("failed to add ingress syncer controller: " + err.Error())
+		return fmt.Errorf("failed to add ingress syncer controller: %w", err)
 	}
 
 	logger.Info("adding pvc syncer controller")
 
 	if err := syncer.AddPVCSyncer(ctx, virtualMgr, hostMgr, c.ClusterName, c.ClusterNamespace); err != nil {
-		return errors.New("failed to add pvc syncer controller: " + err.Error())
+		return fmt.Errorf("failed to add pvc syncer controller: %w", err)
 	}
 
 	logger.Info("adding priorityclass controller")
 
 	if err := syncer.AddPriorityClassSyncer(ctx, virtualMgr, hostMgr, c.ClusterName, c.ClusterNamespace); err != nil {
-		return errors.New("failed to add priorityclass controller: " + err.Error())
+		return fmt.Errorf("failed to add priorityclass controller: %w", err)
 	}
 
 	if err := syncer.AddEventSyncer(ctx, virtualMgr, hostMgr, c.ClusterName, c.ClusterNamespace, virtEventRecorder); err != nil {
-		return errors.New("failed to add event syncer controller: " + err.Error())
+		return fmt.Errorf("failed to add event syncer controller: %w", err)
 	}
 
 	return nil

k3k-kubelet/node.go

@@ -2,27 +2,21 @@ package main
 
 import (
 	"crypto/tls"
-	"crypto/x509"
-	"errors"
 	"fmt"
-	"net"
 	"net/http"
 
 	"github.com/virtual-kubelet/virtual-kubelet/node/nodeutil"
 	"k8s.io/client-go/util/retry"
 
-	certutil "github.com/rancher/dynamiclistener/cert"
-
 	"github.com/rancher/k3k/pkg/controller"
-	"github.com/rancher/k3k/pkg/controller/certs"
 	"github.com/rancher/k3k/pkg/controller/cluster/server"
-	"github.com/rancher/k3k/pkg/controller/cluster/server/bootstrap"
+	"github.com/rancher/k3k/pkg/k3s"
 )
 
 func (k *kubelet) registerNode(agentIP, podIP string, cfg config) error {
-	tlsConfig, err := loadTLSConfig(cfg, k.name, k.token, agentIP, podIP)
+	tlsConfig, err := loadTLSConfig(cfg, k.token, agentIP, podIP)
 	if err != nil {
-		return errors.New("unable to get tls config: " + err.Error())
+		return fmt.Errorf("unable to get tls config: %w", err)
 	}
 
 	mux := http.NewServeMux()
@@ -39,7 +33,7 @@ func (k *kubelet) registerNode(agentIP, podIP string, cfg config) error {
 		},
 	)
 	if err != nil {
-		return errors.New("unable to start kubelet: " + err.Error())
+		return fmt.Errorf("unable to start kubelet: %w", err)
 	}
 
 	k.node = node
@@ -63,56 +57,35 @@ func nodeOpt(mux *http.ServeMux, tlsConfig *tls.Config, port int) nodeutil.NodeO
 	}
 }
 
-func loadTLSConfig(cfg config, nodeName, token, agentIP, podIP string) (*tls.Config, error) {
-	var b *bootstrap.ControlRuntimeBootstrap
+// loadTLSConfig function will request kubelet serving crt from k3s server and will use it to
+// register a new node to the server, note that we use serving cert to allow adding IPSans to
+// the certificate request
+func loadTLSConfig(cfg config, token, agentIP, podIP string) (*tls.Config, error) {
+	serviceName := fmt.Sprintf("%s.%s", server.ServiceName(cfg.ClusterName), cfg.ClusterNamespace)
 
-	endpoint := fmt.Sprintf("%s.%s", server.ServiceName(cfg.ClusterName), cfg.ClusterNamespace)
+	client := k3s.New(k3s.ClientConfig{
+		ServerIP: serviceName,
+		Token:    token,
+		AgentIP:  agentIP,
+		PodIP:    podIP,
+		NodeName: controller.SafeConcatName(cfg.ClusterName, "server-0"),
+	})
+
+	var tlsCrt *tls.Certificate
 
 	if err := retry.OnError(controller.Backoff, func(err error) bool {
-		return err != nil
+		return err == k3s.ErrServerNotReady
 	}, func() error {
 		var err error
 
-		b, err = bootstrap.DecodedBootstrap(token, endpoint)
+		tlsCrt, err = k3s.GetServingKubeletCrt(client)
 
 		return err
 	}); err != nil {
-		return nil, errors.New("unable to decode bootstrap: " + err.Error())
+		return nil, fmt.Errorf("unable to request serving kubelet certificate: %w", err)
 	}
 
-	altNames := certutil.AltNames{
-		DNSNames: []string{cfg.AgentHostname},
-		IPs: []net.IP{
-			net.ParseIP(agentIP),
-			net.ParseIP(podIP),
-		},
-	}
-
-	cert, key, err := certs.CreateClientCertKey(nodeName, nil, &altNames, []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, 0, b.ServerCA.Content, b.ServerCAKey.Content)
-	if err != nil {
-		return nil, errors.New("unable to get cert and key: " + err.Error())
-	}
-
-	clientCert, err := tls.X509KeyPair(cert, key)
-	if err != nil {
-		return nil, errors.New("unable to get key pair: " + err.Error())
-	}
-
-	// create rootCA CertPool
-	certs, err := certutil.ParseCertsPEM([]byte(b.ServerCA.Content))
-	if err != nil {
-		return nil, errors.New("unable to create ca certs: " + err.Error())
-	}
-
-	if len(certs) < 1 {
-		return nil, errors.New("ca cert is not parsed correctly")
-	}
-
-	pool := x509.NewCertPool()
-	pool.AddCert(certs[0])
-
 	return &tls.Config{
-		RootCAs:      pool,
-		Certificates: []tls.Certificate{clientCert},
+		Certificates: []tls.Certificate{*tlsCrt},
 	}, nil
 }

pkg/controller/cluster/cluster.go

@@ -42,6 +42,7 @@ import (
 	"github.com/rancher/k3k/pkg/controller/cluster/server/bootstrap"
 	"github.com/rancher/k3k/pkg/controller/kubeconfig"
 	"github.com/rancher/k3k/pkg/controller/policy"
+	"github.com/rancher/k3k/pkg/k3s"
 )
 
 const (
@@ -293,7 +294,7 @@ func (c *ClusterReconciler) Reconcile(ctx context.Context, req reconcile.Request
 
 	// if there was an error during the reconciliation, return
 	if reconcilerErr != nil {
-		if errors.Is(reconcilerErr, bootstrap.ErrServerNotReady) {
+		if errors.Is(reconcilerErr, k3s.ErrServerNotReady) {
 			log.V(1).Info("Server not ready, requeueing")
 			return reconcile.Result{RequeueAfter: time.Second * 10}, nil
 		}
@@ -449,31 +450,12 @@ func (c *ClusterReconciler) ensureBootstrapSecret(ctx context.Context, cluster *
 	log := ctrl.LoggerFrom(ctx)
 	log.V(1).Info("Ensuring bootstrap secret")
 
-	bootstrapData, err := bootstrap.GenerateBootstrapData(ctx, cluster, serviceIP, token)
+	data, err := bootstrap.Fetch(ctx, serviceIP, token)
 	if err != nil {
 		return err
 	}
 
-	bootstrapSecret := &corev1.Secret{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      controller.SafeConcatNameWithPrefix(cluster.Name, "bootstrap"),
-			Namespace: cluster.Namespace,
-		},
-	}
-
-	_, err = controllerutil.CreateOrUpdate(ctx, c.Client, bootstrapSecret, func() error {
-		if err := controllerutil.SetControllerReference(cluster, bootstrapSecret, c.Scheme); err != nil {
-			return err
-		}
-
-		bootstrapSecret.Data = map[string][]byte{
-			"bootstrap": bootstrapData,
-		}
-
-		return nil
-	})
-
-	return err
+	return bootstrap.SaveToSecret(ctx, c.Client, c.Scheme, cluster, data)
 }
 
 // ensureKubeconfigSecret will create or update the Secret containing the kubeconfig data from the k3s server

pkg/controller/cluster/server/bootstrap/bootstrap.go

@@ -2,167 +2,68 @@ package bootstrap
 
 import (
 	"context"
-	"crypto/tls"
-	"encoding/base64"
 	"encoding/json"
 	"errors"
-	"fmt"
-	"net/http"
-	"syscall"
-	"time"
 
+	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 
 	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	ctrl "sigs.k8s.io/controller-runtime"
 
 	"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
 	"github.com/rancher/k3k/pkg/controller"
+	"github.com/rancher/k3k/pkg/k3s"
 )
 
-var ErrServerNotReady = errors.New("server not ready")
+const (
+	TLSDir = "/var/lib/rancher/k3s/server/tls/"
+)
 
-type ControlRuntimeBootstrap struct {
-	ServerCA        content `json:"serverCA"`
-	ServerCAKey     content `json:"serverCAKey"`
-	ClientCA        content `json:"clientCA"`
-	ClientCAKey     content `json:"clientCAKey"`
-	ETCDServerCA    content `json:"etcdServerCA"`
-	ETCDServerCAKey content `json:"etcdServerCAKey"`
+// Fetch requests bootstrap data from k3s using the token and decodes it,
+// to avoid double encoding when stored as secret.
+func Fetch(ctx context.Context, ip, token string) (*k3s.BootstrapData, error) {
+	log := ctrl.LoggerFrom(ctx)
+	log.V(1).Info("Fetching bootstrap data from K3s API")
+
+	return fetchFromK3sServer(ip, token)
 }
 
-type content struct {
-	Timestamp string
-	Content   string
-}
-
-// Generate generates the bootstrap for the cluster:
-// 1- use the server token to get the bootstrap data from k3s
-// 2- save the bootstrap data as a secret
-func GenerateBootstrapData(ctx context.Context, cluster *v1beta1.Cluster, ip, token string) ([]byte, error) {
-	bootstrap, err := requestBootstrap(token, ip)
+// SaveToSecret marshals the bootstrap data and stores it in a Secret owned by the cluster,
+// creating the Secret if it does not exist or updating it otherwise.
+func SaveToSecret(ctx context.Context, c client.Client, scheme *runtime.Scheme, cluster *v1beta1.Cluster, data *k3s.BootstrapData) error {
+	bootstrapData, err := json.Marshal(data)
 	if err != nil {
-		return nil, fmt.Errorf("failed to request bootstrap secret: %w", err)
+		return err
 	}
 
-	if err := decodeBootstrap(bootstrap); err != nil {
-		return nil, fmt.Errorf("failed to decode bootstrap secret: %w", err)
-	}
-
-	return json.Marshal(bootstrap)
-}
-
-func requestBootstrap(token, serverIP string) (*ControlRuntimeBootstrap, error) {
-	url := "https://" + serverIP + "/v1-k3s/server-bootstrap"
-
-	client := http.Client{
-		Transport: &http.Transport{
-			TLSClientConfig: &tls.Config{
-				InsecureSkipVerify: true,
-			},
+	secret := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      controller.SafeConcatNameWithPrefix(cluster.Name, "bootstrap"),
+			Namespace: cluster.Namespace,
 		},
-		Timeout: 5 * time.Second,
 	}
 
-	req, err := http.NewRequest(http.MethodGet, url, nil)
-	if err != nil {
-		return nil, err
-	}
-
-	req.Header.Add("Authorization", "Basic "+basicAuth("server", token))
-
-	resp, err := client.Do(req)
-	if err != nil {
-		if errors.Is(err, syscall.ECONNREFUSED) {
-			return nil, ErrServerNotReady
+	_, err = controllerutil.CreateOrUpdate(ctx, c, secret, func() error {
+		if err := controllerutil.SetControllerReference(cluster, secret, scheme); err != nil {
+			return err
 		}
 
-		return nil, err
-	}
+		secret.Data = map[string][]byte{
+			"bootstrap": bootstrapData,
+		}
 
-	defer func() {
-		_ = resp.Body.Close()
-	}()
+		return nil
+	})
 
-	var runtimeBootstrap ControlRuntimeBootstrap
-	if err := json.NewDecoder(resp.Body).Decode(&runtimeBootstrap); err != nil {
-		return nil, err
-	}
-
-	return &runtimeBootstrap, nil
+	return err
 }
 
-func basicAuth(username, password string) string {
-	auth := username + ":" + password
-	return base64.StdEncoding.EncodeToString([]byte(auth))
-}
-
-func decodeBootstrap(bootstrap *ControlRuntimeBootstrap) error {
-	// client-ca
-	decoded, err := base64.StdEncoding.DecodeString(bootstrap.ClientCA.Content)
-	if err != nil {
-		return err
-	}
-
-	bootstrap.ClientCA.Content = string(decoded)
-
-	// client-ca-key
-	decoded, err = base64.StdEncoding.DecodeString(bootstrap.ClientCAKey.Content)
-	if err != nil {
-		return err
-	}
-
-	bootstrap.ClientCAKey.Content = string(decoded)
-
-	// server-ca
-	decoded, err = base64.StdEncoding.DecodeString(bootstrap.ServerCA.Content)
-	if err != nil {
-		return err
-	}
-
-	bootstrap.ServerCA.Content = string(decoded)
-
-	// server-ca-key
-	decoded, err = base64.StdEncoding.DecodeString(bootstrap.ServerCAKey.Content)
-	if err != nil {
-		return err
-	}
-
-	bootstrap.ServerCAKey.Content = string(decoded)
-
-	// etcd-ca
-	decoded, err = base64.StdEncoding.DecodeString(bootstrap.ETCDServerCA.Content)
-	if err != nil {
-		return err
-	}
-
-	bootstrap.ETCDServerCA.Content = string(decoded)
-
-	// etcd-ca-key
-	decoded, err = base64.StdEncoding.DecodeString(bootstrap.ETCDServerCAKey.Content)
-	if err != nil {
-		return err
-	}
-
-	bootstrap.ETCDServerCAKey.Content = string(decoded)
-
-	return nil
-}
-
-func DecodedBootstrap(token, ip string) (*ControlRuntimeBootstrap, error) {
-	bootstrap, err := requestBootstrap(token, ip)
-	if err != nil {
-		return nil, err
-	}
-
-	if err := decodeBootstrap(bootstrap); err != nil {
-		return nil, err
-	}
-
-	return bootstrap, nil
-}
-
-func GetFromSecret(ctx context.Context, client client.Client, cluster *v1beta1.Cluster) (*ControlRuntimeBootstrap, error) {
+// LoadFromSecret reads the bootstrap data of a certain cluster and returns the decoded content.
+func LoadFromSecret(ctx context.Context, client client.Client, cluster *v1beta1.Cluster) (*k3s.BootstrapData, error) {
 	key := types.NamespacedName{
 		Name:      controller.SafeConcatNameWithPrefix(cluster.Name, "bootstrap"),
 		Namespace: cluster.Namespace,
@@ -178,9 +79,18 @@ func GetFromSecret(ctx context.Context, client client.Client, cluster *v1beta1.C
 		return nil, errors.New("empty bootstrap")
 	}
 
-	var bootstrap ControlRuntimeBootstrap
+	var bootstrap k3s.BootstrapData
 
 	err := json.Unmarshal(bootstrapData, &bootstrap)
 
 	return &bootstrap, err
 }
+
+func fetchFromK3sServer(serviceIP, token string) (*k3s.BootstrapData, error) {
+	client := k3s.New(k3s.ClientConfig{
+		ServerIP: serviceIP,
+		Token:    token,
+	})
+
+	return k3s.GetServerBootstrap(client)
+}

pkg/controller/cluster/server/server.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"path/filepath"
 	"sort"
 	"strings"
 	"text/template"
@@ -26,12 +27,22 @@ import (
 )
 
 const (
-	serverName     = "server"
-	configName     = "server-config"
-	initConfigName = "init-server-config"
+	initConfigName   = "init-server-config"
+	configName       = "server-config"
+	serverName       = "server"
+	k3sInitConfigDir = "/opt/rancher/k3s/init"
+	k3sConfigDir     = "/opt/rancher/k3s/server"
+	k3sRunDir        = "/run"
+	k3sCNIDir        = "/var/lib/cni"
+	k3sKubeletDir    = "/var/lib/kubelet"
+	k3sDataDir       = "/var/lib/rancher/k3s"
+	k3sETCDDataDir   = "/var/lib/rancher/k3s/server/db/etcd"
+	k3sManifestDir   = "/var/lib/rancher/k3s/server/manifests"
+	k3sTLSDir        = "/var/lib/rancher/k3s/server/tls"
+	k3sLogDir        = "/var/log"
+	k3sVarRunDir     = "/var/run"
 )
 
-// Server
 type Server struct {
 	cluster          *v1beta1.Cluster
 	client           client.Client
@@ -70,7 +81,7 @@ func (s *Server) podSpec(ctx context.Context, image, name string, persistent boo
 		PriorityClassName: s.cluster.Spec.PriorityClass,
 		Volumes: []corev1.Volume{
 			{
-				Name: "initconfig",
+				Name: "init-config",
 				VolumeSource: corev1.VolumeSource{
 					Secret: &corev1.SecretVolumeSource{
 						SecretName: configSecretName(s.cluster.Name, true),
@@ -104,25 +115,25 @@ func (s *Server) podSpec(ctx context.Context, image, name string, persistent boo
 				},
 			},
 			{
-				Name: "varrun",
+				Name: "var-run",
 				VolumeSource: corev1.VolumeSource{
 					EmptyDir: &corev1.EmptyDirVolumeSource{},
 				},
 			},
 			{
-				Name: "varlibcni",
+				Name: "var-lib-cni",
 				VolumeSource: corev1.VolumeSource{
 					EmptyDir: &corev1.EmptyDirVolumeSource{},
 				},
 			},
 			{
-				Name: "varlog",
+				Name: "var-log",
 				VolumeSource: corev1.VolumeSource{
 					EmptyDir: &corev1.EmptyDirVolumeSource{},
 				},
 			},
 			{
-				Name: "varlibkubelet",
+				Name: "var-lib-kubelet",
 				VolumeSource: corev1.VolumeSource{
 					EmptyDir: &corev1.EmptyDirVolumeSource{},
 				},
@@ -154,42 +165,42 @@ func (s *Server) podSpec(ctx context.Context, image, name string, persistent boo
 				VolumeMounts: []corev1.VolumeMount{
 					{
 						Name:      "config",
-						MountPath: "/opt/rancher/k3s/server",
+						MountPath: k3sConfigDir,
 						ReadOnly:  false,
 					},
 					{
-						Name:      "initconfig",
-						MountPath: "/opt/rancher/k3s/init",
+						Name:      "init-config",
+						MountPath: k3sInitConfigDir,
 						ReadOnly:  false,
 					},
 					{
 						Name:      "run",
-						MountPath: "/run",
+						MountPath: k3sRunDir,
 						ReadOnly:  false,
 					},
 					{
-						Name:      "varrun",
-						MountPath: "/var/run",
+						Name:      "var-run",
+						MountPath: k3sVarRunDir,
 						ReadOnly:  false,
 					},
 					{
-						Name:      "varlibcni",
-						MountPath: "/var/lib/cni",
+						Name:      "var-lib-cni",
+						MountPath: k3sCNIDir,
 						ReadOnly:  false,
 					},
 					{
-						Name:      "varlibkubelet",
-						MountPath: "/var/lib/kubelet",
+						Name:      "var-lib-kubelet",
+						MountPath: k3sKubeletDir,
 						ReadOnly:  false,
 					},
 					{
-						Name:      "varlibrancherk3s",
-						MountPath: "/var/lib/rancher/k3s",
+						Name:      "var-lib-rancher-k3s",
+						MountPath: k3sDataDir,
 						ReadOnly:  false,
 					},
 					{
-						Name:      "varlog",
-						MountPath: "/var/log",
+						Name:      "var-log",
+						MountPath: k3sLogDir,
 						ReadOnly:  false,
 					},
 				},
@@ -206,7 +217,7 @@ func (s *Server) podSpec(ctx context.Context, image, name string, persistent boo
 	podSpec.Containers[0].Command = cmd
 	if !persistent {
 		podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{
-			Name: "varlibrancherk3s",
+			Name: "var-lib-rancher-k3s",
 			VolumeSource: corev1.VolumeSource{
 				EmptyDir: &corev1.EmptyDirVolumeSource{},
 			},
@@ -405,7 +416,7 @@ func (s *Server) setupDynamicPersistence() corev1.PersistentVolumeClaim {
 			APIVersion: "v1",
 		},
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      "varlibrancherk3s",
+			Name:      "var-lib-rancher-k3s",
 			Namespace: s.cluster.Namespace,
 		},
 		Spec: corev1.PersistentVolumeClaimSpec{
@@ -441,9 +452,9 @@ func (s *Server) setupStartCommand() (string, error) {
 	}
 
 	if err := tmplCmd.Execute(&output, map[string]string{
-		"ETCD_DIR":      "/var/lib/rancher/k3s/server/db/etcd",
-		"INIT_CONFIG":   "/opt/rancher/k3s/init/config.yaml",
-		"SERVER_CONFIG": "/opt/rancher/k3s/server/config.yaml",
+		"ETCD_DIR":      k3sETCDDataDir,
+		"INIT_CONFIG":   filepath.Join(k3sInitConfigDir, "config.yaml"),
+		"SERVER_CONFIG": filepath.Join(k3sConfigDir, "config.yaml"),
 		"CLUSTER_MODE":  mode,
 		"K3K_MODE":      string(s.cluster.Spec.Mode),
 		"EXTRA_ARGS":    strings.Join(s.cluster.Spec.ServerArgs, " "),
@@ -511,7 +522,6 @@ func (s *Server) mountCACert(volumeName, certName, secretName string, subPathMou
 	)
 
 	// avoid re-adding secretName in case of combined secret
-
 	volume = &corev1.Volume{
 		Name: volumeName,
 		VolumeSource: corev1.VolumeSource{
@@ -528,22 +538,19 @@ func (s *Server) mountCACert(volumeName, certName, secretName string, subPathMou
 		mountFile = strings.TrimPrefix(certName, "etcd-")
 	}
 
-	// add the mount for the cert except for the service account token
-	if certName != "service" {
+	for _, crtOrKey := range []string{"crt", "key"} {
+		// skip adding cert mount for service account token
+		if certName == "service" && crtOrKey == "crt" {
+			continue
+		}
+
 		mounts = append(mounts, corev1.VolumeMount{
 			Name:      volumeName,
-			MountPath: fmt.Sprintf("/var/lib/rancher/k3s/server/tls%s/%s.crt", etcdPrefix, mountFile),
-			SubPath:   subPathMount + ".crt",
+			MountPath: filepath.Join(k3sTLSDir, etcdPrefix, mountFile) + "." + crtOrKey,
+			SubPath:   subPathMount + "." + crtOrKey,
 		})
 	}
 
-	// add the mount for the key
-	mounts = append(mounts, corev1.VolumeMount{
-		Name:      volumeName,
-		MountPath: fmt.Sprintf("/var/lib/rancher/k3s/server/tls%s/%s.key", etcdPrefix, mountFile),
-		SubPath:   subPathMount + ".key",
-	})
-
 	return volume, mounts
 }
 
@@ -603,7 +610,7 @@ func (s *Server) buildAddonsVolumes(ctx context.Context) ([]corev1.Volume, []cor
 
 		volumeMount := corev1.VolumeMount{
 			Name:      name,
-			MountPath: "/var/lib/rancher/k3s/server/manifests/" + addon.SecretRef,
+			MountPath: filepath.Join(k3sManifestDir, addon.SecretRef),
 			ReadOnly:  true,
 		}
 		mounts = append(mounts, volumeMount)

pkg/controller/cluster/statefulset.go

@@ -11,7 +11,6 @@ import (
 
 	"go.etcd.io/etcd/api/v3/v3rpc/rpctypes"
 	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/util/retry"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
@@ -32,6 +31,7 @@ import (
 	"github.com/rancher/k3k/pkg/controller/certs"
 	"github.com/rancher/k3k/pkg/controller/cluster/server"
 	"github.com/rancher/k3k/pkg/controller/cluster/server/bootstrap"
+	"github.com/rancher/k3k/pkg/k3s"
 )
 
 const (
@@ -44,7 +44,7 @@ type StatefulSetReconciler struct {
 	Scheme *runtime.Scheme
 }
 
-// Add adds a new controller to the manager
+// AddStatefulSetController adds a new statefulset controller to the manager
 func AddStatefulSetController(ctx context.Context, mgr manager.Manager, maxConcurrentReconciles int) error {
 	// initialize a new Reconciler
 	reconciler := StatefulSetReconciler{
@@ -180,21 +180,14 @@ func (p *StatefulSetReconciler) getETCDTLS(ctx context.Context, cluster *v1beta1
 	log := ctrl.LoggerFrom(ctx)
 	log.V(1).Info("Generating ETCD TLS client certificate", "cluster", cluster)
 
-	token, err := p.clusterToken(ctx, cluster)
-	if err != nil {
-		return nil, err
-	}
-
-	endpoint := server.ServiceName(cluster.Name) + "." + cluster.Namespace
-
-	var b *bootstrap.ControlRuntimeBootstrap
+	var b *k3s.BootstrapData
 
 	if err := retry.OnError(k3kcontroller.Backoff, func(err error) bool {
 		return true
 	}, func() error {
 		var err error
 
-		b, err = bootstrap.DecodedBootstrap(token, endpoint)
+		b, err = bootstrap.LoadFromSecret(ctx, p.Client, cluster)
 
 		return err
 	}); err != nil {
@@ -265,29 +258,6 @@ func removePeer(ctx context.Context, client *clientv3.Client, name, address stri
 	return nil
 }
 
-func (p *StatefulSetReconciler) clusterToken(ctx context.Context, cluster *v1beta1.Cluster) (string, error) {
-	var tokenSecret corev1.Secret
-
-	nn := types.NamespacedName{
-		Name:      TokenSecretName(cluster.Name),
-		Namespace: cluster.Namespace,
-	}
-
-	if cluster.Spec.TokenSecretRef != nil {
-		nn.Name = TokenSecretName(cluster.Name)
-	}
-
-	if err := p.Client.Get(ctx, nn, &tokenSecret); err != nil {
-		return "", err
-	}
-
-	if _, ok := tokenSecret.Data["token"]; !ok {
-		return "", fmt.Errorf("no token field in secret %s/%s", nn.Namespace, nn.Name)
-	}
-
-	return string(tokenSecret.Data["token"]), nil
-}
-
 func (p *StatefulSetReconciler) handleDeletion(ctx context.Context, sts *appsv1.StatefulSet) (ctrl.Result, error) {
 	log := ctrl.LoggerFrom(ctx)
 

pkg/controller/cluster/status.go

@@ -11,7 +11,7 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 
 	"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
-	"github.com/rancher/k3k/pkg/controller/cluster/server/bootstrap"
+	"github.com/rancher/k3k/pkg/k3s"
 )
 
 const (
@@ -57,7 +57,7 @@ func (c *ClusterReconciler) updateStatus(ctx context.Context, cluster *v1beta1.C
 		return
 	}
 
-	if errors.Is(reconcileErr, bootstrap.ErrServerNotReady) {
+	if errors.Is(reconcileErr, k3s.ErrServerNotReady) {
 		cluster.Status.Phase = v1beta1.ClusterProvisioning
 		meta.SetStatusCondition(&cluster.Status.Conditions, metav1.Condition{
 			Type:    ConditionReady,

pkg/controller/kubeconfig/kubeconfig.go

@@ -40,7 +40,7 @@ func New() *KubeConfig {
 }
 
 func (k *KubeConfig) Generate(ctx context.Context, client client.Client, cluster *v1beta1.Cluster, hostServerIP string, port int) (*clientcmdapi.Config, error) {
-	bootstrapData, err := bootstrap.GetFromSecret(ctx, client, cluster)
+	bootstrapData, err := bootstrap.LoadFromSecret(ctx, client, cluster)
 	if err != nil {
 		return nil, err
 	}

pkg/k3s/bootstrap.go

@@ -0,0 +1,89 @@
+package k3s
+
+import (
+	"encoding/base64"
+	"fmt"
+	"net/http"
+)
+
+type BootstrapData struct {
+	ServerCA        cert `json:"serverCA"`
+	ServerCAKey     cert `json:"serverCAKey"`
+	ClientCA        cert `json:"clientCA"`
+	ClientCAKey     cert `json:"clientCAKey"`
+	ETCDServerCA    cert `json:"etcdServerCA"`
+	ETCDServerCAKey cert `json:"etcdServerCAKey"`
+}
+
+type cert struct {
+	Timestamp string
+	Content   string
+}
+
+func GetServerBootstrap(c *Client) (*BootstrapData, error) {
+	endpoint := "/v1-k3s/server-bootstrap"
+
+	bootstrap, err := do[BootstrapData](c, endpoint, "server", http.MethodGet)
+	if err != nil {
+		return nil, err
+	}
+
+	// we still need to decode each certs since the bootstrap data endpoint base64 encode each cert
+	if err := decode(&bootstrap); err != nil {
+		return nil, fmt.Errorf("failed to decode bootstrap secret: %w", err)
+	}
+
+	return &bootstrap, nil
+}
+
+func decode(data *BootstrapData) error {
+	// client-ca
+	decoded, err := base64.StdEncoding.DecodeString(data.ClientCA.Content)
+	if err != nil {
+		return err
+	}
+
+	data.ClientCA.Content = string(decoded)
+
+	// client-ca-key
+	decoded, err = base64.StdEncoding.DecodeString(data.ClientCAKey.Content)
+	if err != nil {
+		return err
+	}
+
+	data.ClientCAKey.Content = string(decoded)
+
+	// server-ca
+	decoded, err = base64.StdEncoding.DecodeString(data.ServerCA.Content)
+	if err != nil {
+		return err
+	}
+
+	data.ServerCA.Content = string(decoded)
+
+	// server-ca-key
+	decoded, err = base64.StdEncoding.DecodeString(data.ServerCAKey.Content)
+	if err != nil {
+		return err
+	}
+
+	data.ServerCAKey.Content = string(decoded)
+
+	// etcd-ca
+	decoded, err = base64.StdEncoding.DecodeString(data.ETCDServerCA.Content)
+	if err != nil {
+		return err
+	}
+
+	data.ETCDServerCA.Content = string(decoded)
+
+	// etcd-ca-key
+	decoded, err = base64.StdEncoding.DecodeString(data.ETCDServerCAKey.Content)
+	if err != nil {
+		return err
+	}
+
+	data.ETCDServerCAKey.Content = string(decoded)
+
+	return nil
+}

pkg/k3s/certs.go

@@ -0,0 +1,22 @@
+package k3s
+
+import (
+	"crypto/tls"
+	"net/http"
+)
+
+func GetServingKubeletCrt(c *Client) (*tls.Certificate, error) {
+	endpoint := "/v1-k3s/serving-kubelet.crt"
+
+	tlsCrtData, err := c.do(endpoint, "node", http.MethodGet)
+	if err != nil {
+		return nil, err
+	}
+
+	tlsCrt, err := tls.X509KeyPair(tlsCrtData, tlsCrtData)
+	if err != nil {
+		return nil, err
+	}
+
+	return &tlsCrt, nil
+}

pkg/k3s/client.go

@@ -0,0 +1,124 @@
+package k3s
+
+import (
+	"crypto/tls"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+	"syscall"
+	"time"
+)
+
+type ClientConfig struct {
+	AgentIP  string
+	NodeName string
+	PodIP    string
+	ServerIP string
+	Token    string
+}
+type Client struct {
+	config        ClientConfig
+	httpClient    *http.Client
+	staticHeaders http.Header
+}
+
+var ErrServerNotReady = errors.New("server not ready")
+
+func New(config ClientConfig) *Client {
+	httpClient := &http.Client{
+		Transport: http.DefaultTransport,
+		Timeout:   5 * time.Second,
+	}
+
+	// skip TLS verify for k3s server
+	if transport, ok := httpClient.Transport.(*http.Transport); ok {
+		transport.TLSClientConfig = &tls.Config{
+			// This is insecure because the K3s CA hasn't been setup yet.
+			InsecureSkipVerify: true,
+		}
+	}
+
+	headers := http.Header{}
+
+	if config.Token != "" {
+		headers.Set("k3s-Node-Password", config.Token)
+	}
+
+	if config.NodeName != "" {
+		headers.Set("k3s-Node-Name", config.NodeName)
+	}
+
+	var nodeIPs []string
+	if config.AgentIP != "" {
+		nodeIPs = append(nodeIPs, config.AgentIP)
+	}
+
+	if config.PodIP != "" {
+		nodeIPs = append(nodeIPs, config.PodIP)
+	}
+
+	if len(nodeIPs) > 0 {
+		headers.Set("k3s-Node-IP", strings.Join(nodeIPs, ","))
+	}
+
+	return &Client{
+		httpClient:    httpClient,
+		config:        config,
+		staticHeaders: headers,
+	}
+}
+
+func do[T any](c *Client, endpoint, user, method string) (T, error) {
+	var response T
+
+	respBody, err := c.do(endpoint, user, method)
+	if err != nil {
+		return response, err
+	}
+
+	// unmarshal the json data to the generic struct
+	if err := json.Unmarshal(respBody, &response); err != nil {
+		return response, err
+	}
+
+	return response, nil
+}
+
+func (c *Client) do(endpoint, user, method string) ([]byte, error) {
+	url := "https://" + c.config.ServerIP + endpoint
+
+	req, err := http.NewRequest(method, url, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	req.SetBasicAuth(user, c.config.Token)
+
+	for headerName, headerValues := range c.staticHeaders {
+		for _, headerValue := range headerValues {
+			req.Header.Add(headerName, headerValue)
+		}
+	}
+
+	resp, err := c.httpClient.Do(req)
+	if err != nil {
+		if errors.Is(err, syscall.ECONNREFUSED) {
+			return nil, ErrServerNotReady
+		}
+
+		return nil, err
+	}
+
+	if resp.StatusCode >= http.StatusBadRequest {
+		return nil, fmt.Errorf("failed executing '%s' request to k3s server: status code: %s", endpoint, resp.Status)
+	}
+
+	defer func() {
+		_ = resp.Body.Close()
+	}()
+
+	return io.ReadAll(resp.Body)
+}