Refactor bootstrap data management (#869)

* Refactor bootstrap data management

* Change errors from errors.New to fmt.Errorf

Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>

pkg/controller/cluster/cluster.go

@@ -42,6 +42,7 @@ import (
 	"github.com/rancher/k3k/pkg/controller/cluster/server/bootstrap"
 	"github.com/rancher/k3k/pkg/controller/kubeconfig"
 	"github.com/rancher/k3k/pkg/controller/policy"
+	"github.com/rancher/k3k/pkg/k3s"
 )
 
 const (
@@ -293,7 +294,7 @@ func (c *ClusterReconciler) Reconcile(ctx context.Context, req reconcile.Request
 
 	// if there was an error during the reconciliation, return
 	if reconcilerErr != nil {
-		if errors.Is(reconcilerErr, bootstrap.ErrServerNotReady) {
+		if errors.Is(reconcilerErr, k3s.ErrServerNotReady) {
 			log.V(1).Info("Server not ready, requeueing")
 			return reconcile.Result{RequeueAfter: time.Second * 10}, nil
 		}
@@ -449,31 +450,12 @@ func (c *ClusterReconciler) ensureBootstrapSecret(ctx context.Context, cluster *
 	log := ctrl.LoggerFrom(ctx)
 	log.V(1).Info("Ensuring bootstrap secret")
 
-	bootstrapData, err := bootstrap.GenerateBootstrapData(ctx, cluster, serviceIP, token)
+	data, err := bootstrap.Fetch(ctx, serviceIP, token)
 	if err != nil {
 		return err
 	}
 
-	bootstrapSecret := &corev1.Secret{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      controller.SafeConcatNameWithPrefix(cluster.Name, "bootstrap"),
-			Namespace: cluster.Namespace,
-		},
-	}
-
-	_, err = controllerutil.CreateOrUpdate(ctx, c.Client, bootstrapSecret, func() error {
-		if err := controllerutil.SetControllerReference(cluster, bootstrapSecret, c.Scheme); err != nil {
-			return err
-		}
-
-		bootstrapSecret.Data = map[string][]byte{
-			"bootstrap": bootstrapData,
-		}
-
-		return nil
-	})
-
-	return err
+	return bootstrap.SaveToSecret(ctx, c.Client, c.Scheme, cluster, data)
 }
 
 // ensureKubeconfigSecret will create or update the Secret containing the kubeconfig data from the k3s server

pkg/controller/cluster/server/bootstrap/bootstrap.go

@@ -2,167 +2,68 @@ package bootstrap
 
 import (
 	"context"
-	"crypto/tls"
-	"encoding/base64"
 	"encoding/json"
 	"errors"
-	"fmt"
-	"net/http"
-	"syscall"
-	"time"
 
+	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 
 	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	ctrl "sigs.k8s.io/controller-runtime"
 
 	"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
 	"github.com/rancher/k3k/pkg/controller"
+	"github.com/rancher/k3k/pkg/k3s"
 )
 
-var ErrServerNotReady = errors.New("server not ready")
+const (
+	TLSDir = "/var/lib/rancher/k3s/server/tls/"
+)
 
-type ControlRuntimeBootstrap struct {
-	ServerCA        content `json:"serverCA"`
-	ServerCAKey     content `json:"serverCAKey"`
-	ClientCA        content `json:"clientCA"`
-	ClientCAKey     content `json:"clientCAKey"`
-	ETCDServerCA    content `json:"etcdServerCA"`
-	ETCDServerCAKey content `json:"etcdServerCAKey"`
+// Fetch requests bootstrap data from k3s using the token and decodes it,
+// to avoid double encoding when stored as secret.
+func Fetch(ctx context.Context, ip, token string) (*k3s.BootstrapData, error) {
+	log := ctrl.LoggerFrom(ctx)
+	log.V(1).Info("Fetching bootstrap data from K3s API")
+
+	return fetchFromK3sServer(ip, token)
 }
 
-type content struct {
-	Timestamp string
-	Content   string
-}
-
-// Generate generates the bootstrap for the cluster:
-// 1- use the server token to get the bootstrap data from k3s
-// 2- save the bootstrap data as a secret
-func GenerateBootstrapData(ctx context.Context, cluster *v1beta1.Cluster, ip, token string) ([]byte, error) {
-	bootstrap, err := requestBootstrap(token, ip)
+// SaveToSecret marshals the bootstrap data and stores it in a Secret owned by the cluster,
+// creating the Secret if it does not exist or updating it otherwise.
+func SaveToSecret(ctx context.Context, c client.Client, scheme *runtime.Scheme, cluster *v1beta1.Cluster, data *k3s.BootstrapData) error {
+	bootstrapData, err := json.Marshal(data)
 	if err != nil {
-		return nil, fmt.Errorf("failed to request bootstrap secret: %w", err)
+		return err
 	}
 
-	if err := decodeBootstrap(bootstrap); err != nil {
-		return nil, fmt.Errorf("failed to decode bootstrap secret: %w", err)
-	}
-
-	return json.Marshal(bootstrap)
-}
-
-func requestBootstrap(token, serverIP string) (*ControlRuntimeBootstrap, error) {
-	url := "https://" + serverIP + "/v1-k3s/server-bootstrap"
-
-	client := http.Client{
-		Transport: &http.Transport{
-			TLSClientConfig: &tls.Config{
-				InsecureSkipVerify: true,
-			},
+	secret := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      controller.SafeConcatNameWithPrefix(cluster.Name, "bootstrap"),
+			Namespace: cluster.Namespace,
 		},
-		Timeout: 5 * time.Second,
 	}
 
-	req, err := http.NewRequest(http.MethodGet, url, nil)
-	if err != nil {
-		return nil, err
-	}
-
-	req.Header.Add("Authorization", "Basic "+basicAuth("server", token))
-
-	resp, err := client.Do(req)
-	if err != nil {
-		if errors.Is(err, syscall.ECONNREFUSED) {
-			return nil, ErrServerNotReady
+	_, err = controllerutil.CreateOrUpdate(ctx, c, secret, func() error {
+		if err := controllerutil.SetControllerReference(cluster, secret, scheme); err != nil {
+			return err
 		}
 
-		return nil, err
-	}
+		secret.Data = map[string][]byte{
+			"bootstrap": bootstrapData,
+		}
 
-	defer func() {
-		_ = resp.Body.Close()
-	}()
+		return nil
+	})
 
-	var runtimeBootstrap ControlRuntimeBootstrap
-	if err := json.NewDecoder(resp.Body).Decode(&runtimeBootstrap); err != nil {
-		return nil, err
-	}
-
-	return &runtimeBootstrap, nil
+	return err
 }
 
-func basicAuth(username, password string) string {
-	auth := username + ":" + password
-	return base64.StdEncoding.EncodeToString([]byte(auth))
-}
-
-func decodeBootstrap(bootstrap *ControlRuntimeBootstrap) error {
-	// client-ca
-	decoded, err := base64.StdEncoding.DecodeString(bootstrap.ClientCA.Content)
-	if err != nil {
-		return err
-	}
-
-	bootstrap.ClientCA.Content = string(decoded)
-
-	// client-ca-key
-	decoded, err = base64.StdEncoding.DecodeString(bootstrap.ClientCAKey.Content)
-	if err != nil {
-		return err
-	}
-
-	bootstrap.ClientCAKey.Content = string(decoded)
-
-	// server-ca
-	decoded, err = base64.StdEncoding.DecodeString(bootstrap.ServerCA.Content)
-	if err != nil {
-		return err
-	}
-
-	bootstrap.ServerCA.Content = string(decoded)
-
-	// server-ca-key
-	decoded, err = base64.StdEncoding.DecodeString(bootstrap.ServerCAKey.Content)
-	if err != nil {
-		return err
-	}
-
-	bootstrap.ServerCAKey.Content = string(decoded)
-
-	// etcd-ca
-	decoded, err = base64.StdEncoding.DecodeString(bootstrap.ETCDServerCA.Content)
-	if err != nil {
-		return err
-	}
-
-	bootstrap.ETCDServerCA.Content = string(decoded)
-
-	// etcd-ca-key
-	decoded, err = base64.StdEncoding.DecodeString(bootstrap.ETCDServerCAKey.Content)
-	if err != nil {
-		return err
-	}
-
-	bootstrap.ETCDServerCAKey.Content = string(decoded)
-
-	return nil
-}
-
-func DecodedBootstrap(token, ip string) (*ControlRuntimeBootstrap, error) {
-	bootstrap, err := requestBootstrap(token, ip)
-	if err != nil {
-		return nil, err
-	}
-
-	if err := decodeBootstrap(bootstrap); err != nil {
-		return nil, err
-	}
-
-	return bootstrap, nil
-}
-
-func GetFromSecret(ctx context.Context, client client.Client, cluster *v1beta1.Cluster) (*ControlRuntimeBootstrap, error) {
+// LoadFromSecret reads the bootstrap data of a certain cluster and returns the decoded content.
+func LoadFromSecret(ctx context.Context, client client.Client, cluster *v1beta1.Cluster) (*k3s.BootstrapData, error) {
 	key := types.NamespacedName{
 		Name:      controller.SafeConcatNameWithPrefix(cluster.Name, "bootstrap"),
 		Namespace: cluster.Namespace,
@@ -178,9 +79,18 @@ func GetFromSecret(ctx context.Context, client client.Client, cluster *v1beta1.C
 		return nil, errors.New("empty bootstrap")
 	}
 
-	var bootstrap ControlRuntimeBootstrap
+	var bootstrap k3s.BootstrapData
 
 	err := json.Unmarshal(bootstrapData, &bootstrap)
 
 	return &bootstrap, err
 }
+
+func fetchFromK3sServer(serviceIP, token string) (*k3s.BootstrapData, error) {
+	client := k3s.New(k3s.ClientConfig{
+		ServerIP: serviceIP,
+		Token:    token,
+	})
+
+	return k3s.GetServerBootstrap(client)
+}

pkg/controller/cluster/server/server.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"path/filepath"
 	"sort"
 	"strings"
 	"text/template"
@@ -26,12 +27,22 @@ import (
 )
 
 const (
-	serverName     = "server"
-	configName     = "server-config"
-	initConfigName = "init-server-config"
+	initConfigName   = "init-server-config"
+	configName       = "server-config"
+	serverName       = "server"
+	k3sInitConfigDir = "/opt/rancher/k3s/init"
+	k3sConfigDir     = "/opt/rancher/k3s/server"
+	k3sRunDir        = "/run"
+	k3sCNIDir        = "/var/lib/cni"
+	k3sKubeletDir    = "/var/lib/kubelet"
+	k3sDataDir       = "/var/lib/rancher/k3s"
+	k3sETCDDataDir   = "/var/lib/rancher/k3s/server/db/etcd"
+	k3sManifestDir   = "/var/lib/rancher/k3s/server/manifests"
+	k3sTLSDir        = "/var/lib/rancher/k3s/server/tls"
+	k3sLogDir        = "/var/log"
+	k3sVarRunDir     = "/var/run"
 )
 
-// Server
 type Server struct {
 	cluster          *v1beta1.Cluster
 	client           client.Client
@@ -70,7 +81,7 @@ func (s *Server) podSpec(ctx context.Context, image, name string, persistent boo
 		PriorityClassName: s.cluster.Spec.PriorityClass,
 		Volumes: []corev1.Volume{
 			{
-				Name: "initconfig",
+				Name: "init-config",
 				VolumeSource: corev1.VolumeSource{
 					Secret: &corev1.SecretVolumeSource{
 						SecretName: configSecretName(s.cluster.Name, true),
@@ -104,25 +115,25 @@ func (s *Server) podSpec(ctx context.Context, image, name string, persistent boo
 				},
 			},
 			{
-				Name: "varrun",
+				Name: "var-run",
 				VolumeSource: corev1.VolumeSource{
 					EmptyDir: &corev1.EmptyDirVolumeSource{},
 				},
 			},
 			{
-				Name: "varlibcni",
+				Name: "var-lib-cni",
 				VolumeSource: corev1.VolumeSource{
 					EmptyDir: &corev1.EmptyDirVolumeSource{},
 				},
 			},
 			{
-				Name: "varlog",
+				Name: "var-log",
 				VolumeSource: corev1.VolumeSource{
 					EmptyDir: &corev1.EmptyDirVolumeSource{},
 				},
 			},
 			{
-				Name: "varlibkubelet",
+				Name: "var-lib-kubelet",
 				VolumeSource: corev1.VolumeSource{
 					EmptyDir: &corev1.EmptyDirVolumeSource{},
 				},
@@ -154,42 +165,42 @@ func (s *Server) podSpec(ctx context.Context, image, name string, persistent boo
 				VolumeMounts: []corev1.VolumeMount{
 					{
 						Name:      "config",
-						MountPath: "/opt/rancher/k3s/server",
+						MountPath: k3sConfigDir,
 						ReadOnly:  false,
 					},
 					{
-						Name:      "initconfig",
-						MountPath: "/opt/rancher/k3s/init",
+						Name:      "init-config",
+						MountPath: k3sInitConfigDir,
 						ReadOnly:  false,
 					},
 					{
 						Name:      "run",
-						MountPath: "/run",
+						MountPath: k3sRunDir,
 						ReadOnly:  false,
 					},
 					{
-						Name:      "varrun",
-						MountPath: "/var/run",
+						Name:      "var-run",
+						MountPath: k3sVarRunDir,
 						ReadOnly:  false,
 					},
 					{
-						Name:      "varlibcni",
-						MountPath: "/var/lib/cni",
+						Name:      "var-lib-cni",
+						MountPath: k3sCNIDir,
 						ReadOnly:  false,
 					},
 					{
-						Name:      "varlibkubelet",
-						MountPath: "/var/lib/kubelet",
+						Name:      "var-lib-kubelet",
+						MountPath: k3sKubeletDir,
 						ReadOnly:  false,
 					},
 					{
-						Name:      "varlibrancherk3s",
-						MountPath: "/var/lib/rancher/k3s",
+						Name:      "var-lib-rancher-k3s",
+						MountPath: k3sDataDir,
 						ReadOnly:  false,
 					},
 					{
-						Name:      "varlog",
-						MountPath: "/var/log",
+						Name:      "var-log",
+						MountPath: k3sLogDir,
 						ReadOnly:  false,
 					},
 				},
@@ -206,7 +217,7 @@ func (s *Server) podSpec(ctx context.Context, image, name string, persistent boo
 	podSpec.Containers[0].Command = cmd
 	if !persistent {
 		podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{
-			Name: "varlibrancherk3s",
+			Name: "var-lib-rancher-k3s",
 			VolumeSource: corev1.VolumeSource{
 				EmptyDir: &corev1.EmptyDirVolumeSource{},
 			},
@@ -405,7 +416,7 @@ func (s *Server) setupDynamicPersistence() corev1.PersistentVolumeClaim {
 			APIVersion: "v1",
 		},
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      "varlibrancherk3s",
+			Name:      "var-lib-rancher-k3s",
 			Namespace: s.cluster.Namespace,
 		},
 		Spec: corev1.PersistentVolumeClaimSpec{
@@ -441,9 +452,9 @@ func (s *Server) setupStartCommand() (string, error) {
 	}
 
 	if err := tmplCmd.Execute(&output, map[string]string{
-		"ETCD_DIR":      "/var/lib/rancher/k3s/server/db/etcd",
-		"INIT_CONFIG":   "/opt/rancher/k3s/init/config.yaml",
-		"SERVER_CONFIG": "/opt/rancher/k3s/server/config.yaml",
+		"ETCD_DIR":      k3sETCDDataDir,
+		"INIT_CONFIG":   filepath.Join(k3sInitConfigDir, "config.yaml"),
+		"SERVER_CONFIG": filepath.Join(k3sConfigDir, "config.yaml"),
 		"CLUSTER_MODE":  mode,
 		"K3K_MODE":      string(s.cluster.Spec.Mode),
 		"EXTRA_ARGS":    strings.Join(s.cluster.Spec.ServerArgs, " "),
@@ -511,7 +522,6 @@ func (s *Server) mountCACert(volumeName, certName, secretName string, subPathMou
 	)
 
 	// avoid re-adding secretName in case of combined secret
-
 	volume = &corev1.Volume{
 		Name: volumeName,
 		VolumeSource: corev1.VolumeSource{
@@ -528,22 +538,19 @@ func (s *Server) mountCACert(volumeName, certName, secretName string, subPathMou
 		mountFile = strings.TrimPrefix(certName, "etcd-")
 	}
 
-	// add the mount for the cert except for the service account token
-	if certName != "service" {
+	for _, crtOrKey := range []string{"crt", "key"} {
+		// skip adding cert mount for service account token
+		if certName == "service" && crtOrKey == "crt" {
+			continue
+		}
+
 		mounts = append(mounts, corev1.VolumeMount{
 			Name:      volumeName,
-			MountPath: fmt.Sprintf("/var/lib/rancher/k3s/server/tls%s/%s.crt", etcdPrefix, mountFile),
-			SubPath:   subPathMount + ".crt",
+			MountPath: filepath.Join(k3sTLSDir, etcdPrefix, mountFile) + "." + crtOrKey,
+			SubPath:   subPathMount + "." + crtOrKey,
 		})
 	}
 
-	// add the mount for the key
-	mounts = append(mounts, corev1.VolumeMount{
-		Name:      volumeName,
-		MountPath: fmt.Sprintf("/var/lib/rancher/k3s/server/tls%s/%s.key", etcdPrefix, mountFile),
-		SubPath:   subPathMount + ".key",
-	})
-
 	return volume, mounts
 }
 
@@ -603,7 +610,7 @@ func (s *Server) buildAddonsVolumes(ctx context.Context) ([]corev1.Volume, []cor
 
 		volumeMount := corev1.VolumeMount{
 			Name:      name,
-			MountPath: "/var/lib/rancher/k3s/server/manifests/" + addon.SecretRef,
+			MountPath: filepath.Join(k3sManifestDir, addon.SecretRef),
 			ReadOnly:  true,
 		}
 		mounts = append(mounts, volumeMount)

pkg/controller/cluster/statefulset.go

@@ -11,7 +11,6 @@ import (
 
 	"go.etcd.io/etcd/api/v3/v3rpc/rpctypes"
 	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/util/retry"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
@@ -32,6 +31,7 @@ import (
 	"github.com/rancher/k3k/pkg/controller/certs"
 	"github.com/rancher/k3k/pkg/controller/cluster/server"
 	"github.com/rancher/k3k/pkg/controller/cluster/server/bootstrap"
+	"github.com/rancher/k3k/pkg/k3s"
 )
 
 const (
@@ -44,7 +44,7 @@ type StatefulSetReconciler struct {
 	Scheme *runtime.Scheme
 }
 
-// Add adds a new controller to the manager
+// AddStatefulSetController adds a new statefulset controller to the manager
 func AddStatefulSetController(ctx context.Context, mgr manager.Manager, maxConcurrentReconciles int) error {
 	// initialize a new Reconciler
 	reconciler := StatefulSetReconciler{
@@ -180,21 +180,14 @@ func (p *StatefulSetReconciler) getETCDTLS(ctx context.Context, cluster *v1beta1
 	log := ctrl.LoggerFrom(ctx)
 	log.V(1).Info("Generating ETCD TLS client certificate", "cluster", cluster)
 
-	token, err := p.clusterToken(ctx, cluster)
-	if err != nil {
-		return nil, err
-	}
-
-	endpoint := server.ServiceName(cluster.Name) + "." + cluster.Namespace
-
-	var b *bootstrap.ControlRuntimeBootstrap
+	var b *k3s.BootstrapData
 
 	if err := retry.OnError(k3kcontroller.Backoff, func(err error) bool {
 		return true
 	}, func() error {
 		var err error
 
-		b, err = bootstrap.DecodedBootstrap(token, endpoint)
+		b, err = bootstrap.LoadFromSecret(ctx, p.Client, cluster)
 
 		return err
 	}); err != nil {
@@ -265,29 +258,6 @@ func removePeer(ctx context.Context, client *clientv3.Client, name, address stri
 	return nil
 }
 
-func (p *StatefulSetReconciler) clusterToken(ctx context.Context, cluster *v1beta1.Cluster) (string, error) {
-	var tokenSecret corev1.Secret
-
-	nn := types.NamespacedName{
-		Name:      TokenSecretName(cluster.Name),
-		Namespace: cluster.Namespace,
-	}
-
-	if cluster.Spec.TokenSecretRef != nil {
-		nn.Name = TokenSecretName(cluster.Name)
-	}
-
-	if err := p.Client.Get(ctx, nn, &tokenSecret); err != nil {
-		return "", err
-	}
-
-	if _, ok := tokenSecret.Data["token"]; !ok {
-		return "", fmt.Errorf("no token field in secret %s/%s", nn.Namespace, nn.Name)
-	}
-
-	return string(tokenSecret.Data["token"]), nil
-}
-
 func (p *StatefulSetReconciler) handleDeletion(ctx context.Context, sts *appsv1.StatefulSet) (ctrl.Result, error) {
 	log := ctrl.LoggerFrom(ctx)
 

pkg/controller/cluster/status.go

@@ -11,7 +11,7 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 
 	"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
-	"github.com/rancher/k3k/pkg/controller/cluster/server/bootstrap"
+	"github.com/rancher/k3k/pkg/k3s"
 )
 
 const (
@@ -57,7 +57,7 @@ func (c *ClusterReconciler) updateStatus(ctx context.Context, cluster *v1beta1.C
 		return
 	}
 
-	if errors.Is(reconcileErr, bootstrap.ErrServerNotReady) {
+	if errors.Is(reconcileErr, k3s.ErrServerNotReady) {
 		cluster.Status.Phase = v1beta1.ClusterProvisioning
 		meta.SetStatusCondition(&cluster.Status.Conditions, metav1.Condition{
 			Type:    ConditionReady,