External datastore support (#879)

* Adding new k3s function to get the config
* Adding a new way to get the bootstrap data directly from the server Pod
* Gate the bootstrap fetch if external datastore is in use
---------

Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>

pkg/apis/k3k.io/v1beta1/types.go

@@ -241,6 +241,15 @@ type ClusterSpec struct {
 type SecretMount struct {
 	// Embeds SecretName, Items, DefaultMode, and Optional
 	corev1.SecretVolumeSource `json:",inline"`
+	// Name is the name of the secret mount volume that will be used
+	// as the name of volume and volume mount for the server or agent pod
+	// if empty then the secret name will be used instead.
+	//
+	// +kubebuilder:validation:MinLength=1
+	// +kubebuilder:validation:MaxLength=63
+	// +kubebuilder:validation:Pattern=`^[a-z0-9]([-a-z0-9]*[a-z0-9])?$`
+	// +optional
+	Name string `json:"name,omitempty"`
 	// MountPath is the path within server and agent pods where the
 	// secret contents will be mounted.
 	//

pkg/controller/cluster/agent/agent.go

@@ -29,11 +29,11 @@ type Config struct {
 	scheme  *runtime.Scheme
 }
 
-func NewConfig(cluster *v1beta1.Cluster, client ctrlruntimeclient.Client, scheme *runtime.Scheme) *Config {
+func NewConfig(cluster *v1beta1.Cluster, client ctrlruntimeclient.Client) *Config {
 	return &Config{
 		cluster: cluster,
 		client:  client,
-		scheme:  scheme,
+		scheme:  client.Scheme(),
 	}
 }
 

pkg/controller/cluster/cluster.go

@@ -12,9 +12,9 @@ import (
 	"k8s.io/apimachinery/pkg/api/equality"
 	"k8s.io/apimachinery/pkg/api/meta"
 	"k8s.io/apimachinery/pkg/labels"
-	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/discovery"
+	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/client-go/tools/record"
 	"k8s.io/client-go/util/workqueue"
@@ -87,7 +87,7 @@ type Config struct {
 type ClusterReconciler struct {
 	DiscoveryClient *discovery.DiscoveryClient
 	Client          client.Client
-	Scheme          *runtime.Scheme
+	RestCfg         *rest.Config
 	PortAllocator   *agent.PortAllocator
 
 	record.EventRecorder
@@ -113,7 +113,7 @@ func Add(ctx context.Context, mgr manager.Manager, config *Config, maxConcurrent
 	reconciler := ClusterReconciler{
 		DiscoveryClient: discoveryClient,
 		Client:          mgr.GetClient(),
-		Scheme:          mgr.GetScheme(),
+		RestCfg:         mgr.GetConfig(),
 		EventRecorder:   eventRecorder,
 		PortAllocator:   portAllocator,
 		Config: Config{
@@ -462,12 +462,17 @@ func (c *ClusterReconciler) ensureBootstrapSecret(ctx context.Context, cluster *
 	log := ctrl.LoggerFrom(ctx)
 	log.V(1).Info("Ensuring bootstrap secret")
 
-	data, err := bootstrap.Fetch(ctx, serviceIP, token)
+	k3sClient := k3s.New(k3s.ClientConfig{
+		ServerIP: serviceIP,
+		Token:    token,
+	})
+
+	data, err := bootstrap.Fetch(ctx, k3sClient, cluster, c.RestCfg)
 	if err != nil {
 		return err
 	}
 
-	return bootstrap.SaveToSecret(ctx, c.Client, c.Scheme, cluster, data)
+	return bootstrap.SaveToSecret(ctx, c.Client, cluster, data)
 }
 
 // ensureKubeconfigSecret will create or update the Secret containing the kubeconfig data from the k3s server
@@ -495,7 +500,7 @@ func (c *ClusterReconciler) ensureKubeconfigSecret(ctx context.Context, cluster
 	}
 
 	_, err = controllerutil.CreateOrUpdate(ctx, c.Client, kubeconfigSecret, func() error {
-		if err := controllerutil.SetControllerReference(cluster, kubeconfigSecret, c.Scheme); err != nil {
+		if err := controllerutil.SetControllerReference(cluster, kubeconfigSecret, c.Client.Scheme()); err != nil {
 			return err
 		}
 
@@ -516,7 +521,7 @@ func (c *ClusterReconciler) createClusterConfigs(ctx context.Context, cluster *v
 		return err
 	}
 
-	if err := controllerutil.SetControllerReference(cluster, initServerConfig, c.Scheme); err != nil {
+	if err := controllerutil.SetControllerReference(cluster, initServerConfig, c.Client.Scheme()); err != nil {
 		return err
 	}
 
@@ -532,7 +537,7 @@ func (c *ClusterReconciler) createClusterConfigs(ctx context.Context, cluster *v
 		return err
 	}
 
-	if err := controllerutil.SetControllerReference(cluster, serverConfig, c.Scheme); err != nil {
+	if err := controllerutil.SetControllerReference(cluster, serverConfig, c.Client.Scheme()); err != nil {
 		return err
 	}
 
@@ -625,7 +630,7 @@ func (c *ClusterReconciler) ensureNetworkPolicy(ctx context.Context, cluster *v1
 	currentNetworkPolicy := expectedNetworkPolicy.DeepCopy()
 
 	result, err := controllerutil.CreateOrUpdate(ctx, c.Client, currentNetworkPolicy, func() error {
-		if err := controllerutil.SetControllerReference(cluster, currentNetworkPolicy, c.Scheme); err != nil {
+		if err := controllerutil.SetControllerReference(cluster, currentNetworkPolicy, c.Client.Scheme()); err != nil {
 			return err
 		}
 
@@ -653,7 +658,7 @@ func (c *ClusterReconciler) ensureClusterService(ctx context.Context, cluster *v
 	currentService := expectedService.DeepCopy()
 
 	result, err := controllerutil.CreateOrUpdate(ctx, c.Client, currentService, func() error {
-		if err := controllerutil.SetControllerReference(cluster, currentService, c.Scheme); err != nil {
+		if err := controllerutil.SetControllerReference(cluster, currentService, c.Client.Scheme()); err != nil {
 			return err
 		}
 
@@ -688,7 +693,7 @@ func (c *ClusterReconciler) ensureIngress(ctx context.Context, cluster *v1beta1.
 	currentServerIngress := expectedServerIngress.DeepCopy()
 
 	result, err := controllerutil.CreateOrUpdate(ctx, c.Client, currentServerIngress, func() error {
-		if err := controllerutil.SetControllerReference(cluster, currentServerIngress, c.Scheme); err != nil {
+		if err := controllerutil.SetControllerReference(cluster, currentServerIngress, c.Client.Scheme()); err != nil {
 			return err
 		}
 
@@ -828,7 +833,7 @@ func (c *ClusterReconciler) server(ctx context.Context, cluster *v1beta1.Cluster
 
 	// create headless service for the statefulset
 	serverStatefulService := server.StatefulServerService()
-	if err := controllerutil.SetControllerReference(cluster, serverStatefulService, c.Scheme); err != nil {
+	if err := controllerutil.SetControllerReference(cluster, serverStatefulService, c.Client.Scheme()); err != nil {
 		return err
 	}
 
@@ -848,7 +853,7 @@ func (c *ClusterReconciler) server(ctx context.Context, cluster *v1beta1.Cluster
 
 	currentServerStatefulSet := expectedServerStatefulSet.DeepCopy()
 	result, err := controllerutil.CreateOrUpdate(ctx, c.Client, currentServerStatefulSet, func() error {
-		if err := controllerutil.SetControllerReference(cluster, currentServerStatefulSet, c.Scheme); err != nil {
+		if err := controllerutil.SetControllerReference(cluster, currentServerStatefulSet, c.Client.Scheme()); err != nil {
 			return err
 		}
 
@@ -896,7 +901,7 @@ func (c *ClusterReconciler) bindClusterRoles(ctx context.Context, cluster *v1bet
 }
 
 func (c *ClusterReconciler) ensureAgent(ctx context.Context, cluster *v1beta1.Cluster, serviceIP, token string) error {
-	config := agent.NewConfig(cluster, c.Client, c.Scheme)
+	config := agent.NewConfig(cluster, c.Client)
 
 	var agentEnsurer agent.ResourceEnsurer
 	if cluster.Spec.Mode == agent.VirtualNodeMode {

pkg/controller/cluster/mounts/mounts.go

@@ -41,8 +41,13 @@ func buildSecretMountVolume(secretMount v1beta1.SecretMount) (corev1.Volume, cor
 		},
 	}
 
+	volName := secretMount.Name
+	if volName == "" {
+		volName = secretMount.SecretName
+	}
+
 	vol := corev1.Volume{
-		Name: secretMount.SecretName,
+		Name: volName,
 		VolumeSource: corev1.VolumeSource{
 			Projected: &corev1.ProjectedVolumeSource{
 				Sources: projectedVolSources,
@@ -51,7 +56,7 @@ func buildSecretMountVolume(secretMount v1beta1.SecretMount) (corev1.Volume, cor
 	}
 
 	volMount := corev1.VolumeMount{
-		Name:      secretMount.SecretName,
+		Name:      volName,
 		MountPath: secretMount.MountPath,
 		SubPath:   secretMount.SubPath,
 	}

pkg/controller/cluster/server/bootstrap/bootstrap.go

@@ -5,8 +5,8 @@ import (
 	"encoding/json"
 	"errors"
 
-	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/rest"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 
@@ -19,22 +19,29 @@ import (
 	"github.com/rancher/k3k/pkg/k3s"
 )
 
-const (
-	TLSDir = "/var/lib/rancher/k3s/server/tls/"
-)
-
 // Fetch requests bootstrap data from k3s using the token and decodes it,
 // to avoid double encoding when stored as secret.
-func Fetch(ctx context.Context, ip, token string) (*k3s.BootstrapData, error) {
+func Fetch(ctx context.Context, k3sClient *k3s.Client, cluster *v1beta1.Cluster, restConfig *rest.Config) (*k3s.BootstrapData, error) {
 	log := ctrl.LoggerFrom(ctx)
-	log.V(1).Info("Fetching bootstrap data from K3s API")
 
-	return fetchFromK3sServer(ip, token)
+	config, err := k3sClient.GetServerConfig()
+	if err != nil {
+		return nil, err
+	}
+
+	if config.ClusterInit {
+		log.V(1).Info("Fetching bootstrap data from K3s server API")
+		return k3sClient.GetServerBootstrap()
+	}
+
+	log.V(1).Info("Fetching bootstrap data from K3s server Pod")
+
+	return k3s.ReadBootstrapFromK3sPod(ctx, restConfig, cluster.Name, cluster.Namespace)
 }
 
 // SaveToSecret marshals the bootstrap data and stores it in a Secret owned by the cluster,
 // creating the Secret if it does not exist or updating it otherwise.
-func SaveToSecret(ctx context.Context, c client.Client, scheme *runtime.Scheme, cluster *v1beta1.Cluster, data *k3s.BootstrapData) error {
+func SaveToSecret(ctx context.Context, c client.Client, cluster *v1beta1.Cluster, data *k3s.BootstrapData) error {
 	bootstrapData, err := json.Marshal(data)
 	if err != nil {
 		return err
@@ -48,7 +55,7 @@ func SaveToSecret(ctx context.Context, c client.Client, scheme *runtime.Scheme,
 	}
 
 	_, err = controllerutil.CreateOrUpdate(ctx, c, secret, func() error {
-		if err := controllerutil.SetControllerReference(cluster, secret, scheme); err != nil {
+		if err := controllerutil.SetControllerReference(cluster, secret, c.Scheme()); err != nil {
 			return err
 		}
 
@@ -85,12 +92,3 @@ func LoadFromSecret(ctx context.Context, client client.Client, cluster *v1beta1.
 
 	return &bootstrap, err
 }
-
-func fetchFromK3sServer(serviceIP, token string) (*k3s.BootstrapData, error) {
-	client := k3s.New(k3s.ClientConfig{
-		ServerIP: serviceIP,
-		Token:    token,
-	})
-
-	return k3s.GetServerBootstrap(client)
-}

pkg/controller/cluster/server/template.go

@@ -131,7 +131,7 @@ configure_cgroups() {
 	EXTRA_ARGS="$EXTRA_ARGS --kubelet-arg=kubelet-cgroups=$root_cgroup_parent/k3s --kubelet-arg=cgroup-root=$root_cgroup_parent"
 }
 
-EXTRA_ARGS={{.EXTRA_ARGS}}
+EXTRA_ARGS="{{.EXTRA_ARGS}}"
 configure_cgroups
 
 case "{{.CLUSTER_MODE}}" in

pkg/controller/cluster/statefulset.go

@@ -72,12 +72,6 @@ func (p *StatefulSetReconciler) Reconcile(ctx context.Context, req reconcile.Req
 		return reconcile.Result{}, ctrlruntimeclient.IgnoreNotFound(err)
 	}
 
-	// If the StatefulSet is being deleted, we need to remove the finalizers from its pods
-	// and remove the finalizer from the StatefulSet itself.
-	if !sts.DeletionTimestamp.IsZero() {
-		return p.handleDeletion(ctx, &sts)
-	}
-
 	// get cluster name from the object
 	clusterKey := clusterNamespacedName(&sts)
 
@@ -88,6 +82,12 @@ func (p *StatefulSetReconciler) Reconcile(ctx context.Context, req reconcile.Req
 		}
 	}
 
+	// If the StatefulSet is being deleted, we need to remove the finalizers from its pods
+	// and remove the finalizer from the StatefulSet itself.
+	if !sts.DeletionTimestamp.IsZero() {
+		return p.handleDeletion(ctx, &sts)
+	}
+
 	podList, err := p.listPods(ctx, &sts)
 	if err != nil {
 		return reconcile.Result{}, err
@@ -106,6 +106,26 @@ func (p *StatefulSetReconciler) Reconcile(ctx context.Context, req reconcile.Req
 		}
 	}
 
+	// skip adding finalizer to server pods if etcd is not used
+	clusterToken, err := getClusterToken(ctx, p.Client, &cluster)
+	if err != nil {
+		return reconcile.Result{}, err
+	}
+
+	k3sClient := k3s.New(k3s.ClientConfig{
+		ServerIP: fmt.Sprintf("%s.%s", server.ServiceName(cluster.Name), cluster.Namespace),
+		Token:    clusterToken,
+	})
+
+	config, err := k3sClient.GetServerConfig()
+	if err != nil {
+		return reconcile.Result{}, err
+	}
+
+	if !config.ClusterInit {
+		return reconcile.Result{}, nil
+	}
+
 	for _, pod := range podList.Items {
 		if err := p.handleServerPod(ctx, cluster, &pod); err != nil {
 			return reconcile.Result{}, err

pkg/controller/cluster/token.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"crypto/rand"
 	"encoding/hex"
+	"errors"
 	"fmt"
 
 	"k8s.io/apimachinery/pkg/types"
@@ -73,7 +74,7 @@ func (c *ClusterReconciler) ensureTokenSecret(ctx context.Context, cluster *v1be
 	key = client.ObjectKeyFromObject(&tokenSecret)
 
 	result, err := controllerutil.CreateOrUpdate(ctx, c.Client, &tokenSecret, func() error {
-		return controllerutil.SetControllerReference(cluster, &tokenSecret, c.Scheme)
+		return controllerutil.SetControllerReference(cluster, &tokenSecret, c.Client.Scheme())
 	})
 
 	if result != controllerutil.OperationResultNone {
@@ -113,3 +114,21 @@ func TokenSecretObj(token, name, namespace string) corev1.Secret {
 func TokenSecretName(clusterName string) string {
 	return controller.SafeConcatNameWithPrefix(clusterName, "token")
 }
+
+func getClusterToken(ctx context.Context, client client.Client, cluster *v1beta1.Cluster) (string, error) {
+	tokenSecretName := TokenSecretName(cluster.Name)
+	if cluster.Spec.TokenSecretRef != nil && cluster.Spec.TokenSecretRef.Name != "" {
+		tokenSecretName = cluster.Spec.TokenSecretRef.Name
+	}
+
+	var tokenSecret corev1.Secret
+	if err := client.Get(ctx, types.NamespacedName{Name: tokenSecretName, Namespace: cluster.Namespace}, &tokenSecret); err != nil {
+		return "", err
+	}
+
+	if tokenSecret.Data != nil {
+		return string(tokenSecret.Data["token"]), nil
+	}
+
+	return "", errors.New("token secret is empty")
+}

pkg/k3s/bootstrap.go

@@ -1,9 +1,25 @@
 package k3s
 
 import (
+	"bytes"
+	"context"
 	"encoding/base64"
 	"fmt"
 	"net/http"
+	"path/filepath"
+
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/remotecommand"
+
+	corev1 "k8s.io/api/core/v1"
+
+	"github.com/rancher/k3k/pkg/controller"
+)
+
+const (
+	TLSDir = "/var/lib/rancher/k3s/server/tls/"
 )
 
 type BootstrapData struct {
@@ -20,20 +36,29 @@ type cert struct {
 	Content   string
 }
 
-func GetServerBootstrap(c *Client) (*BootstrapData, error) {
+type K3SConfig struct {
+	ClusterInit bool `json:"ClusterInit"`
+}
+
+func (c *Client) GetServerConfig() (*K3SConfig, error) {
+	endpoint := "/v1-k3s/config"
+	return do[*K3SConfig](c, endpoint, "node", http.MethodGet)
+}
+
+func (c *Client) GetServerBootstrap() (*BootstrapData, error) {
 	endpoint := "/v1-k3s/server-bootstrap"
 
-	bootstrap, err := do[BootstrapData](c, endpoint, "server", http.MethodGet)
+	bootstrap, err := do[*BootstrapData](c, endpoint, "server", http.MethodGet)
 	if err != nil {
 		return nil, err
 	}
 
 	// we still need to decode each certs since the bootstrap data endpoint base64 encode each cert
-	if err := decode(&bootstrap); err != nil {
+	if err := decode(bootstrap); err != nil {
 		return nil, fmt.Errorf("failed to decode bootstrap secret: %w", err)
 	}
 
-	return &bootstrap, nil
+	return bootstrap, nil
 }
 
 func decode(data *BootstrapData) error {
@@ -87,3 +112,76 @@ func decode(data *BootstrapData) error {
 
 	return nil
 }
+
+func ReadBootstrapFromK3sPod(ctx context.Context, restConfig *rest.Config, clusterName, clusterNamespace string) (*BootstrapData, error) {
+	clientset, err := kubernetes.NewForConfig(restConfig)
+	if err != nil {
+		return nil, err
+	}
+
+	// using the first server in the statefulset to get the bootstrap data
+	serverPodName := controller.SafeConcatNameWithPrefix(clusterName, "server-0")
+
+	// skipping etcd since reading from pod only required with external datastore
+	bootstrapCerts := map[string]string{
+		"server-ca.crt": "",
+		"server-ca.key": "",
+		"client-ca.crt": "",
+		"client-ca.key": "",
+	}
+	for certName := range bootstrapCerts {
+		command := []string{"cat", filepath.Join(TLSDir, certName)}
+
+		certData, err := podExec(ctx, clientset, restConfig, clusterNamespace, serverPodName, command)
+		if err != nil {
+			return nil, err
+		}
+
+		bootstrapCerts[certName] = string(certData)
+	}
+
+	bootstrap := &BootstrapData{
+		ServerCA:    cert{Content: bootstrapCerts["server-ca.crt"]},
+		ServerCAKey: cert{Content: bootstrapCerts["server-ca.key"]},
+		ClientCA:    cert{Content: bootstrapCerts["client-ca.crt"]},
+		ClientCAKey: cert{Content: bootstrapCerts["client-ca.key"]},
+	}
+
+	return bootstrap, nil
+}
+
+func podExec(ctx context.Context, clientset *kubernetes.Clientset, config *rest.Config, namespace, name string, command []string) ([]byte, error) {
+	req := clientset.CoreV1().RESTClient().Post().
+		Resource("pods").
+		Name(name).
+		Namespace(namespace).
+		SubResource("exec")
+
+	scheme := runtime.NewScheme()
+	if err := corev1.AddToScheme(scheme); err != nil {
+		return nil, fmt.Errorf("error adding to scheme: %v", err)
+	}
+
+	parameterCodec := runtime.NewParameterCodec(scheme)
+
+	req.VersionedParams(&corev1.PodExecOptions{
+		Command: command,
+		Stdout:  true,
+	}, parameterCodec)
+
+	exec, err := remotecommand.NewSPDYExecutor(config, http.MethodPost, req.URL())
+	if err != nil {
+		return nil, fmt.Errorf("error while creating Executor: %v", err)
+	}
+
+	var stdout bytes.Buffer
+
+	err = exec.StreamWithContext(ctx, remotecommand.StreamOptions{
+		Stdout: &stdout,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("error in Stream: %v", err)
+	}
+
+	return stdout.Bytes(), nil
+}

pkg/k3s/bootstrap_test.go

@@ -0,0 +1,152 @@
+package k3s
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func Test_GetServerConfig(t *testing.T) {
+	tests := []struct {
+		name            string
+		serverResponse  string
+		clientConfig    ClientConfig
+		expectedConfig  *K3SConfig
+		expectedErr     error
+		isServerRunning bool
+	}{
+		{
+			name:            "server not ready",
+			isServerRunning: false,
+			serverResponse:  "",
+			expectedConfig:  nil,
+			expectedErr:     ErrServerNotReady,
+			clientConfig: ClientConfig{
+				ServerIP: "127.0.0.1:33333",
+			},
+		},
+		{
+			name:            "cluster init is true",
+			isServerRunning: true,
+			serverResponse:  `{"ClusterInit": true}`,
+			expectedConfig:  &K3SConfig{ClusterInit: true},
+			expectedErr:     ErrServerNotReady,
+			clientConfig:    ClientConfig{},
+		},
+		{
+			name:            "cluster init is false",
+			isServerRunning: true,
+			serverResponse:  `{"ClusterInit": false}`,
+			expectedConfig:  &K3SConfig{ClusterInit: false},
+			expectedErr:     ErrServerNotReady,
+			clientConfig:    ClientConfig{},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			mux := http.NewServeMux()
+
+			mockServer := httptest.NewUnstartedServer(mux)
+			if tt.isServerRunning {
+				mockServer.StartTLS()
+				defer mockServer.Close()
+			}
+
+			mux.Handle("/v1-k3s/config", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				_, err := w.Write([]byte(tt.serverResponse))
+				require.NoError(t, err)
+			}))
+
+			if tt.clientConfig.ServerIP == "" {
+				u, err := url.Parse(mockServer.URL)
+				require.NoError(t, err)
+
+				tt.clientConfig.ServerIP = u.Host
+			}
+
+			k3sClient := New(tt.clientConfig)
+
+			k3sConfig, err := k3sClient.GetServerConfig()
+			if err != nil && tt.expectedErr != nil {
+				assert.EqualError(t, err, tt.expectedErr.Error())
+			} else {
+				require.NoError(t, err)
+			}
+
+			assert.Equal(t, tt.expectedConfig, k3sConfig)
+		})
+	}
+}
+
+func Test_GetServerBootstrap(t *testing.T) {
+	fakeBootstrap := &BootstrapData{
+		ClientCA: cert{
+			Content: "dGVzdA==",
+		},
+		ClientCAKey: cert{
+			Content: "dGVzdA==",
+		},
+		ServerCA: cert{
+			Content: "dGVzdA==",
+		},
+		ServerCAKey: cert{
+			Content: "dGVzdA==",
+		},
+		ETCDServerCA: cert{
+			Content: "dGVzdA==",
+		},
+		ETCDServerCAKey: cert{
+			Content: "dGVzdA==",
+		},
+	}
+
+	expectedBootstrap := &BootstrapData{
+		ClientCA: cert{
+			Content: "test",
+		},
+		ClientCAKey: cert{
+			Content: "test",
+		},
+		ServerCA: cert{
+			Content: "test",
+		},
+		ServerCAKey: cert{
+			Content: "test",
+		},
+		ETCDServerCA: cert{
+			Content: "test",
+		},
+		ETCDServerCAKey: cert{
+			Content: "test",
+		},
+	}
+
+	mux := http.NewServeMux()
+
+	mockServer := httptest.NewTLSServer(mux)
+	defer mockServer.Close()
+
+	serverResponse, err := json.Marshal(fakeBootstrap)
+	require.NoError(t, err)
+
+	mux.Handle("/v1-k3s/server-bootstrap", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		_, err := w.Write(serverResponse)
+		require.NoError(t, err)
+	}))
+
+	u, err := url.Parse(mockServer.URL)
+	require.NoError(t, err)
+
+	k3sClient := New(ClientConfig{ServerIP: u.Host})
+
+	bootstrap, err := k3sClient.GetServerBootstrap()
+	require.NoError(t, err)
+
+	assert.Equal(t, expectedBootstrap, bootstrap)
+}

pkg/k3s/certs.go

@@ -5,7 +5,7 @@ import (
 	"net/http"
 )
 
-func GetServingKubeletCrt(c *Client) (*tls.Certificate, error) {
+func (c *Client) GetServingKubeletCrt() (*tls.Certificate, error) {
 	endpoint := "/v1-k3s/serving-kubelet.crt"
 
 	tlsCrtData, err := c.do(endpoint, "node", http.MethodGet)

pkg/k3s/certs_test.go

@@ -0,0 +1,58 @@
+package k3s
+
+import (
+	"crypto/tls"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	fakeCertPEM = `-----BEGIN CERTIFICATE-----
+MIIBgTCCASegAwIBAgIUDZWXjBG6lXVaJ7cv3NAaaCX4QxkwCgYIKoZIzj0EAwIw
+FjEUMBIGA1UEAwwLazNrLWt1YmVsZXQwHhcNMjYwNjExMTEwMzQ3WhcNMzYwNjA4
+MTEwMzQ3WjAWMRQwEgYDVQQDDAtrM2sta3ViZWxldDBZMBMGByqGSM49AgEGCCqG
+SM49AwEHA0IABFrEAV6qpR7m8VUXL1mL9/bmuLa1QvXkiUXhWvuJ+dg7G3p1kSNC
+35d4w3IAN626oyMpMD1FL9kw5U6Gd17bcoSjUzBRMB0GA1UdDgQWBBS9KJuhT7sK
+3nwpq1W6KkJ34PRsrDAfBgNVHSMEGDAWgBS9KJuhT7sK3nwpq1W6KkJ34PRsrDAP
+BgNVHRMBAf8EBTADAQH/MAoGCCqGSM49BAMCA0gAMEUCIQCEg8kpyTuhsbvj9+p+
+B+pbwZb+fgGO3iYuawrvVYZwHAIgIu6PYkP0ZGsUjhMkZUUNfQgZ42Lwq1CmKnqv
+rWa+ZUI=
+-----END CERTIFICATE-----
+`
+	fakeKeyPEM = `-----BEGIN PRIVATE KEY-----
+MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQg/030b66JWbB6xK5m
+1UuPpRbu2OLo7NyZQ1f/u1mxfq+hRANCAARaxAFeqqUe5vFVFy9Zi/f25ri2tUL1
+5IlF4Vr7ifnYOxt6dZEjQt+XeMNyADetuqMjKTA9RS/ZMOVOhnde23KE
+-----END PRIVATE KEY-----
+`
+)
+
+func Test_GetServingKubeletCert(t *testing.T) {
+	mux := http.NewServeMux()
+
+	mockServer := httptest.NewTLSServer(mux)
+	defer mockServer.Close()
+
+	mux.Handle("/v1-k3s/serving-kubelet.crt", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		_, err := w.Write([]byte(fakeCertPEM + fakeKeyPEM))
+		require.NoError(t, err)
+	}))
+
+	u, err := url.Parse(mockServer.URL)
+	require.NoError(t, err)
+
+	k3sClient := New(ClientConfig{ServerIP: u.Host})
+
+	expectedCert, err := tls.X509KeyPair([]byte(fakeCertPEM), []byte(fakeKeyPEM))
+	require.NoError(t, err)
+
+	cert, err := k3sClient.GetServingKubeletCrt()
+	require.NoError(t, err)
+
+	assert.Equal(t, expectedCert, *cert)
+}

pkg/k3s/client.go

@@ -27,6 +27,12 @@ type Client struct {
 
 var ErrServerNotReady = errors.New("server not ready")
 
+const (
+	k3sNodePasswordHeader = "k3s-Node-Password"
+	k3sNodeIPHeader       = "k3s-Node-IP"
+	k3sNodeNameHeader     = "k3s-Node-Name"
+)
+
 func New(config ClientConfig) *Client {
 	httpClient := &http.Client{
 		Transport: http.DefaultTransport,
@@ -44,11 +50,11 @@ func New(config ClientConfig) *Client {
 	headers := http.Header{}
 
 	if config.Token != "" {
-		headers.Set("k3s-Node-Password", config.Token)
+		headers.Set(k3sNodePasswordHeader, config.Token)
 	}
 
 	if config.NodeName != "" {
-		headers.Set("k3s-Node-Name", config.NodeName)
+		headers.Set(k3sNodeNameHeader, config.NodeName)
 	}
 
 	var nodeIPs []string

pkg/k3s/client_test.go

@@ -0,0 +1,50 @@
+package k3s
+
+import (
+	"net/http"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_NewClient(t *testing.T) {
+	tests := []struct {
+		name            string
+		clientConfig    ClientConfig
+		expectedHeaders http.Header
+	}{
+		{
+			name:            "empty config",
+			clientConfig:    ClientConfig{},
+			expectedHeaders: nil,
+		},
+		{
+			name: "client config passed",
+			clientConfig: ClientConfig{
+				Token:    "test_token",
+				ServerIP: "0.0.0.0:1234",
+				NodeName: "test_node",
+				AgentIP:  "1.1.1.1",
+				PodIP:    "2.2.2.2",
+			},
+			expectedHeaders: map[string][]string{
+				k3sNodePasswordHeader: {"test_token"},
+				k3sNodeNameHeader:     {"test_node"},
+				k3sNodeIPHeader:       {"1.1.1.1,2.2.2.2"},
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			client := New(tt.clientConfig)
+			assert.Len(t, client.staticHeaders, len(tt.expectedHeaders))
+
+			for expectedHeader, expectedValue := range tt.expectedHeaders {
+				value, ok := client.staticHeaders[http.CanonicalHeaderKey(expectedHeader)]
+				assert.True(t, ok, "expected header %s is not found", expectedHeader)
+				assert.Equal(t, expectedValue, value)
+			}
+		})
+	}
+}