External datastore support (#879)

* Adding new k3s function to get the config
* Adding a new way to get the bootstrap data directly from the server Pod
* Gate the bootstrap fetch if external datastore is in use
---------

Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>

pkg/controller/cluster/agent/agent.go

@@ -29,11 +29,11 @@ type Config struct {
 	scheme  *runtime.Scheme
 }
 
-func NewConfig(cluster *v1beta1.Cluster, client ctrlruntimeclient.Client, scheme *runtime.Scheme) *Config {
+func NewConfig(cluster *v1beta1.Cluster, client ctrlruntimeclient.Client) *Config {
 	return &Config{
 		cluster: cluster,
 		client:  client,
-		scheme:  scheme,
+		scheme:  client.Scheme(),
 	}
 }
 

pkg/controller/cluster/cluster.go

@@ -12,9 +12,9 @@ import (
 	"k8s.io/apimachinery/pkg/api/equality"
 	"k8s.io/apimachinery/pkg/api/meta"
 	"k8s.io/apimachinery/pkg/labels"
-	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/discovery"
+	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/client-go/tools/record"
 	"k8s.io/client-go/util/workqueue"
@@ -87,7 +87,7 @@ type Config struct {
 type ClusterReconciler struct {
 	DiscoveryClient *discovery.DiscoveryClient
 	Client          client.Client
-	Scheme          *runtime.Scheme
+	RestCfg         *rest.Config
 	PortAllocator   *agent.PortAllocator
 
 	record.EventRecorder
@@ -113,7 +113,7 @@ func Add(ctx context.Context, mgr manager.Manager, config *Config, maxConcurrent
 	reconciler := ClusterReconciler{
 		DiscoveryClient: discoveryClient,
 		Client:          mgr.GetClient(),
-		Scheme:          mgr.GetScheme(),
+		RestCfg:         mgr.GetConfig(),
 		EventRecorder:   eventRecorder,
 		PortAllocator:   portAllocator,
 		Config: Config{
@@ -462,12 +462,17 @@ func (c *ClusterReconciler) ensureBootstrapSecret(ctx context.Context, cluster *
 	log := ctrl.LoggerFrom(ctx)
 	log.V(1).Info("Ensuring bootstrap secret")
 
-	data, err := bootstrap.Fetch(ctx, serviceIP, token)
+	k3sClient := k3s.New(k3s.ClientConfig{
+		ServerIP: serviceIP,
+		Token:    token,
+	})
+
+	data, err := bootstrap.Fetch(ctx, k3sClient, cluster, c.RestCfg)
 	if err != nil {
 		return err
 	}
 
-	return bootstrap.SaveToSecret(ctx, c.Client, c.Scheme, cluster, data)
+	return bootstrap.SaveToSecret(ctx, c.Client, cluster, data)
 }
 
 // ensureKubeconfigSecret will create or update the Secret containing the kubeconfig data from the k3s server
@@ -495,7 +500,7 @@ func (c *ClusterReconciler) ensureKubeconfigSecret(ctx context.Context, cluster
 	}
 
 	_, err = controllerutil.CreateOrUpdate(ctx, c.Client, kubeconfigSecret, func() error {
-		if err := controllerutil.SetControllerReference(cluster, kubeconfigSecret, c.Scheme); err != nil {
+		if err := controllerutil.SetControllerReference(cluster, kubeconfigSecret, c.Client.Scheme()); err != nil {
 			return err
 		}
 
@@ -516,7 +521,7 @@ func (c *ClusterReconciler) createClusterConfigs(ctx context.Context, cluster *v
 		return err
 	}
 
-	if err := controllerutil.SetControllerReference(cluster, initServerConfig, c.Scheme); err != nil {
+	if err := controllerutil.SetControllerReference(cluster, initServerConfig, c.Client.Scheme()); err != nil {
 		return err
 	}
 
@@ -532,7 +537,7 @@ func (c *ClusterReconciler) createClusterConfigs(ctx context.Context, cluster *v
 		return err
 	}
 
-	if err := controllerutil.SetControllerReference(cluster, serverConfig, c.Scheme); err != nil {
+	if err := controllerutil.SetControllerReference(cluster, serverConfig, c.Client.Scheme()); err != nil {
 		return err
 	}
 
@@ -625,7 +630,7 @@ func (c *ClusterReconciler) ensureNetworkPolicy(ctx context.Context, cluster *v1
 	currentNetworkPolicy := expectedNetworkPolicy.DeepCopy()
 
 	result, err := controllerutil.CreateOrUpdate(ctx, c.Client, currentNetworkPolicy, func() error {
-		if err := controllerutil.SetControllerReference(cluster, currentNetworkPolicy, c.Scheme); err != nil {
+		if err := controllerutil.SetControllerReference(cluster, currentNetworkPolicy, c.Client.Scheme()); err != nil {
 			return err
 		}
 
@@ -653,7 +658,7 @@ func (c *ClusterReconciler) ensureClusterService(ctx context.Context, cluster *v
 	currentService := expectedService.DeepCopy()
 
 	result, err := controllerutil.CreateOrUpdate(ctx, c.Client, currentService, func() error {
-		if err := controllerutil.SetControllerReference(cluster, currentService, c.Scheme); err != nil {
+		if err := controllerutil.SetControllerReference(cluster, currentService, c.Client.Scheme()); err != nil {
 			return err
 		}
 
@@ -688,7 +693,7 @@ func (c *ClusterReconciler) ensureIngress(ctx context.Context, cluster *v1beta1.
 	currentServerIngress := expectedServerIngress.DeepCopy()
 
 	result, err := controllerutil.CreateOrUpdate(ctx, c.Client, currentServerIngress, func() error {
-		if err := controllerutil.SetControllerReference(cluster, currentServerIngress, c.Scheme); err != nil {
+		if err := controllerutil.SetControllerReference(cluster, currentServerIngress, c.Client.Scheme()); err != nil {
 			return err
 		}
 
@@ -828,7 +833,7 @@ func (c *ClusterReconciler) server(ctx context.Context, cluster *v1beta1.Cluster
 
 	// create headless service for the statefulset
 	serverStatefulService := server.StatefulServerService()
-	if err := controllerutil.SetControllerReference(cluster, serverStatefulService, c.Scheme); err != nil {
+	if err := controllerutil.SetControllerReference(cluster, serverStatefulService, c.Client.Scheme()); err != nil {
 		return err
 	}
 
@@ -848,7 +853,7 @@ func (c *ClusterReconciler) server(ctx context.Context, cluster *v1beta1.Cluster
 
 	currentServerStatefulSet := expectedServerStatefulSet.DeepCopy()
 	result, err := controllerutil.CreateOrUpdate(ctx, c.Client, currentServerStatefulSet, func() error {
-		if err := controllerutil.SetControllerReference(cluster, currentServerStatefulSet, c.Scheme); err != nil {
+		if err := controllerutil.SetControllerReference(cluster, currentServerStatefulSet, c.Client.Scheme()); err != nil {
 			return err
 		}
 
@@ -896,7 +901,7 @@ func (c *ClusterReconciler) bindClusterRoles(ctx context.Context, cluster *v1bet
 }
 
 func (c *ClusterReconciler) ensureAgent(ctx context.Context, cluster *v1beta1.Cluster, serviceIP, token string) error {
-	config := agent.NewConfig(cluster, c.Client, c.Scheme)
+	config := agent.NewConfig(cluster, c.Client)
 
 	var agentEnsurer agent.ResourceEnsurer
 	if cluster.Spec.Mode == agent.VirtualNodeMode {

pkg/controller/cluster/mounts/mounts.go

@@ -41,8 +41,13 @@ func buildSecretMountVolume(secretMount v1beta1.SecretMount) (corev1.Volume, cor
 		},
 	}
 
+	volName := secretMount.Name
+	if volName == "" {
+		volName = secretMount.SecretName
+	}
+
 	vol := corev1.Volume{
-		Name: secretMount.SecretName,
+		Name: volName,
 		VolumeSource: corev1.VolumeSource{
 			Projected: &corev1.ProjectedVolumeSource{
 				Sources: projectedVolSources,
@@ -51,7 +56,7 @@ func buildSecretMountVolume(secretMount v1beta1.SecretMount) (corev1.Volume, cor
 	}
 
 	volMount := corev1.VolumeMount{
-		Name:      secretMount.SecretName,
+		Name:      volName,
 		MountPath: secretMount.MountPath,
 		SubPath:   secretMount.SubPath,
 	}

pkg/controller/cluster/server/bootstrap/bootstrap.go

@@ -5,8 +5,8 @@ import (
 	"encoding/json"
 	"errors"
 
-	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/rest"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 
@@ -19,22 +19,29 @@ import (
 	"github.com/rancher/k3k/pkg/k3s"
 )
 
-const (
-	TLSDir = "/var/lib/rancher/k3s/server/tls/"
-)
-
 // Fetch requests bootstrap data from k3s using the token and decodes it,
 // to avoid double encoding when stored as secret.
-func Fetch(ctx context.Context, ip, token string) (*k3s.BootstrapData, error) {
+func Fetch(ctx context.Context, k3sClient *k3s.Client, cluster *v1beta1.Cluster, restConfig *rest.Config) (*k3s.BootstrapData, error) {
 	log := ctrl.LoggerFrom(ctx)
-	log.V(1).Info("Fetching bootstrap data from K3s API")
 
-	return fetchFromK3sServer(ip, token)
+	config, err := k3sClient.GetServerConfig()
+	if err != nil {
+		return nil, err
+	}
+
+	if config.ClusterInit {
+		log.V(1).Info("Fetching bootstrap data from K3s server API")
+		return k3sClient.GetServerBootstrap()
+	}
+
+	log.V(1).Info("Fetching bootstrap data from K3s server Pod")
+
+	return k3s.ReadBootstrapFromK3sPod(ctx, restConfig, cluster.Name, cluster.Namespace)
 }
 
 // SaveToSecret marshals the bootstrap data and stores it in a Secret owned by the cluster,
 // creating the Secret if it does not exist or updating it otherwise.
-func SaveToSecret(ctx context.Context, c client.Client, scheme *runtime.Scheme, cluster *v1beta1.Cluster, data *k3s.BootstrapData) error {
+func SaveToSecret(ctx context.Context, c client.Client, cluster *v1beta1.Cluster, data *k3s.BootstrapData) error {
 	bootstrapData, err := json.Marshal(data)
 	if err != nil {
 		return err
@@ -48,7 +55,7 @@ func SaveToSecret(ctx context.Context, c client.Client, scheme *runtime.Scheme,
 	}
 
 	_, err = controllerutil.CreateOrUpdate(ctx, c, secret, func() error {
-		if err := controllerutil.SetControllerReference(cluster, secret, scheme); err != nil {
+		if err := controllerutil.SetControllerReference(cluster, secret, c.Scheme()); err != nil {
 			return err
 		}
 
@@ -85,12 +92,3 @@ func LoadFromSecret(ctx context.Context, client client.Client, cluster *v1beta1.
 
 	return &bootstrap, err
 }
-
-func fetchFromK3sServer(serviceIP, token string) (*k3s.BootstrapData, error) {
-	client := k3s.New(k3s.ClientConfig{
-		ServerIP: serviceIP,
-		Token:    token,
-	})
-
-	return k3s.GetServerBootstrap(client)
-}

pkg/controller/cluster/server/template.go

@@ -131,7 +131,7 @@ configure_cgroups() {
 	EXTRA_ARGS="$EXTRA_ARGS --kubelet-arg=kubelet-cgroups=$root_cgroup_parent/k3s --kubelet-arg=cgroup-root=$root_cgroup_parent"
 }
 
-EXTRA_ARGS={{.EXTRA_ARGS}}
+EXTRA_ARGS="{{.EXTRA_ARGS}}"
 configure_cgroups
 
 case "{{.CLUSTER_MODE}}" in

pkg/controller/cluster/statefulset.go

@@ -72,12 +72,6 @@ func (p *StatefulSetReconciler) Reconcile(ctx context.Context, req reconcile.Req
 		return reconcile.Result{}, ctrlruntimeclient.IgnoreNotFound(err)
 	}
 
-	// If the StatefulSet is being deleted, we need to remove the finalizers from its pods
-	// and remove the finalizer from the StatefulSet itself.
-	if !sts.DeletionTimestamp.IsZero() {
-		return p.handleDeletion(ctx, &sts)
-	}
-
 	// get cluster name from the object
 	clusterKey := clusterNamespacedName(&sts)
 
@@ -88,6 +82,12 @@ func (p *StatefulSetReconciler) Reconcile(ctx context.Context, req reconcile.Req
 		}
 	}
 
+	// If the StatefulSet is being deleted, we need to remove the finalizers from its pods
+	// and remove the finalizer from the StatefulSet itself.
+	if !sts.DeletionTimestamp.IsZero() {
+		return p.handleDeletion(ctx, &sts)
+	}
+
 	podList, err := p.listPods(ctx, &sts)
 	if err != nil {
 		return reconcile.Result{}, err
@@ -106,6 +106,26 @@ func (p *StatefulSetReconciler) Reconcile(ctx context.Context, req reconcile.Req
 		}
 	}
 
+	// skip adding finalizer to server pods if etcd is not used
+	clusterToken, err := getClusterToken(ctx, p.Client, &cluster)
+	if err != nil {
+		return reconcile.Result{}, err
+	}
+
+	k3sClient := k3s.New(k3s.ClientConfig{
+		ServerIP: fmt.Sprintf("%s.%s", server.ServiceName(cluster.Name), cluster.Namespace),
+		Token:    clusterToken,
+	})
+
+	config, err := k3sClient.GetServerConfig()
+	if err != nil {
+		return reconcile.Result{}, err
+	}
+
+	if !config.ClusterInit {
+		return reconcile.Result{}, nil
+	}
+
 	for _, pod := range podList.Items {
 		if err := p.handleServerPod(ctx, cluster, &pod); err != nil {
 			return reconcile.Result{}, err

pkg/controller/cluster/token.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"crypto/rand"
 	"encoding/hex"
+	"errors"
 	"fmt"
 
 	"k8s.io/apimachinery/pkg/types"
@@ -73,7 +74,7 @@ func (c *ClusterReconciler) ensureTokenSecret(ctx context.Context, cluster *v1be
 	key = client.ObjectKeyFromObject(&tokenSecret)
 
 	result, err := controllerutil.CreateOrUpdate(ctx, c.Client, &tokenSecret, func() error {
-		return controllerutil.SetControllerReference(cluster, &tokenSecret, c.Scheme)
+		return controllerutil.SetControllerReference(cluster, &tokenSecret, c.Client.Scheme())
 	})
 
 	if result != controllerutil.OperationResultNone {
@@ -113,3 +114,21 @@ func TokenSecretObj(token, name, namespace string) corev1.Secret {
 func TokenSecretName(clusterName string) string {
 	return controller.SafeConcatNameWithPrefix(clusterName, "token")
 }
+
+func getClusterToken(ctx context.Context, client client.Client, cluster *v1beta1.Cluster) (string, error) {
+	tokenSecretName := TokenSecretName(cluster.Name)
+	if cluster.Spec.TokenSecretRef != nil && cluster.Spec.TokenSecretRef.Name != "" {
+		tokenSecretName = cluster.Spec.TokenSecretRef.Name
+	}
+
+	var tokenSecret corev1.Secret
+	if err := client.Get(ctx, types.NamespacedName{Name: tokenSecretName, Namespace: cluster.Namespace}, &tokenSecret); err != nil {
+		return "", err
+	}
+
+	if tokenSecret.Data != nil {
+		return string(tokenSecret.Data["token"]), nil
+	}
+
+	return "", errors.New("token secret is empty")
+}