k3k/pkg/controller/cluster/server/config.go

package server

import (
	"fmt"

	"gopkg.in/yaml.v3"
	"k8s.io/apimachinery/pkg/util/sets"

	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
	"github.com/rancher/k3k/pkg/controller"
)

// serverConfig are few options from k3s server options that will
// construct the yaml config file for k3s server
type serverConfig struct {
	ClusterCIDR        string   `yaml:"cluster-cidr,omitempty"`
	ClusterDNS         string   `yaml:"cluster-dns,omitempty"`
	ClusterInit        bool     `yaml:"cluster-init,omitempty"`
	DisableAgent       bool     `yaml:"disable-agent,omitempty"`
	Disable            []string `yaml:"disable,omitempty"`
	EgressSelectorMode string   `yaml:"egress-selector-mode,omitempty"`
	KubeApiServerArg   []string `yaml:"kube-apiserver-arg,omitempty"`
	Server             string   `yaml:"server,omitempty"`
	ServiceCIDR        string   `yaml:"service-cidr,omitempty"`
	TLSSAN             []string `yaml:"tls-san,omitempty"`
	Token              string   `yaml:"token,omitempty"`
}

func (s *Server) Config(init bool, serviceIP string) (*corev1.Secret, error) {
	name := configSecretName(s.cluster.Name, init)

	serverConfig := buildServerConfig(s.cluster, init, serviceIP, s.token)

	config, err := yaml.Marshal(serverConfig)
	if err != nil {
		return nil, err
	}

	return &corev1.Secret{
		TypeMeta: metav1.TypeMeta{
			Kind:       "Secret",
			APIVersion: "v1",
		},
		ObjectMeta: metav1.ObjectMeta{
			Name:      name,
			Namespace: s.cluster.Namespace,
		},
		Data: map[string][]byte{
			"config.yaml": config,
		},
	}, nil
}

func buildServerConfig(cluster *v1beta1.Cluster, initServer bool, serviceIP, token string) serverConfig {
	sans := sets.NewString(cluster.Spec.TLSSANs...)
	sans.Insert(
		serviceIP,
		ServiceName(cluster.Name),
		fmt.Sprintf("%s.%s", ServiceName(cluster.Name), cluster.Namespace),
	)

	cluster.Status.TLSSANs = sans.List()

	serverConfig := serverConfig{
		ClusterInit: true,
		Token:       token,
		TLSSAN:      cluster.Status.TLSSANs,
		ServiceCIDR: cluster.Status.ServiceCIDR,
		ClusterCIDR: cluster.Status.ClusterCIDR,
		ClusterDNS:  cluster.Spec.ClusterDNS,
	}

	if !initServer {
		serverConfig.Server = "https://" + serviceIP
	}

	// shared and hcp modes both run K3s with --disable-agent (agentless server).
	switch cluster.Spec.Mode {
	case "", v1beta1.SharedClusterMode:
		serverConfig.DisableAgent = true
		serverConfig.EgressSelectorMode = "disabled"
		serverConfig.Disable = []string{"servicelb", "traefik", "metrics-server", "local-storage"}
	case v1beta1.HCPClusterMode:
		serverConfig.DisableAgent = true
		// Tunnel apiserver egress through the k3s-agent WebSocket: the
		// apiserver has no route to the virtual cluster's pod CIDR and
		// bypasses kube-proxy when dialing pod IPs (webhooks, log/exec).
		// "cluster" is the only safe mode — "agent" lets pod dials go
		// direct (no route, fails); "pod" only permits pod IPs the agent
		// has already watched, so a newly-created pod's IP is rejected
		// and tears down the remotedialer session, making kubelet streams
		// flaky. See k3s pkg/agent/tunnel/tunnel.go.
		serverConfig.EgressSelectorMode = "cluster"
		// Disable the apiserver's built-in endpoint reconciler so K3k can
		// own default/kubernetes Endpoints and point it at the externally
		// reachable host:port (NodePort / LB / Ingress).
		serverConfig.KubeApiServerArg = append(serverConfig.KubeApiServerArg, "endpoint-reconciler-type=none")
	case v1beta1.VirtualClusterMode:
		// no extra config for virtual mode
	}

	return serverConfig
}

func configSecretName(clusterName string, init bool) string {
	if !init {
		return controller.SafeConcatNameWithPrefix(clusterName, configName)
	}

	return controller.SafeConcatNameWithPrefix(clusterName, initConfigName)
}