mirror of
https://github.com/rancher/k3k.git
synced 2026-03-02 17:50:36 +00:00
413 lines
9.5 KiB
Go
413 lines
9.5 KiB
Go
package server
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"strings"
|
|
"text/template"
|
|
|
|
"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
|
|
"github.com/rancher/k3k/pkg/controller"
|
|
"github.com/rancher/k3k/pkg/controller/cluster/agent"
|
|
apps "k8s.io/api/apps/v1"
|
|
v1 "k8s.io/api/core/v1"
|
|
"k8s.io/apimachinery/pkg/api/resource"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/types"
|
|
"k8s.io/apimachinery/pkg/util/intstr"
|
|
"k8s.io/utils/ptr"
|
|
"sigs.k8s.io/controller-runtime/pkg/client"
|
|
)
|
|
|
|
const (
|
|
k3kSystemNamespace = "k3k-system"
|
|
serverName = "server"
|
|
configName = "server-config"
|
|
initConfigName = "init-server-config"
|
|
)
|
|
|
|
// Server
|
|
type Server struct {
|
|
cluster *v1alpha1.Cluster
|
|
client client.Client
|
|
mode string
|
|
token string
|
|
k3SImage string
|
|
k3SImagePullPolicy string
|
|
}
|
|
|
|
func New(cluster *v1alpha1.Cluster, client client.Client, token, mode string, k3SImage string, k3SImagePullPolicy string) *Server {
|
|
return &Server{
|
|
cluster: cluster,
|
|
client: client,
|
|
token: token,
|
|
mode: mode,
|
|
k3SImage: k3SImage,
|
|
k3SImagePullPolicy: k3SImagePullPolicy,
|
|
}
|
|
}
|
|
|
|
func (s *Server) podSpec(image, name string, persistent bool, startupCmd string) v1.PodSpec {
|
|
podSpec := v1.PodSpec{
|
|
NodeSelector: s.cluster.Spec.NodeSelector,
|
|
PriorityClassName: s.cluster.Spec.PriorityClass,
|
|
Volumes: []v1.Volume{
|
|
{
|
|
Name: "initconfig",
|
|
VolumeSource: v1.VolumeSource{
|
|
Secret: &v1.SecretVolumeSource{
|
|
SecretName: configSecretName(s.cluster.Name, true),
|
|
Items: []v1.KeyToPath{
|
|
{
|
|
Key: "config.yaml",
|
|
Path: "config.yaml",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
Name: "config",
|
|
VolumeSource: v1.VolumeSource{
|
|
Secret: &v1.SecretVolumeSource{
|
|
SecretName: configSecretName(s.cluster.Name, false),
|
|
Items: []v1.KeyToPath{
|
|
{
|
|
Key: "config.yaml",
|
|
Path: "config.yaml",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
Name: "run",
|
|
VolumeSource: v1.VolumeSource{
|
|
EmptyDir: &v1.EmptyDirVolumeSource{},
|
|
},
|
|
},
|
|
{
|
|
Name: "varrun",
|
|
VolumeSource: v1.VolumeSource{
|
|
EmptyDir: &v1.EmptyDirVolumeSource{},
|
|
},
|
|
},
|
|
{
|
|
Name: "varlibcni",
|
|
VolumeSource: v1.VolumeSource{
|
|
EmptyDir: &v1.EmptyDirVolumeSource{},
|
|
},
|
|
},
|
|
{
|
|
Name: "varlog",
|
|
VolumeSource: v1.VolumeSource{
|
|
EmptyDir: &v1.EmptyDirVolumeSource{},
|
|
},
|
|
},
|
|
{
|
|
Name: "varlibkubelet",
|
|
VolumeSource: v1.VolumeSource{
|
|
EmptyDir: &v1.EmptyDirVolumeSource{},
|
|
},
|
|
},
|
|
},
|
|
Containers: []v1.Container{
|
|
{
|
|
Name: name,
|
|
Image: image,
|
|
ImagePullPolicy: v1.PullPolicy(s.k3SImagePullPolicy),
|
|
Env: []v1.EnvVar{
|
|
{
|
|
Name: "POD_NAME",
|
|
ValueFrom: &v1.EnvVarSource{
|
|
FieldRef: &v1.ObjectFieldSelector{
|
|
FieldPath: "metadata.name",
|
|
},
|
|
},
|
|
},
|
|
{
|
|
Name: "POD_IP",
|
|
ValueFrom: &v1.EnvVarSource{
|
|
FieldRef: &v1.ObjectFieldSelector{
|
|
FieldPath: "status.podIP",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
VolumeMounts: []v1.VolumeMount{
|
|
{
|
|
Name: "config",
|
|
MountPath: "/opt/rancher/k3s/server",
|
|
ReadOnly: false,
|
|
},
|
|
{
|
|
Name: "initconfig",
|
|
MountPath: "/opt/rancher/k3s/init",
|
|
ReadOnly: false,
|
|
},
|
|
{
|
|
Name: "run",
|
|
MountPath: "/run",
|
|
ReadOnly: false,
|
|
},
|
|
{
|
|
Name: "varrun",
|
|
MountPath: "/var/run",
|
|
ReadOnly: false,
|
|
},
|
|
{
|
|
Name: "varlibcni",
|
|
MountPath: "/var/lib/cni",
|
|
ReadOnly: false,
|
|
},
|
|
{
|
|
Name: "varlibkubelet",
|
|
MountPath: "/var/lib/kubelet",
|
|
ReadOnly: false,
|
|
},
|
|
{
|
|
Name: "varlibrancherk3s",
|
|
MountPath: "/var/lib/rancher/k3s",
|
|
ReadOnly: false,
|
|
},
|
|
{
|
|
Name: "varlog",
|
|
MountPath: "/var/log",
|
|
ReadOnly: false,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
cmd := []string{
|
|
"/bin/sh",
|
|
"-c",
|
|
startupCmd,
|
|
}
|
|
|
|
podSpec.Containers[0].Command = cmd
|
|
if !persistent {
|
|
podSpec.Volumes = append(podSpec.Volumes, v1.Volume{
|
|
Name: "varlibrancherk3s",
|
|
VolumeSource: v1.VolumeSource{
|
|
EmptyDir: &v1.EmptyDirVolumeSource{},
|
|
},
|
|
},
|
|
)
|
|
}
|
|
|
|
// Adding readiness probes to statefulset
|
|
podSpec.Containers[0].ReadinessProbe = &v1.Probe{
|
|
InitialDelaySeconds: 60,
|
|
FailureThreshold: 5,
|
|
TimeoutSeconds: 10,
|
|
ProbeHandler: v1.ProbeHandler{
|
|
TCPSocket: &v1.TCPSocketAction{
|
|
Port: intstr.FromInt(6443),
|
|
},
|
|
},
|
|
}
|
|
podSpec.Containers[0].LivenessProbe = &v1.Probe{
|
|
InitialDelaySeconds: 10,
|
|
FailureThreshold: 3,
|
|
PeriodSeconds: 3,
|
|
ProbeHandler: v1.ProbeHandler{
|
|
Exec: &v1.ExecAction{
|
|
Command: []string{
|
|
"sh",
|
|
"-c",
|
|
`grep -q "rejoin the cluster" /var/log/k3s.log && exit 1 || exit 0`,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
// start the pod unprivileged in shared mode
|
|
if s.mode == agent.VirtualNodeMode {
|
|
podSpec.Containers[0].SecurityContext = &v1.SecurityContext{
|
|
Privileged: ptr.To(true),
|
|
}
|
|
}
|
|
|
|
// specify resource limits if specified for the servers.
|
|
if s.cluster.Spec.ServerLimit != nil {
|
|
podSpec.Containers[0].Resources = v1.ResourceRequirements{
|
|
Limits: s.cluster.Spec.ServerLimit,
|
|
}
|
|
}
|
|
|
|
podSpec.Containers[0].Env = append(podSpec.Containers[0].Env, s.cluster.Spec.ServerEnvs...)
|
|
|
|
return podSpec
|
|
}
|
|
|
|
func (s *Server) StatefulServer(ctx context.Context) (*apps.StatefulSet, error) {
|
|
var (
|
|
replicas int32
|
|
pvClaim v1.PersistentVolumeClaim
|
|
err error
|
|
persistent bool
|
|
)
|
|
|
|
image := controller.K3SImage(s.cluster, s.k3SImage)
|
|
name := controller.SafeConcatNameWithPrefix(s.cluster.Name, serverName)
|
|
|
|
replicas = *s.cluster.Spec.Servers
|
|
|
|
if s.cluster.Spec.Persistence.Type == v1alpha1.DynamicPersistenceMode {
|
|
persistent = true
|
|
pvClaim = s.setupDynamicPersistence()
|
|
}
|
|
|
|
var (
|
|
volumes []v1.Volume
|
|
volumeMounts []v1.VolumeMount
|
|
)
|
|
|
|
for _, addon := range s.cluster.Spec.Addons {
|
|
namespace := k3kSystemNamespace
|
|
if addon.SecretNamespace != "" {
|
|
namespace = addon.SecretNamespace
|
|
}
|
|
|
|
nn := types.NamespacedName{
|
|
Name: addon.SecretRef,
|
|
Namespace: namespace,
|
|
}
|
|
|
|
var addons v1.Secret
|
|
if err := s.client.Get(ctx, nn, &addons); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
clusterAddons := v1.Secret{
|
|
TypeMeta: metav1.TypeMeta{
|
|
Kind: "Secret",
|
|
APIVersion: "v1",
|
|
},
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: addons.Name,
|
|
Namespace: s.cluster.Namespace,
|
|
},
|
|
Data: make(map[string][]byte, len(addons.Data)),
|
|
}
|
|
for k, v := range addons.Data {
|
|
clusterAddons.Data[k] = v
|
|
}
|
|
|
|
if err := s.client.Create(ctx, &clusterAddons); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
name := "varlibrancherk3smanifests" + addon.SecretRef
|
|
volume := v1.Volume{
|
|
Name: name,
|
|
VolumeSource: v1.VolumeSource{
|
|
Secret: &v1.SecretVolumeSource{
|
|
SecretName: addon.SecretRef,
|
|
},
|
|
},
|
|
}
|
|
volumes = append(volumes, volume)
|
|
|
|
volumeMount := v1.VolumeMount{
|
|
Name: name,
|
|
MountPath: "/var/lib/rancher/k3s/server/manifests/" + addon.SecretRef,
|
|
// changes to this part of the filesystem shouldn't be done manually. The secret should be updated instead.
|
|
ReadOnly: true,
|
|
}
|
|
volumeMounts = append(volumeMounts, volumeMount)
|
|
}
|
|
|
|
selector := metav1.LabelSelector{
|
|
MatchLabels: map[string]string{
|
|
"cluster": s.cluster.Name,
|
|
"role": "server",
|
|
},
|
|
}
|
|
|
|
startupCommand, err := s.setupStartCommand()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
podSpec := s.podSpec(image, name, persistent, startupCommand)
|
|
podSpec.Volumes = append(podSpec.Volumes, volumes...)
|
|
podSpec.Containers[0].VolumeMounts = append(podSpec.Containers[0].VolumeMounts, volumeMounts...)
|
|
|
|
ss := &apps.StatefulSet{
|
|
TypeMeta: metav1.TypeMeta{
|
|
Kind: "StatefulSet",
|
|
APIVersion: "apps/v1",
|
|
},
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: name,
|
|
Namespace: s.cluster.Namespace,
|
|
Labels: selector.MatchLabels,
|
|
},
|
|
Spec: apps.StatefulSetSpec{
|
|
Replicas: &replicas,
|
|
ServiceName: headlessServiceName(s.cluster.Name),
|
|
Selector: &selector,
|
|
Template: v1.PodTemplateSpec{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Labels: selector.MatchLabels,
|
|
},
|
|
Spec: podSpec,
|
|
},
|
|
},
|
|
}
|
|
if s.cluster.Spec.Persistence.Type == v1alpha1.DynamicPersistenceMode {
|
|
ss.Spec.VolumeClaimTemplates = []v1.PersistentVolumeClaim{pvClaim}
|
|
}
|
|
|
|
return ss, nil
|
|
}
|
|
|
|
func (s *Server) setupDynamicPersistence() v1.PersistentVolumeClaim {
|
|
return v1.PersistentVolumeClaim{
|
|
TypeMeta: metav1.TypeMeta{
|
|
Kind: "PersistentVolumeClaim",
|
|
APIVersion: "v1",
|
|
},
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "varlibrancherk3s",
|
|
Namespace: s.cluster.Namespace,
|
|
},
|
|
Spec: v1.PersistentVolumeClaimSpec{
|
|
AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce},
|
|
StorageClassName: s.cluster.Spec.Persistence.StorageClassName,
|
|
Resources: v1.VolumeResourceRequirements{
|
|
Requests: v1.ResourceList{
|
|
"storage": resource.MustParse(s.cluster.Status.Persistence.StorageRequestSize),
|
|
},
|
|
},
|
|
},
|
|
}
|
|
}
|
|
|
|
func (s *Server) setupStartCommand() (string, error) {
|
|
var output bytes.Buffer
|
|
|
|
tmpl := singleServerTemplate
|
|
if *s.cluster.Spec.Servers > 1 {
|
|
tmpl = HAServerTemplate
|
|
}
|
|
|
|
tmplCmd, err := template.New("").Parse(tmpl)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
|
|
if err := tmplCmd.Execute(&output, map[string]string{
|
|
"ETCD_DIR": "/var/lib/rancher/k3s/server/db/etcd",
|
|
"INIT_CONFIG": "/opt/rancher/k3s/init/config.yaml",
|
|
"SERVER_CONFIG": "/opt/rancher/k3s/server/config.yaml",
|
|
"EXTRA_ARGS": strings.Join(s.cluster.Spec.ServerArgs, " "),
|
|
}); err != nil {
|
|
return "", err
|
|
}
|
|
|
|
return output.String(), nil
|
|
}
|