github/k3k
1
0
Fork 0
mirror of https://github.com/rancher/k3k.git synced 2026-08-18 20:07:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

Default to dynamic persistence and fix HA restarts (#250)

Browse Source 
* Default to dynamic persistence and fix HA restarts

Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>

---------

Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>
This commit is contained in:
Hussein Galal
2025-02-14 14:26:10 +02:00
committed by GitHub
parent 51a8fd8a8d
commit fa553d25d4
15 changed files with 429 additions and 150 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/controller/cluster/cluster.go
+5 -12
View File
@@ -162,12 +162,10 @@ func (c *ClusterReconciler) reconcileCluster(ctx context.Context, cluster *v1alp
s := server.New(cluster, c.Client, token, string(cluster.Spec.Mode))
if cluster.Spec.Persistence != nil {
cluster.Status.Persistence = cluster.Spec.Persistence
if cluster.Spec.Persistence.StorageRequestSize == "" {
// default to 1G of request size
cluster.Status.Persistence.StorageRequestSize = defaultStoragePersistentSize
}
cluster.Status.Persistence = cluster.Spec.Persistence
if cluster.Spec.Persistence.StorageRequestSize == "" {
// default to 1G of request size
cluster.Status.Persistence.StorageRequestSize = defaultStoragePersistentSize
}
cluster.Status.ClusterCIDR = cluster.Spec.ClusterCIDR
@@ -190,7 +188,6 @@ func (c *ClusterReconciler) reconcileCluster(ctx context.Context, cluster *v1alp
return err
}
// creating statefulsets in case the user chose a persistence type other than ephemeral
if err := c.server(ctx, cluster, s); err != nil {
return err
}
@@ -207,11 +204,7 @@ func (c *ClusterReconciler) reconcileCluster(ctx context.Context, cluster *v1alp
return err
}
if err := c.bindNodeProxyClusterRole(ctx, cluster); err != nil {
return err
}
return nil
return c.bindNodeProxyClusterRole(ctx, cluster)
}
// ensureBootstrapSecret will create or update the Secret containing the bootstrap data from the k3s server
pkg/controller/cluster/cluster_test.go
+3 -1
View File
@@ -40,7 +40,7 @@ var _ = Describe("Cluster Controller", func() {
BeforeEach(func() {
cluster = &v1alpha1.Cluster{
ObjectMeta: metav1.ObjectMeta{
GenerateName: "clusterset-",
GenerateName: "cluster-",
Namespace: namespace,
},
}
@@ -54,6 +54,8 @@ var _ = Describe("Cluster Controller", func() {
Expect(cluster.Spec.Agents).To(Equal(ptr.To[int32](0)))
Expect(cluster.Spec.Servers).To(Equal(ptr.To[int32](1)))
Expect(cluster.Spec.Version).To(BeEmpty())
// TOFIX
//Expect(cluster.Spec.Persistence.Type).To(Equal(v1alpha1.DynamicNodesType))
serverVersion, err := k8s.DiscoveryClient.ServerVersion()
Expect(err).To(Not(HaveOccurred()))
pkg/controller/cluster/server/server.go
+89 -71
View File
@@ -1,8 +1,10 @@
package server
import (
"bytes"
"context"
"strings"
"text/template"
"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
"github.com/rancher/k3k/pkg/controller"
@@ -23,9 +25,7 @@ const (
configName = "server-config"
initConfigName = "init-server-config"
ServerPort = 6443
EphemeralNodesType = "ephemeral"
DynamicNodesType = "dynamic"
ServerPort = 6443
)
// Server
@@ -45,7 +45,7 @@ func New(cluster *v1alpha1.Cluster, client client.Client, token, mode string) *S
}
}
func (s *Server) podSpec(image, name string, persistent bool) v1.PodSpec {
func (s *Server) podSpec(image, name string, persistent bool, startupCmd string) v1.PodSpec {
var limit v1.ResourceList
if s.cluster.Spec.Limit != nil && s.cluster.Spec.Limit.ServerLimit != nil {
limit = s.cluster.Spec.Limit.ServerLimit
@@ -106,6 +106,12 @@ func (s *Server) podSpec(image, name string, persistent bool) v1.PodSpec {
EmptyDir: &v1.EmptyDirVolumeSource{},
},
},
{
Name: "varlibkubelet",
VolumeSource: v1.VolumeSource{
EmptyDir: &v1.EmptyDirVolumeSource{},
},
},
},
Containers: []v1.Container{
{
@@ -123,17 +129,14 @@ func (s *Server) podSpec(image, name string, persistent bool) v1.PodSpec {
},
},
},
},
Command: []string{
"/bin/sh",
"-c",
`
if [ ${POD_NAME: -1} == 0 ]; then
/bin/k3s server --config /opt/rancher/k3s/init/config.yaml ` + strings.Join(s.cluster.Spec.ServerArgs, " ") + `
else
/bin/k3s server --config /opt/rancher/k3s/server/config.yaml ` + strings.Join(s.cluster.Spec.ServerArgs, " ") + `
fi
`,
{
Name: "POD_IP",
ValueFrom: &v1.EnvVarSource{
FieldRef: &v1.ObjectFieldSelector{
FieldPath: "status.podIP",
},
},
},
},
VolumeMounts: []v1.VolumeMount{
{
@@ -181,15 +184,15 @@ func (s *Server) podSpec(image, name string, persistent bool) v1.PodSpec {
},
}
cmd := []string{
"/bin/sh",
"-c",
startupCmd,
}
podSpec.Containers[0].Command = cmd
if !persistent {
podSpec.Volumes = append(podSpec.Volumes, v1.Volume{
Name: "varlibkubelet",
VolumeSource: v1.VolumeSource{
EmptyDir: &v1.EmptyDirVolumeSource{},
},
}, v1.Volume{
Name: "varlibrancherk3s",
VolumeSource: v1.VolumeSource{
EmptyDir: &v1.EmptyDirVolumeSource{},
@@ -221,7 +224,8 @@ func (s *Server) podSpec(image, name string, persistent bool) v1.PodSpec {
func (s *Server) StatefulServer(ctx context.Context) (*apps.StatefulSet, error) {
var (
replicas int32
pvClaims []v1.PersistentVolumeClaim
pvClaim v1.PersistentVolumeClaim
err error
persistent bool
)
image := controller.K3SImage(s.cluster)
@@ -229,48 +233,9 @@ func (s *Server) StatefulServer(ctx context.Context) (*apps.StatefulSet, error)
replicas = *s.cluster.Spec.Servers
if s.cluster.Spec.Persistence != nil && s.cluster.Spec.Persistence.Type != EphemeralNodesType {
if s.cluster.Spec.Persistence.Type == v1alpha1.DynamicNodesType {
persistent = true
pvClaims = []v1.PersistentVolumeClaim{
{
TypeMeta: metav1.TypeMeta{
Kind: "PersistentVolumeClaim",
APIVersion: "v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: "varlibrancherk3s",
Namespace: s.cluster.Namespace,
},
Spec: v1.PersistentVolumeClaimSpec{
AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce},
StorageClassName: &s.cluster.Spec.Persistence.StorageClassName,
Resources: v1.VolumeResourceRequirements{
Requests: v1.ResourceList{
"storage": resource.MustParse(s.cluster.Spec.Persistence.StorageRequestSize),
},
},
},
},
{
TypeMeta: metav1.TypeMeta{
Kind: "PersistentVolumeClaim",
APIVersion: "v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: "varlibkubelet",
Namespace: s.cluster.Namespace,
},
Spec: v1.PersistentVolumeClaimSpec{
Resources: v1.VolumeResourceRequirements{
Requests: v1.ResourceList{
"storage": resource.MustParse(s.cluster.Spec.Persistence.StorageRequestSize),
},
},
AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce},
StorageClassName: &s.cluster.Spec.Persistence.StorageClassName,
},
},
}
pvClaim = s.setupDynamicPersistence()
}
var volumes []v1.Volume
@@ -337,11 +302,15 @@ func (s *Server) StatefulServer(ctx context.Context) (*apps.StatefulSet, error)
},
}
podSpec := s.podSpec(image, name, persistent)
startupCommand, err := s.setupStartCommand()
if err != nil {
return nil, err
}
podSpec := s.podSpec(image, name, persistent, startupCommand)
podSpec.Volumes = append(podSpec.Volumes, volumes...)
podSpec.Containers[0].VolumeMounts = append(podSpec.Containers[0].VolumeMounts, volumeMounts...)
return &apps.StatefulSet{
ss := &apps.StatefulSet{
TypeMeta: metav1.TypeMeta{
Kind: "StatefulSet",
APIVersion: "apps/v1",
@@ -352,10 +321,9 @@ func (s *Server) StatefulServer(ctx context.Context) (*apps.StatefulSet, error)
Labels: selector.MatchLabels,
},
Spec: apps.StatefulSetSpec{
Replicas: &replicas,
ServiceName: headlessServiceName(s.cluster.Name),
Selector: &selector,
VolumeClaimTemplates: pvClaims,
Replicas: &replicas,
ServiceName: headlessServiceName(s.cluster.Name),
Selector: &selector,
Template: v1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: selector.MatchLabels,
@@ -363,5 +331,55 @@ func (s *Server) StatefulServer(ctx context.Context) (*apps.StatefulSet, error)
Spec: podSpec,
},
},
}, nil
}
if s.cluster.Spec.Persistence.Type == v1alpha1.DynamicNodesType {
ss.Spec.VolumeClaimTemplates = []v1.PersistentVolumeClaim{pvClaim}
}
return ss, nil
}
func (s *Server) setupDynamicPersistence() v1.PersistentVolumeClaim {
return v1.PersistentVolumeClaim{
TypeMeta: metav1.TypeMeta{
Kind: "PersistentVolumeClaim",
APIVersion: "v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: "varlibrancherk3s",
Namespace: s.cluster.Namespace,
},
Spec: v1.PersistentVolumeClaimSpec{
AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce},
StorageClassName: s.cluster.Spec.Persistence.StorageClassName,
Resources: v1.VolumeResourceRequirements{
Requests: v1.ResourceList{
"storage": resource.MustParse(s.cluster.Status.Persistence.StorageRequestSize),
},
},
},
}
}
func (s *Server) setupStartCommand() (string, error) {
var output bytes.Buffer
tmpl := singleServerTemplate
if *s.cluster.Spec.Servers > 1 {
tmpl = HAServerTemplate
}
tmplCmd, err := template.New("").Parse(tmpl)
if err != nil {
return "", err
}
if err := tmplCmd.Execute(&output, map[string]string{
"ETCD_DIR": "/var/lib/rancher/k3s/server/db/etcd",
"INIT_CONFIG": "/opt/rancher/k3s/init/config.yaml",
"SERVER_CONFIG": "/opt/rancher/k3s/server/config.yaml",
"EXTRA_ARGS": strings.Join(s.cluster.Spec.ServerArgs, " "),
}); err != nil {
return "", err
}
return output.String(), nil
}
pkg/controller/cluster/server/template.go
+16
View File
@@ -0,0 +1,16 @@
package server
var singleServerTemplate string = `
if [ -d "{{.ETCD_DIR}}" ]; then
# if directory exists then it means its not an initial run
/bin/k3s server --cluster-reset --config {{.INIT_CONFIG}} {{.EXTRA_ARGS}}
fi
rm -f /var/lib/rancher/k3s/server/db/reset-flag
/bin/k3s server --config {{.INIT_CONFIG}} {{.EXTRA_ARGS}}`
var HAServerTemplate string = `
if [ ${POD_NAME: -1} == 0 ] && [ ! -d "{{.ETCD_DIR}}" ]; then
/bin/k3s server --config {{.INIT_CONFIG}} {{.EXTRA_ARGS}}
else
/bin/k3s server --config {{.SERVER_CONFIG}} {{.EXTRA_ARGS}}
fi`