github/k3k
1
0
Fork 0
mirror of https://github.com/rancher/k3k.git synced 2026-08-18 20:07:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add cluster persistence with statefulsets (#55)

Browse Source 
* Add cluster persistence with statefulsets

Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>

* fix comments

Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>

---------

Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>
This commit is contained in:
Hussein Galal
2023-08-08 23:23:55 +03:00
committed by GitHub
parent def1746f1f
commit 9dbd0bef44
11 changed files with 453 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files
charts/k3k/crds/cluster.yaml
+10
View File
@@ -43,6 +43,16 @@ spec:
type: array
items:
type: string
persistence:
type: object
properties:
type:
type: string
default: "ephermal"
storageClassName:
type: string
storageRequestSize:
type: string
expose:
type: object
properties:
cli/cmds/cluster/create.go
+31 -9
View File
@@ -12,6 +12,7 @@ import (
"github.com/rancher/k3k/cli/cmds"
"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
"github.com/rancher/k3k/pkg/controller/cluster"
"github.com/rancher/k3k/pkg/controller/util"
"github.com/sirupsen/logrus"
"github.com/urfave/cli"
@@ -45,15 +46,17 @@ func init() {
}
var (
name string
token string
clusterCIDR string
serviceCIDR string
servers int64
agents int64
serverArgs cli.StringSlice
agentArgs cli.StringSlice
version string
name string
token string
clusterCIDR string
serviceCIDR string
servers int64
agents int64
serverArgs cli.StringSlice
agentArgs cli.StringSlice
persistenceType string
storageClassName string
version string
clusterCreateFlags = []cli.Flag{
cli.StringFlag{
@@ -87,6 +90,17 @@ var (
Usage: "service CIDR",
Destination: &serviceCIDR,
},
cli.StringFlag{
Name: "persistence-type",
Usage: "Persistence mode for the nodes (ephermal, static, dynamic)",
Value: cluster.EphermalNodesType,
Destination: &persistenceType,
},
cli.StringFlag{
Name: "storage-class-name",
Usage: "Storage class name for dynamic persistence type",
Destination: &storageClassName,
},
cli.StringSliceFlag{
Name: "server-args",
Usage: "servers extra arguments",
@@ -185,6 +199,10 @@ func createCluster(clx *cli.Context) error {
}
func validateCreateFlags(clx *cli.Context) error {
if persistenceType != cluster.EphermalNodesType &&
persistenceType != cluster.DynamicNodesType {
return errors.New("invalid persistence type")
}
if token == "" {
return errors.New("empty cluster token")
}
@@ -220,6 +238,10 @@ func newCluster(name, token string, servers, agents int32, clusterCIDR, serviceC
ServerArgs: serverArgs,
AgentArgs: agentArgs,
Version: version,
Persistence: &v1alpha1.PersistenceConfig{
Type: persistenceType,
StorageClassName: storageClassName,
},
},
}
}
pkg/apis/k3k.io/v1alpha1/types.go
+9 -1
View File
@@ -28,7 +28,8 @@ type ClusterSpec struct {
AgentArgs []string `json:"agentArgs,omitempty"`
TLSSANs []string `json:"tlsSANs,omitempty"`
Expose *ExposeConfig `json:"expose,omitempty"`
Persistence *PersistenceConfig `json:"persistence,omitempty"`
Expose *ExposeConfig `json:"expose,omitempty"`
}
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
@@ -40,6 +41,13 @@ type ClusterList struct {
Items []Cluster `json:"items"`
}
type PersistenceConfig struct {
// Type can be ephermal, static, dynamic
Type string `json:"type"`
StorageClassName string `json:"storageClassName,omitempty"`
StorageRequestSize string `json:"storageRequestSize,omitempty"`
}
type ExposeConfig struct {
Ingress *IngressConfig `json:"ingress"`
LoadBalancer *LoadBalancerConfig `json:"loadbalancer"`
pkg/apis/k3k.io/v1alpha1/zz_generated.deepcopy.go
+21
View File
@@ -212,6 +212,11 @@ func (in *ClusterSpec) DeepCopyInto(out *ClusterSpec) {
*out = make([]string, len(*in))
copy(*out, *in)
}
if in.Persistence != nil {
in, out := &in.Persistence, &out.Persistence
*out = new(PersistenceConfig)
**out = **in
}
if in.Expose != nil {
in, out := &in.Expose, &out.Expose
*out = new(ExposeConfig)
@@ -324,3 +329,19 @@ func (in *NodePortConfig) DeepCopy() *NodePortConfig {
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *PersistenceConfig) DeepCopyInto(out *PersistenceConfig) {
*out = *in
return
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PersistenceConfig.
func (in *PersistenceConfig) DeepCopy() *PersistenceConfig {
if in == nil {
return nil
}
out := new(PersistenceConfig)
in.DeepCopyInto(out)
return out
}
pkg/controller/cluster/agent/agent.go
+99 -20
View File
@@ -5,22 +5,22 @@ import (
"github.com/rancher/k3k/pkg/controller/util"
apps "k8s.io/api/apps/v1"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/utils/pointer"
)
const agentName = "k3k-agent"
func Agent(cluster *v1alpha1.Cluster) *apps.Deployment {
image := util.K3SImage(cluster)
const name = "k3k-agent"
return &apps.Deployment{
TypeMeta: metav1.TypeMeta{
Kind: "Deployment",
APIVersion: "apps/v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: cluster.Name + "-" + name,
Name: cluster.Name + "-" + agentName,
Namespace: util.ClusterNamespace(cluster),
},
Spec: apps.DeploymentSpec{
@@ -38,15 +38,89 @@ func Agent(cluster *v1alpha1.Cluster) *apps.Deployment {
"type": "agent",
},
},
Spec: agentPodSpec(image, name, cluster.Spec.AgentArgs),
Spec: agentPodSpec(image, agentName, cluster.Spec.AgentArgs, false),
},
},
}
}
func agentPodSpec(image, name string, args []string) v1.PodSpec {
func StatefulAgent(cluster *v1alpha1.Cluster) *apps.StatefulSet {
image := util.K3SImage(cluster)
return &apps.StatefulSet{
TypeMeta: metav1.TypeMeta{
Kind: "Statefulset",
APIVersion: "apps/v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: cluster.Name + "-" + agentName,
Namespace: util.ClusterNamespace(cluster),
},
Spec: apps.StatefulSetSpec{
ServiceName: cluster.Name + "-" + agentName + "-headless",
Replicas: cluster.Spec.Agents,
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{
"cluster": cluster.Name,
"type": "agent",
},
},
VolumeClaimTemplates: []v1.PersistentVolumeClaim{
{
TypeMeta: metav1.TypeMeta{
Kind: "PersistentVolumeClaim",
APIVersion: "v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: "varlibrancherk3s",
Namespace: util.ClusterNamespace(cluster),
},
Spec: v1.PersistentVolumeClaimSpec{
AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce},
StorageClassName: &cluster.Spec.Persistence.StorageClassName,
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
"storage": resource.MustParse(cluster.Spec.Persistence.StorageRequestSize),
},
},
},
},
{
TypeMeta: metav1.TypeMeta{
Kind: "PersistentVolumeClaim",
APIVersion: "v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: "varlibkubelet",
Namespace: util.ClusterNamespace(cluster),
},
Spec: v1.PersistentVolumeClaimSpec{
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
"storage": resource.MustParse(cluster.Spec.Persistence.StorageRequestSize),
},
},
AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce},
StorageClassName: &cluster.Spec.Persistence.StorageClassName,
},
},
},
Template: v1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: map[string]string{
"cluster": cluster.Name,
"type": "agent",
},
},
Spec: agentPodSpec(image, agentName, cluster.Spec.AgentArgs, true),
},
},
}
}
func agentPodSpec(image, name string, args []string, statefulSet bool) v1.PodSpec {
args = append([]string{"agent", "--config", "/opt/rancher/k3s/config.yaml"}, args...)
return v1.PodSpec{
podSpec := v1.PodSpec{
Volumes: []v1.Volume{
{
Name: "config",
@@ -80,18 +154,6 @@ func agentPodSpec(image, name string, args []string) v1.PodSpec {
EmptyDir: &v1.EmptyDirVolumeSource{},
},
},
{
Name: "varlibkubelet",
VolumeSource: v1.VolumeSource{
EmptyDir: &v1.EmptyDirVolumeSource{},
},
},
{
Name: "varlibrancherk3s",
VolumeSource: v1.VolumeSource{
EmptyDir: &v1.EmptyDirVolumeSource{},
},
},
{
Name: "varlog",
VolumeSource: v1.VolumeSource{
@@ -104,7 +166,7 @@ func agentPodSpec(image, name string, args []string) v1.PodSpec {
Name: name,
Image: image,
SecurityContext: &v1.SecurityContext{
Privileged: pointer.BoolPtr(true),
Privileged: pointer.Bool(true),
},
Command: []string{
"/bin/k3s",
@@ -150,4 +212,21 @@ func agentPodSpec(image, name string, args []string) v1.PodSpec {
},
},
}
if !statefulSet {
podSpec.Volumes = append(podSpec.Volumes, v1.Volume{
Name: "varlibkubelet",
VolumeSource: v1.VolumeSource{
EmptyDir: &v1.EmptyDirVolumeSource{},
},
}, v1.Volume{
Name: "varlibrancherk3s",
VolumeSource: v1.VolumeSource{
EmptyDir: &v1.EmptyDirVolumeSource{},
},
},
)
}
return podSpec
}
pkg/controller/cluster/agent/service.go
+30
View File
@@ -0,0 +1,30 @@
package agent
import (
"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
"github.com/rancher/k3k/pkg/controller/util"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
func StatefulAgentService(cluster *v1alpha1.Cluster) *v1.Service {
return &v1.Service{
TypeMeta: metav1.TypeMeta{
Kind: "Service",
APIVersion: "v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: cluster.Name + "-" + agentName + "-headless",
Namespace: util.ClusterNamespace(cluster),
},
Spec: v1.ServiceSpec{
Type: v1.ServiceTypeClusterIP,
ClusterIP: v1.ClusterIPNone,
Selector: map[string]string{
"cluster": cluster.Name,
"role": "agent",
},
Ports: []v1.ServicePort{},
},
}
}
pkg/controller/cluster/config/agent.go
+2 -1
View File
@@ -29,5 +29,6 @@ func Agent(cluster *v1alpha1.Cluster, serviceIP string) v1.Secret {
func agentData(serviceIP, token string) string {
return fmt.Sprintf(`server: https://%s:6443
token: %s`, serviceIP, token)
token: %s
with-node-id: true`, serviceIP, token)
}
pkg/controller/cluster/config/server.go
+3 -3
View File
@@ -13,7 +13,7 @@ func Server(cluster *v1alpha1.Cluster, init bool, serviceIP string) (*v1.Secret,
name = "k3k-init-server-config"
}
config := serverData(serviceIP, cluster)
config := serverConfigData(serviceIP, cluster)
if init {
config = initConfigData(cluster)
}
@@ -32,8 +32,8 @@ func Server(cluster *v1alpha1.Cluster, init bool, serviceIP string) (*v1.Secret,
}, nil
}
func serverData(serviceIP string, cluster *v1alpha1.Cluster) string {
return "cluster-init: true\nserver: https://" + serviceIP + ":6443" + serverOptions(cluster)
func serverConfigData(serviceIP string, cluster *v1alpha1.Cluster) string {
return "cluster-init: true\nserver: https://" + serviceIP + ":6443\n" + serverOptions(cluster)
}
func initConfigData(cluster *v1alpha1.Cluster) string {
pkg/controller/cluster/controller.go
+97 -2
View File
@@ -25,6 +25,8 @@ import (
const (
clusterController = "k3k-cluster-controller"
clusterFinalizerName = "cluster.k3k.io/finalizer"
EphermalNodesType = "ephermal"
DynamicNodesType = "dynamic"
)
type ClusterReconciler struct {
@@ -97,6 +99,15 @@ func (c *ClusterReconciler) Reconcile(ctx context.Context, req reconcile.Request
}
func (c *ClusterReconciler) createCluster(ctx context.Context, cluster *v1alpha1.Cluster) error {
if cluster.Spec.Persistence == nil {
// default to ephermal nodes
cluster.Spec.Persistence = &v1alpha1.PersistenceConfig{
Type: EphermalNodesType,
}
}
if err := c.Client.Update(ctx, cluster); err != nil {
return util.WrapErr("failed to update cluster with persistence type", err)
}
// create a new namespace for the cluster
if err := c.createNamespace(ctx, cluster); err != nil {
return util.WrapErr("failed to create ns", err)
@@ -112,8 +123,19 @@ func (c *ClusterReconciler) createCluster(ctx context.Context, cluster *v1alpha1
return util.WrapErr("failed to create cluster configs", err)
}
if err := c.createDeployments(ctx, cluster); err != nil {
return util.WrapErr("failed to create servers and agents deployment", err)
// creating statefulsets in case the user chose a persistence type other than ephermal
if cluster.Spec.Persistence.Type != EphermalNodesType {
if cluster.Spec.Persistence.StorageRequestSize == "" {
// default to 1G of request size
cluster.Spec.Persistence.StorageRequestSize = "1G"
}
if err := c.createStatefulSets(ctx, cluster); err != nil {
return util.WrapErr("failed to create servers and agents statefulsets", err)
}
} else {
if err := c.createDeployments(ctx, cluster); err != nil {
return util.WrapErr("failed to create servers and agents deployment", err)
}
}
if cluster.Spec.Expose != nil {
@@ -278,6 +300,79 @@ func (c *ClusterReconciler) createDeployments(ctx context.Context, cluster *v1al
return nil
}
func (c *ClusterReconciler) createStatefulSets(ctx context.Context, cluster *v1alpha1.Cluster) error {
// create headless service for the init statefulset
initServerStatefulService := server.StatefulServerService(cluster, true)
if err := controllerutil.SetControllerReference(cluster, initServerStatefulService, c.Scheme); err != nil {
return err
}
if err := c.Client.Create(ctx, initServerStatefulService); err != nil {
if !apierrors.IsAlreadyExists(err) {
return err
}
}
// create statefulsets for the init server
// the init statefulset must have only 1 replica
initServerStatefulSet := server.StatefulServer(cluster, true)
if err := controllerutil.SetControllerReference(cluster, initServerStatefulSet, c.Scheme); err != nil {
return err
}
if err := c.Client.Create(ctx, initServerStatefulSet); err != nil {
if !apierrors.IsAlreadyExists(err) {
return err
}
}
// create statefulset for the rest of the servers
// create headless service for the server statefulset
serverStatefulService := server.StatefulServerService(cluster, false)
if err := controllerutil.SetControllerReference(cluster, serverStatefulService, c.Scheme); err != nil {
return err
}
if err := c.Client.Create(ctx, serverStatefulService); err != nil {
if !apierrors.IsAlreadyExists(err) {
return err
}
}
serversStatefulSet := server.StatefulServer(cluster, false)
if err := controllerutil.SetControllerReference(cluster, serversStatefulSet, c.Scheme); err != nil {
return err
}
if err := c.Client.Create(ctx, serversStatefulSet); err != nil {
if !apierrors.IsAlreadyExists(err) {
return err
}
}
// create headless service for the agents statefulset
agentStatefulService := agent.StatefulAgentService(cluster)
if err := controllerutil.SetControllerReference(cluster, agentStatefulService, c.Scheme); err != nil {
return err
}
if err := c.Client.Create(ctx, agentStatefulService); err != nil {
if !apierrors.IsAlreadyExists(err) {
return err
}
}
agentsStatefulSet := agent.StatefulAgent(cluster)
if err := controllerutil.SetControllerReference(cluster, agentsStatefulSet, c.Scheme); err != nil {
return err
}
if err := c.Client.Create(ctx, agentsStatefulSet); err != nil {
if !apierrors.IsAlreadyExists(err) {
return err
}
}
return nil
}
func (c *ClusterReconciler) createCIDRPools(ctx context.Context) error {
if err := c.Client.Create(ctx, &v1alpha1.CIDRAllocationPool{}); err != nil {
if !apierrors.IsAlreadyExists(err) {
pkg/controller/cluster/server/server.go
+116 -18
View File
@@ -7,17 +7,23 @@ import (
"github.com/rancher/k3k/pkg/controller/util"
apps "k8s.io/api/apps/v1"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/utils/pointer"
)
const (
serverName = "k3k-server"
initServerName = "k3k-init-server"
)
func Server(cluster *v1alpha1.Cluster, init bool) *apps.Deployment {
var replicas int32
image := util.K3SImage(cluster)
name := "k3k-server"
name := serverName
if init {
name = "k3k-init-server"
name = initServerName
}
replicas = *cluster.Spec.Servers - 1
@@ -51,15 +57,15 @@ func Server(cluster *v1alpha1.Cluster, init bool) *apps.Deployment {
"init": strconv.FormatBool(init),
},
},
Spec: serverPodSpec(image, name, cluster.Spec.ServerArgs),
Spec: serverPodSpec(image, name, cluster.Spec.ServerArgs, false),
},
},
}
}
func serverPodSpec(image, name string, args []string) v1.PodSpec {
func serverPodSpec(image, name string, args []string, statefulSet bool) v1.PodSpec {
args = append([]string{"server", "--config", "/opt/rancher/k3s/config.yaml"}, args...)
return v1.PodSpec{
podSpec := v1.PodSpec{
Volumes: []v1.Volume{
{
Name: "config",
@@ -93,18 +99,6 @@ func serverPodSpec(image, name string, args []string) v1.PodSpec {
EmptyDir: &v1.EmptyDirVolumeSource{},
},
},
{
Name: "varlibkubelet",
VolumeSource: v1.VolumeSource{
EmptyDir: &v1.EmptyDirVolumeSource{},
},
},
{
Name: "varlibrancherk3s",
VolumeSource: v1.VolumeSource{
EmptyDir: &v1.EmptyDirVolumeSource{},
},
},
{
Name: "varlog",
VolumeSource: v1.VolumeSource{
@@ -117,7 +111,7 @@ func serverPodSpec(image, name string, args []string) v1.PodSpec {
Name: name,
Image: image,
SecurityContext: &v1.SecurityContext{
Privileged: pointer.BoolPtr(true),
Privileged: pointer.Bool(true),
},
Command: []string{
"/bin/k3s",
@@ -163,4 +157,108 @@ func serverPodSpec(image, name string, args []string) v1.PodSpec {
},
},
}
if !statefulSet {
podSpec.Volumes = append(podSpec.Volumes, v1.Volume{
Name: "varlibkubelet",
VolumeSource: v1.VolumeSource{
EmptyDir: &v1.EmptyDirVolumeSource{},
},
}, v1.Volume{
Name: "varlibrancherk3s",
VolumeSource: v1.VolumeSource{
EmptyDir: &v1.EmptyDirVolumeSource{},
},
},
)
}
return podSpec
}
func StatefulServer(cluster *v1alpha1.Cluster, init bool) *apps.StatefulSet {
var replicas int32
image := util.K3SImage(cluster)
name := serverName
if init {
name = initServerName
}
replicas = *cluster.Spec.Servers - 1
if init {
replicas = 1
}
return &apps.StatefulSet{
TypeMeta: metav1.TypeMeta{
Kind: "Deployment",
APIVersion: "apps/v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: cluster.Name + "-" + name,
Namespace: util.ClusterNamespace(cluster),
},
Spec: apps.StatefulSetSpec{
Replicas: &replicas,
ServiceName: cluster.Name + "-" + name + "-headless",
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{
"cluster": cluster.Name,
"role": "server",
"init": strconv.FormatBool(init),
},
},
VolumeClaimTemplates: []v1.PersistentVolumeClaim{
{
TypeMeta: metav1.TypeMeta{
Kind: "PersistentVolumeClaim",
APIVersion: "v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: "varlibrancherk3s",
Namespace: util.ClusterNamespace(cluster),
},
Spec: v1.PersistentVolumeClaimSpec{
AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce},
StorageClassName: &cluster.Spec.Persistence.StorageClassName,
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
"storage": resource.MustParse(cluster.Spec.Persistence.StorageRequestSize),
},
},
},
},
{
TypeMeta: metav1.TypeMeta{
Kind: "PersistentVolumeClaim",
APIVersion: "v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: "varlibkubelet",
Namespace: util.ClusterNamespace(cluster),
},
Spec: v1.PersistentVolumeClaimSpec{
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
"storage": resource.MustParse(cluster.Spec.Persistence.StorageRequestSize),
},
},
AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce},
StorageClassName: &cluster.Spec.Persistence.StorageClassName,
},
},
},
Template: v1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: map[string]string{
"cluster": cluster.Name,
"role": "server",
"init": strconv.FormatBool(init),
},
},
Spec: serverPodSpec(image, name, cluster.Spec.ServerArgs, true),
},
},
}
}
pkg/controller/cluster/server/service.go
+35
View File
@@ -1,6 +1,8 @@
package server
import (
"strconv"
"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
"github.com/rancher/k3k/pkg/controller/util"
v1 "k8s.io/api/core/v1"
@@ -42,3 +44,36 @@ func Service(cluster *v1alpha1.Cluster) *v1.Service {
},
}
}
func StatefulServerService(cluster *v1alpha1.Cluster, init bool) *v1.Service {
name := serverName
if init {
name = initServerName
}
return &v1.Service{
TypeMeta: metav1.TypeMeta{
Kind: "Service",
APIVersion: "v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: cluster.Name + "-" + name + "-headless",
Namespace: util.ClusterNamespace(cluster),
},
Spec: v1.ServiceSpec{
Type: v1.ServiceTypeClusterIP,
ClusterIP: v1.ClusterIPNone,
Selector: map[string]string{
"cluster": cluster.Name,
"role": "server",
"init": strconv.FormatBool(init),
},
Ports: []v1.ServicePort{
{
Name: "k3s-server-port",
Protocol: v1.ProtocolTCP,
Port: 6443,
},
},
},
}
}