github/k3k
1
0
Fork 0
mirror of https://github.com/rancher/k3k.git synced 2026-08-18 03:46:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

Orchestrate agents and servers

Browse Source
This commit is contained in:
galal-hussein
2023-01-18 23:10:23 +02:00
parent 81fcacdf41
commit 3cd9dc8e18
7 changed files with 253 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/controller/agent.go
+154
View File
@@ -1 +1,155 @@
package controller
import (
"github.com/galal-hussein/k3k/pkg/apis/k3k.io/v1alpha1"
"github.com/galal-hussein/k3k/pkg/controller/util"
apps "k8s.io/api/apps/v1"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
func agent(cluster *v1alpha1.Cluster) *apps.Deployment {
image := getImage(cluster)
name := "k3k-agent"
return &apps.Deployment{
TypeMeta: metav1.TypeMeta{
Kind: "Deployment",
APIVersion: "apps/v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: cluster.Name + "-" + name,
Namespace: util.ClusterNamespace(cluster),
},
Spec: apps.DeploymentSpec{
Replicas: cluster.Spec.Agents,
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{
"cluster": cluster.Name,
"type": "agent",
},
},
Template: v1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: map[string]string{
"cluster": cluster.Name,
"type": "agent",
},
},
Spec: agentPodSpec(image, name),
},
},
}
}
func agentPodSpec(image, name string) v1.PodSpec {
privileged := true
return v1.PodSpec{
Volumes: []v1.Volume{
{
Name: "config",
VolumeSource: v1.VolumeSource{
Secret: &v1.SecretVolumeSource{
SecretName: name + "-config",
Items: []v1.KeyToPath{
{
Key: "config.yaml",
Path: "config.yaml",
},
},
},
},
},
{
Name: "run",
VolumeSource: v1.VolumeSource{
EmptyDir: &v1.EmptyDirVolumeSource{},
},
},
{
Name: "varrun",
VolumeSource: v1.VolumeSource{
EmptyDir: &v1.EmptyDirVolumeSource{},
},
},
{
Name: "varlibcni",
VolumeSource: v1.VolumeSource{
EmptyDir: &v1.EmptyDirVolumeSource{},
},
},
{
Name: "varlibkubelet",
VolumeSource: v1.VolumeSource{
EmptyDir: &v1.EmptyDirVolumeSource{},
},
},
{
Name: "varlibrancherk3s",
VolumeSource: v1.VolumeSource{
EmptyDir: &v1.EmptyDirVolumeSource{},
},
},
{
Name: "varlog",
VolumeSource: v1.VolumeSource{
EmptyDir: &v1.EmptyDirVolumeSource{},
},
},
},
Containers: []v1.Container{
{
Name: name,
Image: image,
SecurityContext: &v1.SecurityContext{
Privileged: &privileged,
},
Command: []string{
"/bin/sh",
},
Args: []string{
"-c",
"/bin/k3s agent --config /opt/rancher/k3s/config.yaml && true",
},
VolumeMounts: []v1.VolumeMount{
{
Name: "config",
MountPath: "/opt/rancher/k3s/",
ReadOnly: false,
},
{
Name: "run",
MountPath: "/run",
ReadOnly: false,
},
{
Name: "varrun",
MountPath: "/var/run",
ReadOnly: false,
},
{
Name: "varlibcni",
MountPath: "/var/lib/cni",
ReadOnly: false,
},
{
Name: "varlibkubelet",
MountPath: "/var/lib/kubelet",
ReadOnly: false,
},
{
Name: "varlibrancherk3s",
MountPath: "/var/lib/rancher/k3s",
ReadOnly: false,
},
{
Name: "varlog",
MountPath: "/var/log",
ReadOnly: false,
},
},
},
},
}
}
pkg/controller/config/agent.go
+32
View File
@@ -0,0 +1,32 @@
package config
import (
"fmt"
"github.com/galal-hussein/k3k/pkg/apis/k3k.io/v1alpha1"
"github.com/galal-hussein/k3k/pkg/controller/util"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
func AgentConfig(cluster *v1alpha1.Cluster, serviceIP string) v1.Secret {
config := agentConfigData(serviceIP, cluster.Spec.Token)
return v1.Secret{
TypeMeta: metav1.TypeMeta{
Kind: "Secret",
APIVersion: "v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: "k3k-agent-config",
Namespace: util.ClusterNamespace(cluster),
},
Data: map[string][]byte{
"config.yaml": []byte(config),
},
}
}
func agentConfigData(serviceIP, token string) string {
return fmt.Sprintf(`server: https://%s:6443
token: %s`, serviceIP, token)
}
pkg/controller/config.go → pkg/controller/config/server.go
+12 -9
View File
@@ -1,20 +1,21 @@
package controller
package config
import (
"fmt"
"github.com/galal-hussein/k3k/pkg/apis/k3k.io/v1alpha1"
"github.com/galal-hussein/k3k/pkg/controller/util"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
func serverConfig(cluster *v1alpha1.Cluster, init bool, serviceIP string) v1.Secret {
func ServerConfig(cluster *v1alpha1.Cluster, init bool, serviceIP string) v1.Secret {
name := "k3k-server-config"
if init {
name = "k3k-init-server-config"
}
config := configData(serviceIP, cluster.Spec.Token)
config := serverConfigData(serviceIP, cluster.Spec.Token)
if init {
config = initConfigData(cluster.Spec.Token)
}
@@ -25,7 +26,7 @@ func serverConfig(cluster *v1alpha1.Cluster, init bool, serviceIP string) v1.Sec
},
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: clusterNamespace(cluster),
Namespace: util.ClusterNamespace(cluster),
},
Data: map[string][]byte{
"config.yaml": []byte(config),
@@ -33,21 +34,23 @@ func serverConfig(cluster *v1alpha1.Cluster, init bool, serviceIP string) v1.Sec
}
}
func configData(serviceIP, token string) string {
func serverConfigData(serviceIP, token string) string {
return fmt.Sprintf(`cluster-init: true
server: https://%s:6443
token: %s
snapshotter: native
cluster-cidr: 10.40.0.0/16
service-cidr: 10.44.0.0/16
cluster-dns: 10.44.0.10`, serviceIP, token)
cluster-dns: 10.44.0.10
tls-san:
- 0.0.0.0`, serviceIP, token)
}
func initConfigData(token string) string {
return fmt.Sprintf(`cluster-init: true
token: %s
snapshotter: native
cluster-cidr: 10.40.0.0/16
service-cidr: 10.44.0.0/16
cluster-dns: 10.44.0.10`, token)
cluster-dns: 10.44.0.10
tls-san:
- 0.0.0.0`, token)
}
pkg/controller/controller.go
+31 -16
View File
@@ -5,6 +5,8 @@ import (
"fmt"
"github.com/galal-hussein/k3k/pkg/apis/k3k.io/v1alpha1"
"github.com/galal-hussein/k3k/pkg/controller/config"
"github.com/galal-hussein/k3k/pkg/controller/util"
v1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -20,7 +22,6 @@ import (
const (
ClusterController = "k3k-cluster-controller"
NamespacePrefix = "k3k-"
)
type K3KReconciler struct {
@@ -66,14 +67,14 @@ func (r *K3KReconciler) Reconcile(ctx context.Context, req reconcile.Request) (r
// we create a namespace for each new cluster
ns := &v1.Namespace{}
if err := r.Client.Get(ctx, client.ObjectKey{Name: clusterNamespace(&cluster)}, ns); err != nil {
if err := r.Client.Get(ctx, client.ObjectKey{Name: util.ClusterNamespace(&cluster)}, ns); err != nil {
if apierrors.IsNotFound(err) {
klog.Infof("creating new cluster")
return reconcile.Result{}, r.createCluster(ctx, &cluster)
} else {
return reconcile.Result{},
r.wrapErr(fmt.Sprintf("failed to get cluster namespace %s", clusterNamespace(&cluster)), err)
r.wrapErr(fmt.Sprintf("failed to get cluster namespace %s", util.ClusterNamespace(&cluster)), err)
}
}
@@ -85,10 +86,6 @@ func (r *K3KReconciler) handleDeletion(ctx context.Context, cluster *v1alpha1.Cl
return nil
}
func clusterNamespace(cluster *v1alpha1.Cluster) string {
return NamespacePrefix + cluster.Name
}
func (r *K3KReconciler) wrapErr(errString string, err error) error {
klog.Errorf("%s: %v", errString, err)
return err
@@ -98,7 +95,7 @@ func (r *K3KReconciler) createCluster(ctx context.Context, cluster *v1alpha1.Clu
// create a new namespace for the cluster
namespace := &v1.Namespace{
ObjectMeta: metav1.ObjectMeta{
Name: clusterNamespace(cluster),
Name: util.ClusterNamespace(cluster),
},
}
if err := r.Client.Create(ctx, namespace); err != nil {
@@ -114,27 +111,45 @@ func (r *K3KReconciler) createCluster(ctx context.Context, cluster *v1alpha1.Clu
service := v1.Service{}
if err := r.Client.Get(ctx,
client.ObjectKey{
Namespace: clusterNamespace(cluster),
Namespace: util.ClusterNamespace(cluster),
Name: "k3k-server-service"},
&service); err != nil {
return r.wrapErr("failed to get cluster service", err)
}
// create servers configs
initServerConfigMap := serverConfig(cluster, true, service.Spec.ClusterIP)
// create init node config
initServerConfigMap := config.ServerConfig(cluster, true, service.Spec.ClusterIP)
if err := r.Client.Create(ctx, &initServerConfigMap); err != nil {
return r.wrapErr("failed to create init configmap", err)
}
serverConfigMap := serverConfig(cluster, false, service.Spec.ClusterIP)
// create servers configuration
serverConfigMap := config.ServerConfig(cluster, false, service.Spec.ClusterIP)
if err := r.Client.Create(ctx, &serverConfigMap); err != nil {
return r.wrapErr("failed to create configmap", err)
}
// create deployment for the servers
masterNodeDeployment := server(cluster, true)
if err := r.Client.Create(ctx, masterNodeDeployment); err != nil {
return r.wrapErr("failed to create master node deployment", err)
// create deployment for the init server
// the init deployment must have only 1 replica
initNodeDeployment := server(cluster, true)
if err := r.Client.Create(ctx, initNodeDeployment); err != nil {
return r.wrapErr("failed to create init node deployment", err)
}
// create deployment for the rest of the servers
serverNodesDeployment := server(cluster, false)
if err := r.Client.Create(ctx, serverNodesDeployment); err != nil {
return r.wrapErr("failed to create server nodes deployment", err)
}
agentsConfigMap := config.AgentConfig(cluster, service.Spec.ClusterIP)
if err := r.Client.Create(ctx, &agentsConfigMap); err != nil {
return r.wrapErr("failed to create agent config", err)
}
agentsDeployment := agent(cluster)
if err := r.Client.Create(ctx, agentsDeployment); err != nil {
return r.wrapErr("failed to create agent deployment", err)
}
return nil
pkg/controller/server.go
+10 -3
View File
@@ -1,7 +1,10 @@
package controller
import (
"strconv"
"github.com/galal-hussein/k3k/pkg/apis/k3k.io/v1alpha1"
"github.com/galal-hussein/k3k/pkg/controller/util"
apps "k8s.io/api/apps/v1"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -27,22 +30,26 @@ func server(cluster *v1alpha1.Cluster, init bool) *apps.Deployment {
},
ObjectMeta: metav1.ObjectMeta{
Name: cluster.Name + "-" + name,
Namespace: clusterNamespace(cluster),
Namespace: util.ClusterNamespace(cluster),
},
Spec: apps.DeploymentSpec{
Replicas: &replicas,
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{
"cluster": cluster.Name,
"role": "server",
"init": strconv.FormatBool(init),
},
},
Template: v1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: map[string]string{
"cluster": cluster.Name,
"role": "server",
"init": strconv.FormatBool(init),
},
},
Spec: podSpec(image, name),
Spec: serverPodSpec(image, name),
},
},
}
@@ -52,7 +59,7 @@ func getImage(cluster *v1alpha1.Cluster) string {
return "rancher/k3s:" + cluster.Spec.Version
}
func podSpec(image, name string) v1.PodSpec {
func serverPodSpec(image, name string) v1.PodSpec {
privileged := true
return v1.PodSpec{
Volumes: []v1.Volume{
pkg/controller/service.go
+3 -1
View File
@@ -2,6 +2,7 @@ package controller
import (
"github.com/galal-hussein/k3k/pkg/apis/k3k.io/v1alpha1"
"github.com/galal-hussein/k3k/pkg/controller/util"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
@@ -14,12 +15,13 @@ func service(cluster *v1alpha1.Cluster) v1.Service {
},
ObjectMeta: metav1.ObjectMeta{
Name: "k3k-server-service",
Namespace: clusterNamespace(cluster),
Namespace: util.ClusterNamespace(cluster),
},
Spec: v1.ServiceSpec{
Type: v1.ServiceTypeClusterIP,
Selector: map[string]string{
"cluster": cluster.Name,
"role": "server",
},
Ports: []v1.ServicePort{
{
pkg/controller/util/util.go
+11
View File
@@ -0,0 +1,11 @@
package util
import "github.com/galal-hussein/k3k/pkg/apis/k3k.io/v1alpha1"
const (
NamespacePrefix = "k3k-"
)
func ClusterNamespace(cluster *v1alpha1.Cluster) string {
return NamespacePrefix + cluster.Name
}