github/k3k
1
0
Fork 0
mirror of https://github.com/rancher/k3k.git synced 2026-08-19 04:16:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #14 from galal-hussein/add_server_agent_args

Browse Source 
Add server and agent args
This commit is contained in:
Hussein Galal
2023-01-31 02:58:50 +02:00
committed by GitHub
parent 9a21ceb566 e31ab4102c
commit 6640bdbe50
14 changed files with 288 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crd/cluster.yaml
+28 -1
View File
@@ -25,8 +25,35 @@ spec:
type: integer
token:
type: string
ingressClassName:
clusterCIDR:
type: string
serviceCIDR:
type: string
clusterDNS:
type: string
serverArgs:
type: array
items:
type: string
agentArgs:
type: array
items:
type: string
expose:
type: object
properties:
ingress:
type: object
properties:
enabled:
type: boolean
ingressClassName:
type: string
loadbalancer:
type: object
properties:
enabled:
type: boolean
scope: Namespaced
names:
plural: clusters
examples/multiple-servers.yaml
+19
View File
@@ -0,0 +1,19 @@
apiVersion: k3k.io/v1alpha1
kind: Cluster
metadata:
name: example1
namespace: default
spec:
servers: 1
agents: 3
token: test
version: v1.26.0-k3s2
clusterCIDR: 10.30.0.0/16
serviceCIDR: 10.31.0.0/16
clusterDNS: 10.30.0.10
serverArgs:
- "--write-kubeconfig-mode=777"
expose:
ingress:
enabled: true
ingressClassName: "nginx"
examples/multiple-servers/multiple-servers.yaml
-11
View File
@@ -1,11 +0,0 @@
apiVersion: k3k.io/v1alpha1
kind: Cluster
metadata:
name: multiple-servers
namespace: default
spec:
servers: 2
agents: 3
token: test
version: v1.26.0-k3s2
ingressClassName: traefik
examples/single-server.yaml
+19
View File
@@ -0,0 +1,19 @@
apiVersion: k3k.io/v1alpha1
kind: Cluster
metadata:
name: single-server
namespace: default
spec:
servers: 1
agents: 3
token: test
version: v1.26.0-k3s2
clusterCIDR: 10.30.0.0/16
serviceCIDR: 10.31.0.0/16
clusterDNS: 10.30.0.10
serverArgs:
- "--write-kubeconfig-mode=777"
expose:
ingress:
enabled: true
ingressClassName: "nginx"
examples/single-server/single-server.yaml
-11
View File
@@ -1,11 +0,0 @@
apiVersion: k3k.io/v1alpha1
kind: Cluster
metadata:
name: single-server
namespace: default
spec:
servers: 1
agents: 3
token: test
version: v1.26.0-k3s2
ingressClassName: traefik
pkg/apis/k3k.io/v1alpha1/types.go
+27 -6
View File
@@ -15,12 +15,19 @@ type Cluster struct {
}
type ClusterSpec struct {
Name string `json:"name"`
Version string `json:"version"`
Servers *int32 `json:"servers"`
Agents *int32 `json:"agents"`
Token string `json:"token"`
IngressClassName string `json:"ingressClassName"`
Name string `json:"name"`
Version string `json:"version"`
Servers *int32 `json:"servers"`
Agents *int32 `json:"agents"`
Token string `json:"token"`
ClusterCIDR string `json:"clusterCIDR,omitempty"`
ServiceCIDR string `json:"serviceCIDR,omitempty"`
ClusterDNS string `json:"clusterDNS,omitempty"`
ServerArgs []string `json:"serverArgs,omitempty"`
AgentArgs []string `json:"agentArgs,omitempty"`
Expose ExposeConfig `json:"expose,omitempty"`
}
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
@@ -31,3 +38,17 @@ type ClusterList struct {
Items []Cluster `json:"items"`
}
type ExposeConfig struct {
Ingress *IngressConfig `json:"ingress"`
LoadBalancer *LoadBalancerConfig `json:"loadbalancer"`
}
type IngressConfig struct {
Enabled bool `json:"enabled"`
IngressClassName string `json:"ingressClassName"`
}
type LoadBalancerConfig struct {
Enabled bool `json:"enabled"`
}
pkg/apis/k3k.io/v1alpha1/zz_generated.deepcopy.go
+69
View File
@@ -82,6 +82,17 @@ func (in *ClusterSpec) DeepCopyInto(out *ClusterSpec) {
*out = new(int32)
**out = **in
}
if in.ServerArgs != nil {
in, out := &in.ServerArgs, &out.ServerArgs
*out = make([]string, len(*in))
copy(*out, *in)
}
if in.AgentArgs != nil {
in, out := &in.AgentArgs, &out.AgentArgs
*out = make([]string, len(*in))
copy(*out, *in)
}
in.Expose.DeepCopyInto(&out.Expose)
return
}
@@ -94,3 +105,61 @@ func (in *ClusterSpec) DeepCopy() *ClusterSpec {
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *ExposeConfig) DeepCopyInto(out *ExposeConfig) {
*out = *in
if in.Ingress != nil {
in, out := &in.Ingress, &out.Ingress
*out = new(IngressConfig)
**out = **in
}
if in.LoadBalancer != nil {
in, out := &in.LoadBalancer, &out.LoadBalancer
*out = new(LoadBalancerConfig)
**out = **in
}
return
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExposeConfig.
func (in *ExposeConfig) DeepCopy() *ExposeConfig {
if in == nil {
return nil
}
out := new(ExposeConfig)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *IngressConfig) DeepCopyInto(out *IngressConfig) {
*out = *in
return
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IngressConfig.
func (in *IngressConfig) DeepCopy() *IngressConfig {
if in == nil {
return nil
}
out := new(IngressConfig)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *LoadBalancerConfig) DeepCopyInto(out *LoadBalancerConfig) {
*out = *in
return
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LoadBalancerConfig.
func (in *LoadBalancerConfig) DeepCopy() *LoadBalancerConfig {
if in == nil {
return nil
}
out := new(LoadBalancerConfig)
in.DeepCopyInto(out)
return out
}
pkg/controller/cluster/agent/agent.go
+7 -3
View File
@@ -1,6 +1,8 @@
package agent
import (
"strings"
"github.com/galal-hussein/k3k/pkg/apis/k3k.io/v1alpha1"
"github.com/galal-hussein/k3k/pkg/controller/util"
apps "k8s.io/api/apps/v1"
@@ -37,13 +39,13 @@ func Agent(cluster *v1alpha1.Cluster) *apps.Deployment {
"type": "agent",
},
},
Spec: agentPodSpec(image, name),
Spec: agentPodSpec(image, name, cluster.Spec.AgentArgs),
},
},
}
}
func agentPodSpec(image, name string) v1.PodSpec {
func agentPodSpec(image, name string, args []string) v1.PodSpec {
privileged := true
return v1.PodSpec{
Volumes: []v1.Volume{
@@ -110,7 +112,9 @@ func agentPodSpec(image, name string) v1.PodSpec {
},
Args: []string{
"-c",
"/bin/k3s agent --config /opt/rancher/k3s/config.yaml && true",
"/bin/k3s agent --config /opt/rancher/k3s/config.yaml " +
strings.Join(args, " ") +
" && true",
},
VolumeMounts: []v1.VolumeMount{
{
pkg/controller/cluster/config/server.go
+29 -19
View File
@@ -9,17 +9,17 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
func ServerConfig(cluster *v1alpha1.Cluster, init bool, serviceIP string) v1.Secret {
func ServerConfig(cluster *v1alpha1.Cluster, init bool, serviceIP string) (*v1.Secret, error) {
name := "k3k-server-config"
if init {
name = "k3k-init-server-config"
}
config := serverConfigData(serviceIP, cluster.Spec.Token)
config := serverConfigData(serviceIP, cluster)
if init {
config = initConfigData(cluster.Spec.Token)
config = initConfigData(cluster)
}
return v1.Secret{
return &v1.Secret{
TypeMeta: metav1.TypeMeta{
Kind: "Secret",
APIVersion: "v1",
@@ -31,26 +31,36 @@ func ServerConfig(cluster *v1alpha1.Cluster, init bool, serviceIP string) v1.Sec
Data: map[string][]byte{
"config.yaml": []byte(config),
},
}
}, nil
}
func serverConfigData(serviceIP, token string) string {
func serverConfigData(serviceIP string, cluster *v1alpha1.Cluster) string {
opts := serverOptions(cluster)
return fmt.Sprintf(`cluster-init: true
server: https://%s:6443
token: %s
cluster-cidr: 10.40.0.0/16
service-cidr: 10.44.0.0/16
cluster-dns: 10.44.0.10
tls-san:
- 0.0.0.0`, serviceIP, token)
%s`, serviceIP, opts)
}
func initConfigData(token string) string {
func initConfigData(cluster *v1alpha1.Cluster) string {
opts := serverOptions(cluster)
return fmt.Sprintf(`cluster-init: true
token: %s
cluster-cidr: 10.40.0.0/16
service-cidr: 10.44.0.0/16
cluster-dns: 10.44.0.10
tls-san:
- 0.0.0.0`, token)
%s`, opts)
}
func serverOptions(cluster *v1alpha1.Cluster) string {
opts := ""
// TODO: generate token if not found
if cluster.Spec.Token != "" {
opts = fmt.Sprintf("token: %s\n", cluster.Spec.Token)
}
if cluster.Spec.ClusterCIDR != "" {
opts = fmt.Sprintf("%scluster-cidr: %s\n", opts, cluster.Spec.ClusterCIDR)
}
if cluster.Spec.ServiceCIDR != "" {
opts = fmt.Sprintf("%sservice-cidr: %s\n", opts, cluster.Spec.ServiceCIDR)
}
if cluster.Spec.ClusterDNS != "" {
opts = fmt.Sprintf("%scluster-dns: %s\n", opts, cluster.Spec.ClusterDNS)
}
return opts
}
pkg/controller/cluster/controller.go
+20 -12
View File
@@ -119,16 +119,23 @@ func (r *ClusterReconciler) createCluster(ctx context.Context, cluster *v1alpha1
}
// create init node config
initServerConfigMap := config.ServerConfig(cluster, true, service.Spec.ClusterIP)
if err := r.Client.Create(ctx, &initServerConfigMap); err != nil {
initServerConfigMap, err := config.ServerConfig(cluster, true, service.Spec.ClusterIP)
if err != nil {
return util.WrapErr("failed to get init server config", err)
}
if err := r.Client.Create(ctx, initServerConfigMap); err != nil {
if !apierrors.IsAlreadyExists(err) {
return util.WrapErr("failed to create init configmap", err)
}
}
// create servers configuration
serverConfigMap := config.ServerConfig(cluster, false, service.Spec.ClusterIP)
if err := r.Client.Create(ctx, &serverConfigMap); err != nil {
serverConfigMap, err := config.ServerConfig(cluster, false, service.Spec.ClusterIP)
if err != nil {
return util.WrapErr("failed to get server config", err)
}
if err := r.Client.Create(ctx, serverConfigMap); err != nil {
if !apierrors.IsAlreadyExists(err) {
return util.WrapErr("failed to create configmap", err)
}
@@ -165,14 +172,15 @@ func (r *ClusterReconciler) createCluster(ctx context.Context, cluster *v1alpha1
}
}
// create ingress with random port for the server
serverIngress, err := server.Ingress(ctx, cluster, r.Client)
if err != nil {
return util.WrapErr("failed to create ingress object", err)
}
if err := r.Client.Create(ctx, serverIngress); err != nil {
if !apierrors.IsAlreadyExists(err) {
return util.WrapErr("failed to create server ingress", err)
if cluster.Spec.Expose.Ingress.Enabled {
serverIngress, err := server.Ingress(ctx, cluster, r.Client)
if err != nil {
return util.WrapErr("failed to create ingress object", err)
}
if err := r.Client.Create(ctx, serverIngress); err != nil {
if !apierrors.IsAlreadyExists(err) {
return util.WrapErr("failed to create server ingress", err)
}
}
}
pkg/controller/cluster/server/ingress.go
+26 -38
View File
@@ -5,25 +5,27 @@ import (
"github.com/galal-hussein/k3k/pkg/apis/k3k.io/v1alpha1"
"github.com/galal-hussein/k3k/pkg/controller/util"
v1 "k8s.io/api/core/v1"
networkingv1 "k8s.io/api/networking/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/client"
)
var (
pathType = networkingv1.PathTypePrefix
const (
wildcardDNS = ".sslip.io"
nginxSSLPassthroughAnnotation = "nginx.ingress.kubernetes.io/ssl-passthrough"
nginxBackendProtocolAnnotation = "nginx.ingress.kubernetes.io/backend-protocol"
nginxSSLRedirectAnnotation = "nginx.ingress.kubernetes.io/ssl-redirect"
)
func Ingress(ctx context.Context, cluster *v1alpha1.Cluster, client client.Client) (*networkingv1.Ingress, error) {
addresses, err := addresses(ctx, client)
addresses, err := util.Addresses(ctx, client)
if err != nil {
return nil, err
}
ingressRules := ingressRules(cluster, addresses)
return &networkingv1.Ingress{
ingress := &networkingv1.Ingress{
TypeMeta: metav1.TypeMeta{
Kind: "Ingress",
APIVersion: "networking.k8s.io/v1",
@@ -33,47 +35,19 @@ func Ingress(ctx context.Context, cluster *v1alpha1.Cluster, client client.Clien
Namespace: util.ClusterNamespace(cluster),
},
Spec: networkingv1.IngressSpec{
IngressClassName: &cluster.Spec.IngressClassName,
IngressClassName: &cluster.Spec.Expose.Ingress.IngressClassName,
Rules: ingressRules,
},
}, nil
}
// return all the nodes external addresses, if not found then return internal addresses
func addresses(ctx context.Context, client client.Client) ([]string, error) {
addresses := []string{}
nodeList := v1.NodeList{}
if err := client.List(ctx, &nodeList); err != nil {
return nil, err
}
for _, node := range nodeList.Items {
addresses = append(addresses, GetNodeAddress(&node))
}
configureIngressOptions(ingress, cluster.Spec.Expose.Ingress.IngressClassName)
return addresses, nil
}
func GetNodeAddress(node *v1.Node) string {
externalIP := ""
internalIP := ""
for _, ip := range node.Status.Addresses {
if ip.Type == "ExternalIP" && ip.Address != "" {
externalIP = ip.Address
break
} else if ip.Type == "InternalIP" && ip.Address != "" {
internalIP = ip.Address
}
}
if externalIP != "" {
return externalIP
}
return internalIP
return ingress, nil
}
func ingressRules(cluster *v1alpha1.Cluster, addresses []string) []networkingv1.IngressRule {
ingressRules := []networkingv1.IngressRule{}
pathTypePrefix := networkingv1.PathTypePrefix
for _, address := range addresses {
rule := networkingv1.IngressRule{
Host: cluster.Name + "." + address + wildcardDNS,
@@ -82,7 +56,7 @@ func ingressRules(cluster *v1alpha1.Cluster, addresses []string) []networkingv1.
Paths: []networkingv1.HTTPIngressPath{
{
Path: "/",
PathType: &pathType,
PathType: &pathTypePrefix,
Backend: networkingv1.IngressBackend{
Service: &networkingv1.IngressServiceBackend{
Name: "k3k-server-service",
@@ -100,3 +74,17 @@ func ingressRules(cluster *v1alpha1.Cluster, addresses []string) []networkingv1.
}
return ingressRules
}
// configureIngressOptions will configure the ingress object by
// adding tls passthrough capabilities and TLS needed annotations
// it depends on the ingressclassname to configure each ingress
// TODO: add treafik support through ingresstcproutes
func configureIngressOptions(ingress *networkingv1.Ingress, ingressClassName string) {
// initial support for nginx ingress via annotations
if ingressClassName == "nginx" {
ingress.Annotations = make(map[string]string)
ingress.Annotations[nginxSSLPassthroughAnnotation] = "true"
ingress.Annotations[nginxSSLRedirectAnnotation] = "true"
ingress.Annotations[nginxBackendProtocolAnnotation] = "HTTPS"
}
}
pkg/controller/cluster/server/kubeconfig.go
+1 -1
View File
@@ -40,7 +40,7 @@ type content struct {
Content string
}
// GenerateNewKubeConfig generates the kubeconfig for the server:
// GenerateNewKubeConfig generates the kubeconfig for the cluster:
// 1- use the server token to get the bootstrap data from k3s
// 2- generate client admin cert/key
// 3- use the ca cert from the bootstrap data & admin cert/key to write a new kubeconfig
pkg/controller/cluster/server/server.go
+6 -3
View File
@@ -2,6 +2,7 @@ package server
import (
"strconv"
"strings"
"github.com/galal-hussein/k3k/pkg/apis/k3k.io/v1alpha1"
"github.com/galal-hussein/k3k/pkg/controller/util"
@@ -49,13 +50,13 @@ func Server(cluster *v1alpha1.Cluster, init bool) *apps.Deployment {
"init": strconv.FormatBool(init),
},
},
Spec: serverPodSpec(image, name),
Spec: serverPodSpec(image, name, cluster.Spec.ServerArgs),
},
},
}
}
func serverPodSpec(image, name string) v1.PodSpec {
func serverPodSpec(image, name string, args []string) v1.PodSpec {
privileged := true
return v1.PodSpec{
Volumes: []v1.Volume{
@@ -122,7 +123,9 @@ func serverPodSpec(image, name string) v1.PodSpec {
},
Args: []string{
"-c",
"/bin/k3s server --config /opt/rancher/k3s/config.yaml && true",
"/bin/k3s server --config /opt/rancher/k3s/config.yaml " +
strings.Join(args, " ") +
" && true",
},
VolumeMounts: []v1.VolumeMount{
{
pkg/controller/util/util.go
+37
View File
@@ -1,8 +1,12 @@
package util
import (
"context"
"github.com/galal-hussein/k3k/pkg/apis/k3k.io/v1alpha1"
v1 "k8s.io/api/core/v1"
"k8s.io/klog"
"sigs.k8s.io/controller-runtime/pkg/client"
)
const (
@@ -22,3 +26,36 @@ func WrapErr(errString string, err error) error {
klog.Errorf("%s: %v", errString, err)
return err
}
// return all the nodes external addresses, if not found then return internal addresses
func Addresses(ctx context.Context, client client.Client) ([]string, error) {
addresses := []string{}
nodeList := v1.NodeList{}
if err := client.List(ctx, &nodeList); err != nil {
return nil, err
}
for _, node := range nodeList.Items {
addresses = append(addresses, getNodeAddress(&node))
}
return addresses, nil
}
func getNodeAddress(node *v1.Node) string {
externalIP := ""
internalIP := ""
for _, ip := range node.Status.Addresses {
if ip.Type == "ExternalIP" && ip.Address != "" {
externalIP = ip.Address
break
} else if ip.Type == "InternalIP" && ip.Address != "" {
internalIP = ip.Address
}
}
if externalIP != "" {
return externalIP
}
return internalIP
}