diff --git a/README.md b/README.md index df32902c..e9f68b9b 100644 --- a/README.md +++ b/README.md @@ -8,11 +8,47 @@ An example on creating a k3k cluster on an RKE2 host using k3kcli [![asciicast](https://asciinema.org/a/eYlc3dsL2pfP2B50i3Ea8MJJp.svg)](https://asciinema.org/a/eYlc3dsL2pfP2B50i3Ea8MJJp) -## Usage +## Architecture K3K consists of a controller and a cli tool, the controller can be deployed via a helm chart and the cli can be downloaded from the releases page. -### Deploy Controller +### Controller + +The K3K controller will watch a CRD called `clusters.k3k.io`. Once found, the controller will create a separate namespace and it will create a K3S cluster as specified in the spec of the object. + +Each server and agent is created as a separate pod that runs in the new namespace. + +### CLI + +The CLI provides a quick and easy way to create K3K clusters using simple flags, and automatically exposes the K3K clusters so it's accessible via a kubeconfig. + +## Features + +### Isolation + +Each cluster runs in a sperate namespace that can be isolated via netowrk policies and RBAC rules, clusters also run in a sperate network namespace with flannel as the backend CNI. Finally, each cluster has a separate datastore which can be persisted. + +In addition, k3k offers a persistence feature that can help users to persist their datatstore, using dynamic storage class volumes. + +### Portability and Customization + +The "Cluster" object is considered the template of the cluster that you can re-use to spin up multiple clusters in a matter of seconds. + +K3K clusters use K3S internally and leverage all options that can be passed to K3S. Each cluster is exposed to the host cluster via NodePort, LoadBalancers, and Ingresses. + + +| | Separate Namespace (for each tenant) | K3K | vcluster | Separate Cluster (for each tenant) | +|-----------------------|---------------------------------------|------------------------------|-----------------|------------------------------------| +| Isolation | Very weak | Very strong | strong | Very strong | +| Access for tenants | Very restricted | Built-in k8s RBAC / Rancher | Vclustser admin | Cluster admin | +| Cost | Very cheap | Very cheap | cheap | expensive | +| Overhead | Very low | Very low | Very low | Very high | +| Networking | Shared | Separate | shared | separate | +| Cluster Configuration | | Very easy | Very hard | | + +## Usage + +### Deploy K3K Controller [Helm](https://helm.sh) must be installed to use the charts. Please refer to Helm's [documentation](https://helm.sh/docs) to get started. diff --git a/charts/k3k/values.yaml b/charts/k3k/values.yaml index 83c7b5e9..1732381d 100644 --- a/charts/k3k/values.yaml +++ b/charts/k3k/values.yaml @@ -2,10 +2,10 @@ replicaCount: 1 namespace: k3k-system image: - repository: briandowns/k3k + repository: rancher/k3k pullPolicy: Always # Overrides the image tag whose default is the chart appVersion. - tag: "dev" + tag: "v0.0.0-alpha6" imagePullSecrets: [] nameOverride: "" diff --git a/pkg/controller/cluster/server/server.go b/pkg/controller/cluster/server/server.go index f9fdb96c..4e780c8d 100644 --- a/pkg/controller/cluster/server/server.go +++ b/pkg/controller/cluster/server/server.go @@ -2,7 +2,6 @@ package server import ( "context" - "fmt" "strconv" "github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1" @@ -12,6 +11,7 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -20,6 +20,8 @@ const ( serverName = "k3k-" k3kSystemNamespace = serverName + "system" initServerName = serverName + "init-server" + initContainerName = serverName + "server-check" + initContainerImage = "alpine/curl" ) // Server @@ -107,13 +109,11 @@ func (s *Server) Deploy(ctx context.Context, init bool) (*apps.Deployment, error volumeMounts = append(volumeMounts, volumeMount) } - podSpec := s.podSpec(ctx, image, name, false) + podSpec := s.podSpec(ctx, image, name, false, init) podSpec.Volumes = append(podSpec.Volumes, volumes...) podSpec.Containers[0].VolumeMounts = append(podSpec.Containers[0].VolumeMounts, volumeMounts...) - fmt.Printf("XXX - Pod Spec\n %#v\n", podSpec) - return &apps.Deployment{ TypeMeta: metav1.TypeMeta{ Kind: "Deployment", @@ -146,7 +146,7 @@ func (s *Server) Deploy(ctx context.Context, init bool) (*apps.Deployment, error }, nil } -func (s *Server) podSpec(ctx context.Context, image, name string, statefulSet bool) v1.PodSpec { +func (s *Server) podSpec(ctx context.Context, image, name string, statefulSet, init bool) v1.PodSpec { args := append([]string{"server", "--config", "/opt/rancher/k3s/config.yaml"}, s.cluster.Spec.ServerArgs...) podSpec := v1.PodSpec{ @@ -258,6 +258,33 @@ func (s *Server) podSpec(ctx context.Context, image, name string, statefulSet bo }, ) } + + // Adding readiness probes to deployment + podSpec.Containers[0].ReadinessProbe = &v1.Probe{ + InitialDelaySeconds: 60, + FailureThreshold: 5, + TimeoutSeconds: 10, + ProbeHandler: v1.ProbeHandler{ + TCPSocket: &v1.TCPSocketAction{ + Port: intstr.FromInt(6443), + Host: "127.0.0.1", + }, + }, + } + + if !init { + podSpec.InitContainers = []v1.Container{ + { + Name: initContainerName, + Image: initContainerImage, + Command: []string{ + "sh", + "-c", + "until curl -qk https://k3k-server-service.$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace).svc.cluster.local:6443/v1-k3s/readyz; do echo waiting for init server to be up; sleep 2; done", + }, + }, + } + } return podSpec } @@ -342,7 +369,7 @@ func (s *Server) StatefulServer(ctx context.Context, cluster *v1alpha1.Cluster, "init": strconv.FormatBool(init), }, }, - Spec: s.podSpec(ctx, image, name, true), + Spec: s.podSpec(ctx, image, name, true, init), }, }, }