chore(helm): releasing v0.3.1

Co-authored-by: jds <jds9090@kinx.net>

.github/workflows/docker-ci.yml

@@ -12,6 +12,8 @@ jobs:
 
       - name: Checkout
         uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
 
       - name: Generate build-args
         id: build-args
@@ -85,7 +87,13 @@ jobs:
           platforms: linux/amd64,linux/arm64,linux/arm
           push: true
           tags: ${{ steps.meta.outputs.tags }}
-          build-args:
+          build-args: |
+            GIT_LAST_TAG=${{ env.GIT_LAST_TAG }}
+            GIT_HEAD_COMMIT=${{ env.GIT_HEAD_COMMIT }}
+            GIT_TAG_COMMIT=${{ env.GIT_TAG_COMMIT }}
+            GIT_MODIFIED=${{ env.GIT_MODIFIED }}
+            GIT_REPO=${{ env.GIT_REPO }}
+            BUILD_DATE=${{ env.BUILD_DATE }}
 
       - name: Image digest
         run: echo ${{ steps.build-release.outputs.digest }}

Makefile

@@ -3,7 +3,7 @@
 # To re-generate a bundle for another specific version without changing the standard setup, you can:
 # - use the VERSION as arg of the bundle target (e.g make bundle VERSION=0.0.2)
 # - use environment variables to overwrite this value (e.g export VERSION=0.0.2)
-VERSION ?= 0.1.1
+VERSION ?= 0.3.1
 
 # CHANNELS define the bundle channels used in the bundle.
 # Add a new line here if you would like to change its default config. (E.g CHANNELS = "candidate,fast,stable")
@@ -85,7 +85,7 @@ kind: ## Download kind locally if necessary.
 
 CONTROLLER_GEN = $(shell pwd)/bin/controller-gen
 controller-gen: ## Download controller-gen locally if necessary.
-	$(call go-install-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.9.2)
+	$(call go-install-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.11.4)
 
 GOLANGCI_LINT = $(shell pwd)/bin/golangci-lint
 golangci-lint: ## Download golangci-lint locally if necessary.

PROJECT

@@ -16,10 +16,6 @@ resources:
   kind: TenantControlPlane
   path: github.com/clastix/kamaji/api/v1alpha1
   version: v1alpha1
-  webhooks:
-    defaulting: true
-    validation: true
-    webhookVersion: v1
 - api:
     crdVersion: v1
   domain: clastix.io
@@ -27,8 +23,4 @@ resources:
   kind: DataStore
   path: github.com/clastix/kamaji/api/v1alpha1
   version: v1alpha1
-  webhooks:
-    defaulting: true
-    validation: true
-    webhookVersion: v1
 version: "3"

README.md

@@ -5,61 +5,64 @@
   <img src="https://img.shields.io/github/go-mod/go-version/clastix/kamaji"/>
   <a href="https://github.com/clastix/kamaji/releases">
     <img src="https://img.shields.io/github/v/release/clastix/kamaji"/>
+    <img src="https://goreportcard.com/badge/github.com/clastix/kamaji">
   </a>
 </p>
 
-**Kamaji** deploys and operates **Kubernetes** at scale with a fraction of the operational burden.
+![Logo](assets/logo-black.png#gh-light-mode-only)
+![Logo](assets/logo-white.png#gh-dark-mode-only)
 
-<p align="center" style="padding: 6px 6px">
-  <img src="assets/kamaji-logo.png" />
-</p>
+**Kamaji** deploys and operates **Kubernetes Control Plane** at scale with a fraction of the operational burden. Kamaji is special because the Control Plane components are running in a single pod instead of dedicated machines. This solution makes running multiple Control Planes cheaper and easier to deploy and operate.
 
-## Why we are building it?
-Global hyper-scalers are leading the Managed Kubernetes space, while other cloud providers, as well as large corporations, are struggling to offer the same experience to their DevOps teams because of the lack of the right tools. Also, current Kubernetes solutions are mainly designed with an enterprise-first approach and they are too costly when deployed at scale.
-
-**Kamaji** aims to solve these pains by leveraging multi-tenancy and simplifying how to run multiple control planes on the same infrastructure with a fraction of the operational burden.
-
-## How it works
-Kamaji turns any Kubernetes cluster into an _“admin cluster”_ to orchestrate other Kubernetes clusters called _“tenant clusters”_. What makes Kamaji special is that Control Planes of _“tenant clusters”_ are just regular pods running in the _“admin cluster”_ instead of dedicated Virtual Machines. This solution makes running control planes at scale cheaper and easier to deploy and operate.
-
-![Architecture](docs/content/images/kamaji-light.png#gh-light-mode-only)
-![Architecture](docs/content/images/kamaji-dark.png#gh-dark-mode-only)
-
-## Getting started
-
-Please refer to the [Getting Started guide](https://kamaji.clastix.io/getting-started/) to deploy a minimal setup of Kamaji on KinD.
+<img src="docs/content/images/architecture.png"  width="600">
 
 ## Features
 
 - **Self Service Kubernetes:** leave users the freedom to self-provision their Kubernetes clusters according to the assigned boundaries.
-- **Multi-cluster Management:** centrally manage multiple tenant clusters from a single admin cluster. Happy SREs. 
-- **Cheaper Control Planes:** place multiple tenant control planes on a single node, instead of having three nodes for a single control plane.
-- **Stronger Multi-Tenancy:** leave tenants to access the control plane with admin permissions while keeping the tenant isolated at the infrastructure level.
+- **Multi-cluster Management:** centrally manage multiple clusters from a single admin cluster. Happy SREs. 
+- **Cheaper Control Planes:** place multiple control planes on a single node, instead of having three nodes for a single control plane.
+- **Stronger Multi-Tenancy:** leave users to access the control plane with admin permissions while keeping them isolated at the infrastructure level.
 - **Kubernetes Inception:** use Kubernetes to manage Kubernetes by re-using all the Kubernetes goodies you already know and love.
-- **Full APIs compliant:** tenant clusters are fully CNCF compliant built with upstream Kubernetes binaries. A user does not see differences between a Kamaji provisioned cluster and a dedicated cluster.
+- **Full APIs compliant:** all clusters are CNCF compliant built with upstream Kubernetes binaries
 
 ## Roadmap
 
-- [ ] Benchmarking and stress-test
-- [x] Support for dynamic address allocation on native Load Balancer
+- [x] Dynamic address on Load Balancer
 - [x] Zero Downtime Tenant Control Plane upgrade
-- [x] `konnectivity` integration
-- [ ] Provisioning of Tenant Control Plane through Cluster APIs
+- [x] Join worker nodes from anywhere
+- [x] Alternative datastore MySQL and PostgreSQL
+- [x] Pool of multiple datastores
+- [x] Seamless migration between datastores
+- [ ] Automatic assignment to a datastore
+- [ ] Autoscaling of Tenant Control Plane
+- [x] Provisioning through Cluster APIs
 - [ ] Terraform provider
 - [ ] Custom Prometheus metrics for monitoring and alerting
-- [x] `kine` integration for MySQL as datastore
-- [x] `kine` integration for PostgreSQL as datastore
-- [x] Pool of multiple datastores
-- [x] Seamless migration between datastore with the same driver
-- [ ] Automatic assigning of Tenant Control Plane to a datastore
-- [ ] Autoscaling of Tenant Control Plane pods
 
 
 ## Documentation
 Please, check the project's [documentation](https://kamaji.clastix.io/) for getting started with Kamaji.
 
 ## Contributions
-Kamaji is Open Source with Apache 2 license and any contribution is welcome.
+Kamaji is Open Source with Apache 2 license and any contribution is welcome. Open an issue or suggest an enhancement on the GitHub [project's page](https://github.com/clastix/kamaji). Join the [Kubernetes Slack Workspace](https://slack.k8s.io/) and the [`#kamaji`](https://kubernetes.slack.com/archives/C03GLTTMWNN) channel to meet end-users and contributors.
 
-## Community
-Join the [Kubernetes Slack Workspace](https://slack.k8s.io/) and the [`#kamaji`](https://kubernetes.slack.com/archives/C03GLTTMWNN) channel to meet end-users and contributors.
+## FAQs
+Q. What does Kamaji mean?
+
+A. Kamaji is named as the character _Kamaji_ from the Japanese movie [_Spirited Away_](https://en.wikipedia.org/wiki/Spirited_Away).
+
+Q. Is Kamaji another Kubernetes distribution?
+
+A. No, Kamaji is a Kubernetes Operator you can install on top of any Kubernetes cluster to provide hundreds or thousands of managed Kubernetes clusters as a service. We tested Kamaji on vanilla Kubernetes 1.22+, KinD, and Azure AKS. We expect it to work smoothly on other Kubernetes distributions. The tenant clusters made with Kamaji are conformant CNCF Kubernetes clusters as we leverage [`kubeadm`](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/).
+
+Q. Is it safe to run Kubernetes control plane components in a pod instead of dedicated virtual machines?
+
+A. Yes, the tenant control plane components are packaged in the same way they are running in bare metal or virtual nodes. We leverage the `kubeadm` code to set up the control plane components as they were running on their own server. The unchanged images of upstream `kube-apiserver`, `kube-scheduler`, and `kube-controller-manager` are used.
+
+Q. You already provide a Kubernetes multi-tenancy solution with [Capsule](https://capsule.clastix.io). Why does Kamaji matter?
+
+A. A multi-tenancy solution, like Capsule shares the Kubernetes control plane among all tenants keeping tenant namespaces isolated by policies. While the solution is the right choice by balancing between features and ease of usage, there are cases where a tenant user requires access to the control plane, for example, when a tenant requires to manage CRDs on his own. With Kamaji, you can provide cluster admin permissions to the tenant.
+
+Q. Well you convinced me, how to get a try?
+
+A. It is possible to get started with Kamaji on a laptop with [KinD](getting-started.md) installed.

api/v1alpha1/datastore_secret_webhook.go

@@ -1,57 +0,0 @@
-// Copyright 2022 Clastix Labs
-// SPDX-License-Identifier: Apache-2.0
-
-package v1alpha1
-
-import (
-	"context"
-	"fmt"
-	"strings"
-
-	"github.com/go-logr/logr"
-	corev1 "k8s.io/api/core/v1"
-	"k8s.io/apimachinery/pkg/fields"
-	"k8s.io/apimachinery/pkg/runtime"
-	"sigs.k8s.io/controller-runtime/pkg/client"
-)
-
-//+kubebuilder:webhook:path=/validate--v1-secret,mutating=false,failurePolicy=ignore,sideEffects=None,groups="",resources=secrets,verbs=delete,versions=v1,name=vdatastoresecrets.kb.io,admissionReviewVersions=v1
-
-type dataStoreSecretValidator struct {
-	log    logr.Logger
-	client client.Client
-}
-
-func (d *dataStoreSecretValidator) ValidateCreate(context.Context, runtime.Object) error {
-	return nil
-}
-
-func (d *dataStoreSecretValidator) ValidateUpdate(context.Context, runtime.Object, runtime.Object) error {
-	return nil
-}
-
-func (d *dataStoreSecretValidator) ValidateDelete(ctx context.Context, obj runtime.Object) error {
-	secret := obj.(*corev1.Secret) //nolint:forcetypeassert
-
-	dsList := &DataStoreList{}
-
-	if err := d.client.List(ctx, dsList, client.MatchingFieldsSelector{Selector: fields.OneTermEqualSelector(DatastoreUsedSecretNamespacedNameKey, fmt.Sprintf("%s/%s", secret.GetNamespace(), secret.GetName()))}); err != nil {
-		return err
-	}
-
-	if len(dsList.Items) > 0 {
-		var res []string
-
-		for _, ds := range dsList.Items {
-			res = append(res, ds.GetName())
-		}
-
-		return fmt.Errorf("the Secret is used by the following kamajiv1alpha1.DataStores and cannot be deleted (%s)", strings.Join(res, ", "))
-	}
-
-	return nil
-}
-
-func (d *dataStoreSecretValidator) Default(context.Context, runtime.Object) error {
-	return nil
-}

api/v1alpha1/datastore_webhook.go

@@ -1,185 +0,0 @@
-// Copyright 2022 Clastix Labs
-// SPDX-License-Identifier: Apache-2.0
-
-package v1alpha1
-
-import (
-	"context"
-	"fmt"
-
-	"github.com/go-logr/logr"
-	corev1 "k8s.io/api/core/v1"
-	"k8s.io/apimachinery/pkg/api/errors"
-	"k8s.io/apimachinery/pkg/fields"
-	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/apimachinery/pkg/types"
-	ctrl "sigs.k8s.io/controller-runtime"
-	"sigs.k8s.io/controller-runtime/pkg/client"
-)
-
-//+kubebuilder:webhook:path=/mutate-kamaji-clastix-io-v1alpha1-datastore,mutating=true,failurePolicy=fail,sideEffects=None,groups=kamaji.clastix.io,resources=datastores,verbs=create;update,versions=v1alpha1,name=mdatastore.kb.io,admissionReviewVersions=v1
-//+kubebuilder:webhook:path=/validate-kamaji-clastix-io-v1alpha1-datastore,mutating=false,failurePolicy=fail,sideEffects=None,groups=kamaji.clastix.io,resources=datastores,verbs=create;update;delete,versions=v1alpha1,name=vdatastore.kb.io,admissionReviewVersions=v1
-
-func (in *DataStore) SetupWebhookWithManager(mgr ctrl.Manager) error {
-	secretValidator := &dataStoreSecretValidator{
-		log:    mgr.GetLogger().WithName("datastore-secret-webhook"),
-		client: mgr.GetClient(),
-	}
-
-	if err := ctrl.NewWebhookManagedBy(mgr).For(&corev1.Secret{}).WithValidator(secretValidator).Complete(); err != nil {
-		return err
-	}
-
-	dsValidator := &dataStoreValidator{
-		log:    mgr.GetLogger().WithName("datastore-webhook"),
-		client: mgr.GetClient(),
-	}
-
-	return ctrl.NewWebhookManagedBy(mgr).
-		For(in).
-		WithValidator(dsValidator).
-		WithDefaulter(dsValidator).
-		Complete()
-}
-
-type dataStoreValidator struct {
-	log    logr.Logger
-	client client.Client
-}
-
-func (d *dataStoreValidator) ValidateCreate(ctx context.Context, obj runtime.Object) error {
-	ds, ok := obj.(*DataStore)
-	if !ok {
-		return fmt.Errorf("expected *kamajiv1alpha1.DataStore")
-	}
-
-	if err := d.validate(ctx, ds); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (d *dataStoreValidator) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) error {
-	old, ok := oldObj.(*DataStore)
-	if !ok {
-		return fmt.Errorf("expected *kamajiv1alpha1.DataStore")
-	}
-
-	ds, ok := newObj.(*DataStore)
-	if !ok {
-		return fmt.Errorf("expected *kamajiv1alpha1.DataStore")
-	}
-
-	d.log.Info("validate update", "name", ds.GetName())
-
-	if ds.Spec.Driver != old.Spec.Driver {
-		return fmt.Errorf("driver of a DataStore cannot be changed")
-	}
-
-	if err := d.validate(ctx, ds); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (d *dataStoreValidator) ValidateDelete(ctx context.Context, obj runtime.Object) error {
-	ds, ok := obj.(*DataStore)
-	if !ok {
-		return fmt.Errorf("expected *kamajiv1alpha1.DataStore")
-	}
-
-	tcpList := &TenantControlPlaneList{}
-
-	if err := d.client.List(ctx, tcpList, client.MatchingFieldsSelector{Selector: fields.OneTermEqualSelector(TenantControlPlaneUsedDataStoreKey, ds.GetName())}); err != nil {
-		return err
-	}
-
-	if len(tcpList.Items) > 0 {
-		return fmt.Errorf("the DataStore is used by multiple TenantControlPlanes and cannot be removed")
-	}
-
-	return nil
-}
-
-func (d *dataStoreValidator) Default(context.Context, runtime.Object) error {
-	return nil
-}
-
-func (d *dataStoreValidator) validate(ctx context.Context, ds *DataStore) error {
-	if ds.Spec.BasicAuth != nil {
-		if err := d.validateBasicAuth(ctx, ds); err != nil {
-			return err
-		}
-	}
-
-	if err := d.validateTLSConfig(ctx, ds); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (d *dataStoreValidator) validateBasicAuth(ctx context.Context, ds *DataStore) error {
-	if err := d.validateContentReference(ctx, ds.Spec.BasicAuth.Password); err != nil {
-		return fmt.Errorf("basic-auth password is not valid, %w", err)
-	}
-
-	if err := d.validateContentReference(ctx, ds.Spec.BasicAuth.Username); err != nil {
-		return fmt.Errorf("basic-auth username is not valid, %w", err)
-	}
-
-	return nil
-}
-
-func (d *dataStoreValidator) validateTLSConfig(ctx context.Context, ds *DataStore) error {
-	if err := d.validateContentReference(ctx, ds.Spec.TLSConfig.CertificateAuthority.Certificate); err != nil {
-		return fmt.Errorf("CA certificate is not valid, %w", err)
-	}
-
-	if ds.Spec.Driver == EtcdDriver {
-		if ds.Spec.TLSConfig.CertificateAuthority.PrivateKey == nil {
-			return fmt.Errorf("CA private key is required when using the etcd driver")
-		}
-	}
-
-	if ds.Spec.TLSConfig.CertificateAuthority.PrivateKey != nil {
-		if err := d.validateContentReference(ctx, *ds.Spec.TLSConfig.CertificateAuthority.PrivateKey); err != nil {
-			return fmt.Errorf("CA private key is not valid, %w", err)
-		}
-	}
-
-	if err := d.validateContentReference(ctx, ds.Spec.TLSConfig.ClientCertificate.Certificate); err != nil {
-		return fmt.Errorf("client certificate is not valid, %w", err)
-	}
-
-	if err := d.validateContentReference(ctx, ds.Spec.TLSConfig.ClientCertificate.PrivateKey); err != nil {
-		return fmt.Errorf("client private key is not valid, %w", err)
-	}
-
-	return nil
-}
-
-func (d *dataStoreValidator) validateContentReference(ctx context.Context, ref ContentRef) error {
-	switch {
-	case len(ref.Content) > 0:
-		return nil
-	case ref.SecretRef == nil:
-		return fmt.Errorf("the Secret reference is mandatory when bare content is not specified")
-	case len(ref.SecretRef.SecretReference.Name) == 0:
-		return fmt.Errorf("the Secret reference name is mandatory")
-	case len(ref.SecretRef.SecretReference.Namespace) == 0:
-		return fmt.Errorf("the Secret reference namespace is mandatory")
-	}
-
-	if err := d.client.Get(ctx, types.NamespacedName{Name: ref.SecretRef.SecretReference.Name, Namespace: ref.SecretRef.SecretReference.Namespace}, &corev1.Secret{}); err != nil {
-		if errors.IsNotFound(err) {
-			return fmt.Errorf("secret %s/%s is not found", ref.SecretRef.SecretReference.Namespace, ref.SecretRef.SecretReference.Name)
-		}
-
-		return err
-	}
-
-	return nil
-}

api/v1alpha1/tenantcontrolplane_registrysettings.go

@@ -0,0 +1,18 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package v1alpha1
+
+type RegistrySettings struct {
+	// +kubebuilder:default="registry.k8s.io"
+	Registry string `json:"registry,omitempty"`
+	// The tag to append to all the Control Plane container images.
+	// Optional.
+	TagSuffix string `json:"tagSuffix,omitempty"`
+	// +kubebuilder:default="kube-apiserver"
+	APIServerImage string `json:"apiServerImage,omitempty"`
+	// +kubebuilder:default="kube-controller-manager"
+	ControllerManagerImage string `json:"controllerManagerImage,omitempty"`
+	// +kubebuilder:default="kube-scheduler"
+	SchedulerImage string `json:"schedulerImage,omitempty"`
+}

api/v1alpha1/tenantcontrolplane_registrysettings_funcs.go

@@ -0,0 +1,30 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package v1alpha1
+
+import (
+	"fmt"
+)
+
+func (r *RegistrySettings) buildContainerImage(name, tag string) string {
+	image := fmt.Sprintf("%s/%s:%s", r.Registry, name, tag)
+
+	if len(r.TagSuffix) > 0 {
+		image += r.TagSuffix
+	}
+
+	return image
+}
+
+func (r *RegistrySettings) KubeAPIServerImage(version string) string {
+	return r.buildContainerImage(r.APIServerImage, version)
+}
+
+func (r *RegistrySettings) KubeSchedulerImage(version string) string {
+	return r.buildContainerImage(r.SchedulerImage, version)
+}
+
+func (r *RegistrySettings) KubeControllerManagerImage(version string) string {
+	return r.buildContainerImage(r.ControllerManagerImage, version)
+}

api/v1alpha1/tenantcontrolplane_status.go

@@ -183,11 +183,12 @@ type KubernetesStatus struct {
 	Ingress    *KubernetesIngressStatus   `json:"ingress,omitempty"`
 }
 
-// +kubebuilder:validation:Enum=Provisioning;Upgrading;Migrating;Ready;NotReady
+// +kubebuilder:validation:Enum=Provisioning;CertificateAuthorityRotating;Upgrading;Migrating;Ready;NotReady
 type KubernetesVersionStatus string
 
 var (
 	VersionProvisioning KubernetesVersionStatus = "Provisioning"
+	VersionCARotating   KubernetesVersionStatus = "CertificateAuthorityRotating"
 	VersionUpgrading    KubernetesVersionStatus = "Upgrading"
 	VersionMigrating    KubernetesVersionStatus = "Migrating"
 	VersionReady        KubernetesVersionStatus = "Ready"

api/v1alpha1/tenantcontrolplane_types.go

@@ -4,6 +4,7 @@
 package v1alpha1
 
 import (
+	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
@@ -32,7 +33,23 @@ type NetworkProfileSpec struct {
 	DNSServiceIPs []string `json:"dnsServiceIPs,omitempty"`
 }
 
+// +kubebuilder:validation:Enum=Hostname;InternalIP;ExternalIP;InternalDNS;ExternalDNS
+type KubeletPreferredAddressType string
+
+const (
+	NodeHostName    KubeletPreferredAddressType = "Hostname"
+	NodeInternalIP  KubeletPreferredAddressType = "InternalIP"
+	NodeExternalIP  KubeletPreferredAddressType = "ExternalIP"
+	NodeInternalDNS KubeletPreferredAddressType = "InternalDNS"
+	NodeExternalDNS KubeletPreferredAddressType = "ExternalDNS"
+)
+
 type KubeletSpec struct {
+	// Ordered list of the preferred NodeAddressTypes to use for kubelet connections.
+	// Default to Hostname, InternalIP, ExternalIP.
+	// +kubebuilder:default={"Hostname","InternalIP","ExternalIP"}
+	// +kubebuilder:validation:MinItems=1
+	PreferredAddressTypes []KubeletPreferredAddressType `json:"preferredAddressTypes,omitempty"`
 	// CGroupFS defines the  cgroup driver for Kubelet
 	// https://kubernetes.io/docs/tasks/administer-cluster/kubeadm/configure-cgroup-driver/
 	CGroupFS CGroupDriver `json:"cgroupfs,omitempty"`
@@ -76,25 +93,20 @@ type IngressSpec struct {
 	Hostname string `json:"hostname,omitempty"`
 }
 
-// ComponentResourceRequirements describes the compute resource requirements.
-type ComponentResourceRequirements struct {
-	// Limits describes the maximum amount of compute resources allowed.
-	// More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
-	Limits corev1.ResourceList `json:"limits,omitempty" protobuf:"bytes,1,rep,name=limits,casttype=ResourceList,castkey=ResourceName"`
-	// Requests describes the minimum amount of compute resources required.
-	// If Requests is omitted for a container, it defaults to Limits if that is explicitly specified,
-	// otherwise to an implementation-defined value.
-	// More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
-	Requests corev1.ResourceList `json:"requests,omitempty" protobuf:"bytes,2,rep,name=requests,casttype=ResourceList,castkey=ResourceName"`
-}
-
 type ControlPlaneComponentsResources struct {
-	APIServer         *ComponentResourceRequirements `json:"apiServer,omitempty"`
-	ControllerManager *ComponentResourceRequirements `json:"controllerManager,omitempty"`
-	Scheduler         *ComponentResourceRequirements `json:"scheduler,omitempty"`
+	APIServer         *corev1.ResourceRequirements `json:"apiServer,omitempty"`
+	ControllerManager *corev1.ResourceRequirements `json:"controllerManager,omitempty"`
+	Scheduler         *corev1.ResourceRequirements `json:"scheduler,omitempty"`
+	// Define the kine container resources.
+	// Available only if Kamaji is running using Kine as backing storage.
+	Kine *corev1.ResourceRequirements `json:"kine,omitempty"`
 }
 
 type DeploymentSpec struct {
+	// RegistrySettings allows to override the default images for the given Tenant Control Plane instance.
+	// It could be used to point to a different container registry rather than the public one.
+	// +kubebuilder:default={registry:"registry.k8s.io",apiServerImage:"kube-apiserver",controllerManagerImage:"kube-controller-manager",schedulerImage:"kube-scheduler"}
+	RegistrySettings RegistrySettings `json:"registrySettings,omitempty"`
 	// +kubebuilder:default=2
 	Replicas int32 `json:"replicas,omitempty"`
 	// NodeSelector is a selector which must be true for the pod to fit on a node.
@@ -107,6 +119,10 @@ type DeploymentSpec struct {
 	// empty definition that uses the default runtime handler.
 	// More info: https://git.k8s.io/enhancements/keps/sig-node/585-runtime-class
 	RuntimeClassName string `json:"runtimeClassName,omitempty"`
+	// Strategy describes how to replace existing pods with new ones for the given Tenant Control Plane.
+	// Default value is set to Rolling Update, with a blue/green strategy.
+	// +kubebuilder:default={type:"RollingUpdate",rollingUpdate:{maxUnavailable:0,maxSurge:"100%"}}
+	Strategy appsv1.DeploymentStrategy `json:"strategy,omitempty"`
 	// If specified, the Tenant Control Plane pod's tolerations.
 	// More info: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/
 	Tolerations []corev1.Toleration `json:"tolerations,omitempty"`
@@ -125,6 +141,22 @@ type DeploymentSpec struct {
 	// such as kube-apiserver, controller-manager, and scheduler.
 	ExtraArgs          *ControlPlaneExtraArgs `json:"extraArgs,omitempty"`
 	AdditionalMetadata AdditionalMetadata     `json:"additionalMetadata,omitempty"`
+	// AdditionalInitContainers allows adding additional init containers to the Control Plane deployment.
+	AdditionalInitContainers []corev1.Container `json:"additionalInitContainers,omitempty"`
+	// AdditionalContainers allows adding additional containers to the Control Plane deployment.
+	AdditionalContainers []corev1.Container `json:"additionalContainers,omitempty"`
+	// AdditionalVolumes allows to add additional volumes to the Control Plane deployment.
+	AdditionalVolumes []corev1.Volume `json:"additionalVolumes,omitempty"`
+	// AdditionalVolumeMounts allows to mount an additional volume into each component of the Control Plane
+	// (kube-apiserver, controller-manager, and scheduler).
+	AdditionalVolumeMounts *AdditionalVolumeMounts `json:"additionalVolumeMounts,omitempty"`
+}
+
+// AdditionalVolumeMounts allows mounting additional volumes to the Control Plane components.
+type AdditionalVolumeMounts struct {
+	APIServer         []corev1.VolumeMount `json:"apiServer,omitempty"`
+	ControllerManager []corev1.VolumeMount `json:"controllerManager,omitempty"`
+	Scheduler         []corev1.VolumeMount `json:"scheduler,omitempty"`
 }
 
 // ControlPlaneExtraArgs allows specifying additional arguments to the Control Plane components.
@@ -169,8 +201,8 @@ type KonnectivityServerSpec struct {
 	// +kubebuilder:default=registry.k8s.io/kas-network-proxy/proxy-server
 	Image string `json:"image,omitempty"`
 	// Resources define the amount of CPU and memory to allocate to the Konnectivity server.
-	Resources *ComponentResourceRequirements `json:"resources,omitempty"`
-	ExtraArgs ExtraArgs                      `json:"extraArgs,omitempty"`
+	Resources *corev1.ResourceRequirements `json:"resources,omitempty"`
+	ExtraArgs ExtraArgs                    `json:"extraArgs,omitempty"`
 }
 
 type KonnectivityAgentSpec struct {

api/v1alpha1/tenantcontrolplane_webhook.go

@@ -1,166 +0,0 @@
-// Copyright 2022 Clastix Labs
-// SPDX-License-Identifier: Apache-2.0
-
-package v1alpha1
-
-import (
-	"context"
-	"fmt"
-	"strings"
-
-	"github.com/blang/semver"
-	"github.com/go-logr/logr"
-	"github.com/pkg/errors"
-	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/apimachinery/pkg/types"
-	ctrl "sigs.k8s.io/controller-runtime"
-	"sigs.k8s.io/controller-runtime/pkg/client"
-
-	"github.com/clastix/kamaji/internal/upgrade"
-)
-
-//+kubebuilder:webhook:path=/mutate-kamaji-clastix-io-v1alpha1-tenantcontrolplane,mutating=true,failurePolicy=fail,sideEffects=None,groups=kamaji.clastix.io,resources=tenantcontrolplanes,verbs=create;update,versions=v1alpha1,name=mtenantcontrolplane.kb.io,admissionReviewVersions=v1
-//+kubebuilder:webhook:path=/validate-kamaji-clastix-io-v1alpha1-tenantcontrolplane,mutating=false,failurePolicy=fail,sideEffects=None,groups=kamaji.clastix.io,resources=tenantcontrolplanes,verbs=create;update,versions=v1alpha1,name=vtenantcontrolplane.kb.io,admissionReviewVersions=v1
-
-func (in *TenantControlPlane) SetupWebhookWithManager(mgr ctrl.Manager, datastore string) error {
-	validator := &tenantControlPlaneValidator{
-		client:           mgr.GetClient(),
-		defaultDatastore: datastore,
-		log:              mgr.GetLogger().WithName("tenantcontrolplane-webhook"),
-	}
-
-	return ctrl.NewWebhookManagedBy(mgr).
-		For(in).
-		WithValidator(validator).
-		WithDefaulter(validator).
-		Complete()
-}
-
-type tenantControlPlaneValidator struct {
-	client           client.Client
-	defaultDatastore string
-	log              logr.Logger
-}
-
-func (t *tenantControlPlaneValidator) Default(_ context.Context, obj runtime.Object) error {
-	tcp, ok := obj.(*TenantControlPlane)
-	if !ok {
-		return fmt.Errorf("expected *kamajiv1alpha1.TenantControlPlane")
-	}
-
-	if len(tcp.Spec.DataStore) == 0 {
-		tcp.Spec.DataStore = t.defaultDatastore
-	}
-
-	return nil
-}
-
-func (t *tenantControlPlaneValidator) ValidateCreate(_ context.Context, obj runtime.Object) error {
-	tcp, ok := obj.(*TenantControlPlane)
-	if !ok {
-		return fmt.Errorf("expected *kamajiv1alpha1.TenantControlPlane")
-	}
-
-	t.log.Info("validate create", "name", tcp.Name, "namespace", tcp.Namespace)
-
-	ver, err := semver.New(t.normalizeKubernetesVersion(tcp.Spec.Kubernetes.Version))
-	if err != nil {
-		return errors.Wrap(err, "unable to parse the desired Kubernetes version")
-	}
-
-	supportedVer, supportedErr := semver.Make(t.normalizeKubernetesVersion(upgrade.KubeadmVersion))
-	if supportedErr != nil {
-		return errors.Wrap(supportedErr, "unable to parse the Kamaji supported Kubernetes version")
-	}
-
-	if ver.GT(supportedVer) {
-		return fmt.Errorf("unable to create a TenantControlPlane with a Kubernetes version greater than the supported one, actually %s", supportedVer.String())
-	}
-
-	return nil
-}
-
-func (t *tenantControlPlaneValidator) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) error {
-	old, ok := oldObj.(*TenantControlPlane)
-	if !ok {
-		return fmt.Errorf("expected *kamajiv1alpha1.TenantControlPlane")
-	}
-
-	tcp, ok := newObj.(*TenantControlPlane)
-	if !ok {
-		return fmt.Errorf("expected *kamajiv1alpha1.TenantControlPlane")
-	}
-
-	t.log.Info("validate update", "name", tcp.Name, "namespace", tcp.Namespace)
-
-	if err := t.validateVersionUpdate(old, tcp); err != nil {
-		return err
-	}
-	if err := t.validateDataStore(ctx, old, tcp); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (t *tenantControlPlaneValidator) ValidateDelete(context.Context, runtime.Object) error {
-	return nil
-}
-
-func (t *tenantControlPlaneValidator) validateVersionUpdate(oldObj, newObj *TenantControlPlane) error {
-	oldVer, oldErr := semver.Make(t.normalizeKubernetesVersion(oldObj.Spec.Kubernetes.Version))
-	if oldErr != nil {
-		return errors.Wrap(oldErr, "unable to parse the previous Kubernetes version")
-	}
-
-	newVer, newErr := semver.New(t.normalizeKubernetesVersion(newObj.Spec.Kubernetes.Version))
-	if newErr != nil {
-		return errors.Wrap(newErr, "unable to parse the desired Kubernetes version")
-	}
-
-	supportedVer, supportedErr := semver.Make(t.normalizeKubernetesVersion(upgrade.KubeadmVersion))
-	if supportedErr != nil {
-		return errors.Wrap(supportedErr, "unable to parse the Kamaji supported Kubernetes version")
-	}
-
-	switch {
-	case newVer.GT(supportedVer):
-		return fmt.Errorf("unable to upgrade to a version greater than the supported one, actually %s", supportedVer.String())
-	case newVer.LT(oldVer):
-		return fmt.Errorf("unable to downgrade a TenantControlPlane from %s to %s", oldVer.String(), newVer.String())
-	case newVer.Minor-oldVer.Minor > 1:
-		return fmt.Errorf("unable to upgrade to a minor version in a non-sequential mode")
-	}
-
-	return nil
-}
-
-func (t *tenantControlPlaneValidator) validateDataStore(ctx context.Context, oldObj, tcp *TenantControlPlane) error {
-	if oldObj.Spec.DataStore == tcp.Spec.DataStore {
-		return nil
-	}
-
-	previousDatastore, desiredDatastore := &DataStore{}, &DataStore{}
-
-	if err := t.client.Get(ctx, types.NamespacedName{Name: oldObj.Spec.DataStore}, previousDatastore); err != nil {
-		return fmt.Errorf("unable to retrieve old DataStore for validation: %w", err)
-	}
-
-	if err := t.client.Get(ctx, types.NamespacedName{Name: tcp.Spec.DataStore}, desiredDatastore); err != nil {
-		return fmt.Errorf("unable to retrieve old DataStore for validation: %w", err)
-	}
-
-	if previousDatastore.Spec.Driver != desiredDatastore.Spec.Driver {
-		return fmt.Errorf("migration between different Datastore drivers is not supported")
-	}
-
-	return nil
-}
-
-func (t *tenantControlPlaneValidator) normalizeKubernetesVersion(input string) string {
-	if strings.HasPrefix(input, "v") {
-		return strings.Replace(input, "v", "", 1)
-	}
-
-	return input
-}

api/v1alpha1/webhook_suite_test.go

@@ -1,123 +0,0 @@
-// Copyright 2022 Clastix Labs
-// SPDX-License-Identifier: Apache-2.0
-
-package v1alpha1
-
-import (
-	"context"
-	"crypto/tls"
-	"fmt"
-	"net"
-	"path/filepath"
-	"testing"
-	"time"
-
-	. "github.com/onsi/ginkgo/v2"
-	. "github.com/onsi/gomega"
-	admissionv1beta1 "k8s.io/api/admission/v1beta1"
-	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/client-go/rest"
-	ctrl "sigs.k8s.io/controller-runtime"
-	"sigs.k8s.io/controller-runtime/pkg/client"
-	"sigs.k8s.io/controller-runtime/pkg/envtest"
-	logf "sigs.k8s.io/controller-runtime/pkg/log"
-	"sigs.k8s.io/controller-runtime/pkg/log/zap"
-	//+kubebuilder:scaffold:imports
-)
-
-// These tests use Ginkgo (BDD-style Go testing framework). Refer to
-// http://onsi.github.io/ginkgo/ to learn more about Ginkgo.
-
-var (
-	cfg       *rest.Config
-	k8sClient client.Client
-	testEnv   *envtest.Environment
-	ctx       context.Context
-	cancel    context.CancelFunc
-)
-
-func TestAPIs(t *testing.T) {
-	RegisterFailHandler(Fail)
-
-	RunSpecs(t, "Webhook Suite")
-}
-
-var _ = BeforeSuite(func() {
-	logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true)))
-
-	ctx, cancel = context.WithCancel(context.TODO())
-
-	By("bootstrapping test environment")
-	testEnv = &envtest.Environment{
-		CRDDirectoryPaths:     []string{filepath.Join("..", "..", "config", "crd", "bases")},
-		ErrorIfCRDPathMissing: false,
-		WebhookInstallOptions: envtest.WebhookInstallOptions{
-			Paths: []string{filepath.Join("..", "..", "config", "webhook")},
-		},
-	}
-
-	var err error
-	// cfg is defined in this file globally.
-	cfg, err = testEnv.Start()
-	Expect(err).NotTo(HaveOccurred())
-	Expect(cfg).NotTo(BeNil())
-
-	scheme := runtime.NewScheme()
-	err = AddToScheme(scheme)
-	Expect(err).NotTo(HaveOccurred())
-
-	err = admissionv1beta1.AddToScheme(scheme)
-	Expect(err).NotTo(HaveOccurred())
-
-	//+kubebuilder:scaffold:scheme
-
-	k8sClient, err = client.New(cfg, client.Options{Scheme: scheme})
-	Expect(err).NotTo(HaveOccurred())
-	Expect(k8sClient).NotTo(BeNil())
-
-	// start webhook server using Manager
-	webhookInstallOptions := &testEnv.WebhookInstallOptions
-	mgr, err := ctrl.NewManager(cfg, ctrl.Options{
-		Scheme:             scheme,
-		Host:               webhookInstallOptions.LocalServingHost,
-		Port:               webhookInstallOptions.LocalServingPort,
-		CertDir:            webhookInstallOptions.LocalServingCertDir,
-		LeaderElection:     false,
-		MetricsBindAddress: "0",
-	})
-	Expect(err).NotTo(HaveOccurred())
-
-	err = (&TenantControlPlane{}).SetupWebhookWithManager(mgr, "")
-	Expect(err).NotTo(HaveOccurred())
-
-	err = (&DataStore{}).SetupWebhookWithManager(mgr)
-	Expect(err).NotTo(HaveOccurred())
-
-	//+kubebuilder:scaffold:webhook
-
-	go func() {
-		defer GinkgoRecover()
-		err = mgr.Start(ctx)
-		Expect(err).NotTo(HaveOccurred())
-	}()
-
-	// wait for the webhook server to get ready
-	dialer := &net.Dialer{Timeout: time.Second}
-	addrPort := fmt.Sprintf("%s:%d", webhookInstallOptions.LocalServingHost, webhookInstallOptions.LocalServingPort)
-	Eventually(func() error {
-		conn, err := tls.DialWithDialer(dialer, "tcp", addrPort, &tls.Config{InsecureSkipVerify: true})
-		if err != nil {
-			return err
-		}
-		conn.Close()
-
-		return nil
-	}).Should(Succeed())
-})
-
-var _ = AfterSuite(func() {
-	cancel()
-	By("tearing down the test environment")
-	err := testEnv.Stop()
-	Expect(err).NotTo(HaveOccurred())
-})

api/v1alpha1/zz_generated.deepcopy.go

@@ -10,7 +10,7 @@ package v1alpha1
 
 import (
 	"k8s.io/api/core/v1"
-	"k8s.io/apimachinery/pkg/runtime"
+	runtime "k8s.io/apimachinery/pkg/runtime"
 )
 
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
@@ -58,6 +58,42 @@ func (in *AdditionalMetadata) DeepCopy() *AdditionalMetadata {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *AdditionalVolumeMounts) DeepCopyInto(out *AdditionalVolumeMounts) {
+	*out = *in
+	if in.APIServer != nil {
+		in, out := &in.APIServer, &out.APIServer
+		*out = make([]v1.VolumeMount, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+	if in.ControllerManager != nil {
+		in, out := &in.ControllerManager, &out.ControllerManager
+		*out = make([]v1.VolumeMount, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+	if in.Scheduler != nil {
+		in, out := &in.Scheduler, &out.Scheduler
+		*out = make([]v1.VolumeMount, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AdditionalVolumeMounts.
+func (in *AdditionalVolumeMounts) DeepCopy() *AdditionalVolumeMounts {
+	if in == nil {
+		return nil
+	}
+	out := new(AdditionalVolumeMounts)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *AddonSpec) DeepCopyInto(out *AddonSpec) {
 	*out = *in
@@ -254,35 +290,6 @@ func (in *ClientCertificate) DeepCopy() *ClientCertificate {
 	return out
 }
 
-// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *ComponentResourceRequirements) DeepCopyInto(out *ComponentResourceRequirements) {
-	*out = *in
-	if in.Limits != nil {
-		in, out := &in.Limits, &out.Limits
-		*out = make(v1.ResourceList, len(*in))
-		for key, val := range *in {
-			(*out)[key] = val.DeepCopy()
-		}
-	}
-	if in.Requests != nil {
-		in, out := &in.Requests, &out.Requests
-		*out = make(v1.ResourceList, len(*in))
-		for key, val := range *in {
-			(*out)[key] = val.DeepCopy()
-		}
-	}
-}
-
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ComponentResourceRequirements.
-func (in *ComponentResourceRequirements) DeepCopy() *ComponentResourceRequirements {
-	if in == nil {
-		return nil
-	}
-	out := new(ComponentResourceRequirements)
-	in.DeepCopyInto(out)
-	return out
-}
-
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ContentRef) DeepCopyInto(out *ContentRef) {
 	*out = *in
@@ -335,17 +342,22 @@ func (in *ControlPlaneComponentsResources) DeepCopyInto(out *ControlPlaneCompone
 	*out = *in
 	if in.APIServer != nil {
 		in, out := &in.APIServer, &out.APIServer
-		*out = new(ComponentResourceRequirements)
+		*out = new(v1.ResourceRequirements)
 		(*in).DeepCopyInto(*out)
 	}
 	if in.ControllerManager != nil {
 		in, out := &in.ControllerManager, &out.ControllerManager
-		*out = new(ComponentResourceRequirements)
+		*out = new(v1.ResourceRequirements)
 		(*in).DeepCopyInto(*out)
 	}
 	if in.Scheduler != nil {
 		in, out := &in.Scheduler, &out.Scheduler
-		*out = new(ComponentResourceRequirements)
+		*out = new(v1.ResourceRequirements)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.Kine != nil {
+		in, out := &in.Kine, &out.Kine
+		*out = new(v1.ResourceRequirements)
 		(*in).DeepCopyInto(*out)
 	}
 }
@@ -547,9 +559,25 @@ func (in *DataStoreStatus) DeepCopy() *DataStoreStatus {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *DatastoreUsedSecret) DeepCopyInto(out *DatastoreUsedSecret) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DatastoreUsedSecret.
+func (in *DatastoreUsedSecret) DeepCopy() *DatastoreUsedSecret {
+	if in == nil {
+		return nil
+	}
+	out := new(DatastoreUsedSecret)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *DeploymentSpec) DeepCopyInto(out *DeploymentSpec) {
 	*out = *in
+	out.RegistrySettings = in.RegistrySettings
 	if in.NodeSelector != nil {
 		in, out := &in.NodeSelector, &out.NodeSelector
 		*out = make(map[string]string, len(*in))
@@ -557,6 +585,7 @@ func (in *DeploymentSpec) DeepCopyInto(out *DeploymentSpec) {
 			(*out)[key] = val
 		}
 	}
+	in.Strategy.DeepCopyInto(&out.Strategy)
 	if in.Tolerations != nil {
 		in, out := &in.Tolerations, &out.Tolerations
 		*out = make([]v1.Toleration, len(*in))
@@ -587,6 +616,32 @@ func (in *DeploymentSpec) DeepCopyInto(out *DeploymentSpec) {
 		(*in).DeepCopyInto(*out)
 	}
 	in.AdditionalMetadata.DeepCopyInto(&out.AdditionalMetadata)
+	if in.AdditionalInitContainers != nil {
+		in, out := &in.AdditionalInitContainers, &out.AdditionalInitContainers
+		*out = make([]v1.Container, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+	if in.AdditionalContainers != nil {
+		in, out := &in.AdditionalContainers, &out.AdditionalContainers
+		*out = make([]v1.Container, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+	if in.AdditionalVolumes != nil {
+		in, out := &in.AdditionalVolumes, &out.AdditionalVolumes
+		*out = make([]v1.Volume, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+	if in.AdditionalVolumeMounts != nil {
+		in, out := &in.AdditionalVolumeMounts, &out.AdditionalVolumeMounts
+		*out = new(AdditionalVolumeMounts)
+		(*in).DeepCopyInto(*out)
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DeploymentSpec.
@@ -757,7 +812,7 @@ func (in *KonnectivityServerSpec) DeepCopyInto(out *KonnectivityServerSpec) {
 	*out = *in
 	if in.Resources != nil {
 		in, out := &in.Resources, &out.Resources
-		*out = new(ComponentResourceRequirements)
+		*out = new(v1.ResourceRequirements)
 		(*in).DeepCopyInto(*out)
 	}
 	if in.ExtraArgs != nil {
@@ -901,6 +956,11 @@ func (in *KubeconfigsStatus) DeepCopy() *KubeconfigsStatus {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *KubeletSpec) DeepCopyInto(out *KubeletSpec) {
 	*out = *in
+	if in.PreferredAddressTypes != nil {
+		in, out := &in.PreferredAddressTypes, &out.PreferredAddressTypes
+		*out = make([]KubeletPreferredAddressType, len(*in))
+		copy(*out, *in)
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeletSpec.
@@ -965,7 +1025,7 @@ func (in *KubernetesServiceStatus) DeepCopy() *KubernetesServiceStatus {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *KubernetesSpec) DeepCopyInto(out *KubernetesSpec) {
 	*out = *in
-	out.Kubelet = in.Kubelet
+	in.Kubelet.DeepCopyInto(&out.Kubelet)
 	if in.AdmissionControllers != nil {
 		in, out := &in.AdmissionControllers, &out.AdmissionControllers
 		*out = make(AdmissionControllers, len(*in))
@@ -1067,6 +1127,21 @@ func (in *PublicKeyPrivateKeyPairStatus) DeepCopy() *PublicKeyPrivateKeyPairStat
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *RegistrySettings) DeepCopyInto(out *RegistrySettings) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RegistrySettings.
+func (in *RegistrySettings) DeepCopy() *RegistrySettings {
+	if in == nil {
+		return nil
+	}
+	out := new(RegistrySettings)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *SecretReference) DeepCopyInto(out *SecretReference) {
 	*out = *in
@@ -1233,3 +1308,18 @@ func (in *TenantControlPlaneStatus) DeepCopy() *TenantControlPlaneStatus {
 	in.DeepCopyInto(out)
 	return out
 }
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *TenantControlPlaneStatusDataStore) DeepCopyInto(out *TenantControlPlaneStatusDataStore) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TenantControlPlaneStatusDataStore.
+func (in *TenantControlPlaneStatusDataStore) DeepCopy() *TenantControlPlaneStatusDataStore {
+	if in == nil {
+		return nil
+	}
+	out := new(TenantControlPlaneStatusDataStore)
+	in.DeepCopyInto(out)
+	return out
+}

assets/kamaji-logo.svg

@@ -1 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="11.85 8.10 202.80 187.55"><title>Kamaji</title><path d="M32.1 13.7c-2.4.9-6.3 3.5-8.6 5.8-7.7 7.7-7.5 5-7.5 82.5 0 77.4-.2 74.8 7.5 82.5 7.7 7.8 4.2 7.5 90 7.5s82.3.3 90-7.5c7.7-7.7 7.5-5.1 7.5-82.5s.2-74.8-7.5-82.5c-7.8-7.8-4.1-7.5-90.4-7.4-66.7 0-77.2.3-81 1.6zm160.5 9.9c1.9.9 4.4 3.1 5.7 4.8l2.2 3.1v141l-2.2 3.1c-4.8 6.7-1.1 6.4-84.8 6.4s-80 .3-84.8-6.4l-2.2-3.1v-141l2.2-3.1c4.8-6.6.8-6.4 84.6-6.4 68 0 76.3.2 79.3 1.6z"/><path d="M90.1 33.7c-5.1 2.5-7.3 6.7-6.8 13.1.3 4.1 1 5.9 3.3 8.4s2.5 3 .9 2.3c-2-.7-25.1-4.6-29-4.9-1.1 0-2 .5-2 1.4 0 1.1-1.2 1.5-4.9 1.5-6.7 0-6.8 1.9-.4 4 8.2 2.7 9 3.4 3.3 3.5-5.3 0-8.2 1.1-7.1 2.8.7 1.2-2.7 2.2-8.1 2.2-7 0-6.5 2.4 1.1 5.1l3.9 1.4-2.9.5c-4.3.8-3.2 2.3 2.8 4.1l5.3 1.5-5.2 2.7c-8.2 4.2-8.3 5.8-.4 6.1 5.6.2 7.3 1.1 4.2 2.1-2.3.7-2.8 3.1-.9 3.7.7.3-.5 2-2.8 4-5.6 5.3-4 6.4 6.2 4.5 4.4-.8 8.1-1.3 8.3-1.2.2.2-1.3 2.4-3.3 4.8-2 2.4-3.6 4.7-3.6 5.2 0 .4 1.4.5 3 .3 2.9-.4 4 .5 2 1.7-.5.3-1 1.3-1 2.2 0 1.6 2.2 1.5 6.5-.3 1.7-.7 1.6-.2-.9 3-5.4 7.2.7 6.5 13.6-1.4 2.7-1.7 5.1-3 5.4-3 .3 0-.9 2.1-2.7 4.6-4.5 6.6-2.5 7.9 3.7 2.3 4.6-4.3 4.7-4.3 3-1.2-1.9 3.8-2.1 5.6-.4 5.1.6-.2 7.1-7.1 14.3-15.4 7.2-8.2 13.7-14.9 14.5-14.9.8 0 7.3 6.7 14.6 15 7.2 8.2 13.7 15.1 14.3 15.3 1.6.5 1.4-1.4-.5-5-1.6-3.2-1.6-3.2 3.2 1 6 5.1 7.8 4 3.5-2.2-1.8-2.5-3-4.6-2.7-4.6.3 0 2.7 1.3 5.4 3 12.9 7.9 19 8.6 13.6 1.4-2.5-3.2-2.6-3.7-.9-3 5.9 2.5 7.7 1.7 5.6-2.3-.9-1.5-.6-1.7 2-1.3 3.8.6 3.7-.5-.7-5.7-2-2.3-3.5-4.4-3.2-4.6.2-.2 2.1 0 4.3.4 13.9 3 16.4 1.8 9.8-4.3-2.1-1.9-3.2-3.6-2.5-3.6 2 0 1.4-2.8-.9-3.5-3.2-1-1.3-2 4.2-2.1 7.9-.2 7.8-1.9-.4-6.1l-5.2-2.7 5.4-1.6c6.4-1.8 7.9-4 2.9-4.1h-3.3l3.9-1.5c7.3-2.6 8.4-5.4 2.2-5.4-5.1 0-9.6-1.1-9-2.2 1.1-1.7-1.8-2.8-7.1-2.8-5.7-.1-4.9-.8 3.3-3.5 6.4-2.1 6.3-4-.4-4-3.7 0-4.9-.4-4.9-1.5 0-.9-.9-1.4-2-1.4-3.9.3-27 4.2-29 4.9-1.6.7-1.4.2.9-2.3 3.7-4 4.7-11.3 2.2-16.1-4.8-9.2-18.8-9.3-23.8 0-4.4 8.3.2 18.4 9.5 20.5 3 .6 2.8.8-5.5 4l-8.8 3.3-8.7-3.3c-8.1-3.2-8.4-3.4-5.5-4.1 1.7-.3 4.3-1.5 5.7-2.7 13.1-10.3.6-30.4-14.4-23.1zm77.6 98.4c-3.6 2.1-.8 7.7 3.2 6.4 2.1-.6 3.5-3.1 2.5-4.6-1.1-1.8-4-2.7-5.7-1.8zm8.3 3.9c0 1.9.5 2.1 6.3 1.8 4.7-.2 6.2-.7 6.2-1.8s-1.5-1.6-6.2-1.8c-5.8-.3-6.3-.1-6.3 1.8zm-135.6.3c-.2.7-.3 7.4-.2 14.8l.3 13.4 3.3.3c3.1.3 3.2.2 3.2-3.4 0-2.5.7-4.6 2.1-6l2.1-2.3 5 6c3.9 4.7 5.6 5.9 7.8 5.9 1.6 0 3.1-.3 3.3-.8.3-.4-2.1-4-5.4-8.1-3.2-4-5.9-7.6-5.9-8 0-.4 2.5-3.1 5.5-6.1 3-3 5.5-5.8 5.5-6.2 0-.4-1.5-.8-3.3-.8-2.8 0-4.4 1-9.6 6.5-3.5 3.6-6.5 6.5-6.7 6.5-.2 0-.4-2.9-.4-6.5V135h-3c-1.7 0-3.3.6-3.6 1.3zm31.2 7c-1.1.8-1.5 1.9-1 3 .5 1.4 1.3 1.6 4 1.1 4.2-.8 8.4.2 8.4 2 0 .8-1.8 1.5-5.1 1.9-6 .7-8.9 2.9-8.9 6.6 0 3.2.8 4.4 3.7 6 2.9 1.5 5.2 1.4 8.6-.3 2.3-1.3 2.7-1.3 2.7 0 0 .9 1.1 1.4 3 1.4h3v-8.6c0-8.1-.1-8.7-2.9-11.5-2.5-2.5-3.7-2.9-8.3-2.9-3 0-6.2.6-7.2 1.3zm11.2 13.9c-.2 1.7-1.1 2.4-3.2 2.6-3.3.4-5.1-1-4.3-3.2.4-1.1 1.9-1.6 4.2-1.6 3.2 0 3.6.3 3.3 2.2zm13.4-4l.3 11.3h6l.5-7.8c.5-7.6 1.5-9.6 4.7-9.7 3 0 4.3 3.2 4.3 10.6v7.4h3c3 0 3 0 3-5.9 0-7.3 1.2-10.7 4.1-11.6 3.8-1.3 5.9 2.5 5.9 10.6v6.9h6v-9c0-8.3-.2-9.3-2.5-11.5-2.9-3-9.8-3.5-12.7-.8-1.7 1.5-1.9 1.5-3.6 0-2.2-2-9.2-2.3-11.1-.5-1.1 1-1.4 1-1.8 0-.3-.6-1.8-1.2-3.4-1.2h-3l.3 11.2zm45.4-9.9c-1.1.8-1.5 1.9-1 3 .5 1.4 1.3 1.6 4 1.1 4.2-.8 8.4.2 8.4 2 0 .8-1.8 1.5-5.1 1.9-6 .7-8.9 2.9-8.9 6.6 0 3.2.8 4.4 3.7 6 2.9 1.5 5.2 1.4 8.6-.3 2.3-1.3 2.7-1.3 2.7 0 0 .9 1.1 1.4 3 1.4h3v-8.6c0-8.1-.1-8.7-2.9-11.5-2.5-2.5-3.7-2.9-8.3-2.9-3 0-6.2.6-7.2 1.3zm11.2 13.9c-.2 1.7-1.1 2.4-3.2 2.6-3.3.4-5.1-1-4.3-3.2.4-1.1 1.9-1.6 4.2-1.6 3.2 0 3.6.3 3.3 2.2zm13-2.5c-.3 12.8-.3 12.8-2.7 12.8-1.5 0-2.7.8-3.1 2-2 5.4 9.4 4.3 11.9-1.2.6-1.3 1.1-7.7 1.1-14.3v-12h-6.9l-.3 12.7zm13.4-1.5l.3 11.3h6v-22l-3.3-.3-3.3-.3.3 11.3z"/></svg>

charts/kamaji/Chart.yaml

@@ -1,10 +1,8 @@
 apiVersion: v2
-appVersion: v0.1.1
-description: Kamaji is a tool aimed to build and operate a Managed Kubernetes Service
-  with a fraction of the operational burden. With Kamaji, you can deploy and operate
-  hundreds of Kubernetes clusters as a hyper-scaler.
+appVersion: v0.3.1
+description: Kamaji deploys and operates Kubernetes at scale with a fraction of the operational burden. Kamaji turns any Kubernetes cluster into an “admin cluster” to orchestrate other Kubernetes clusters called “tenant clusters”. Kamaji is special because the Control Plane components are running in a single pod instead of dedicated machines. This solution makes running multiple Control Planes cheaper and easier to deploy and operate.
 home: https://github.com/clastix/kamaji
-icon: https://github.com/clastix/kamaji/raw/master/assets/kamaji-logo.png
+icon: https://github.com/clastix/kamaji/raw/master/assets/logo-colored.png
 kubeVersion: ">=1.21.0-0"
 maintainers:
 - email: dario@tranchitella.eu
@@ -13,14 +11,12 @@ maintainers:
   name: Massimiliano Giovagnoli
 - email: me@bsctl.io
   name: Adriano Pezzuto
-- email: iam@mendrugory.com
-  name: Gonzalo Gabriel Jiménez Fuentes
 name: kamaji
 sources:
 - https://github.com/clastix/kamaji
 type: application
-version: 0.10.3
+version: 0.12.2
 annotations:
   catalog.cattle.io/certified: partner
   catalog.cattle.io/release-name: kamaji
-  catalog.cattle.io/display-name: Kamaji - Managed Kubernetes Service
+  catalog.cattle.io/display-name: Kamaji

charts/kamaji/README.md

@@ -1,8 +1,8 @@
 # kamaji
 
-![Version: 0.10.3](https://img.shields.io/badge/Version-0.10.3-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v0.1.1](https://img.shields.io/badge/AppVersion-v0.1.1-informational?style=flat-square)
+![Version: 0.12.2](https://img.shields.io/badge/Version-0.12.2-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v0.3.1](https://img.shields.io/badge/AppVersion-v0.3.1-informational?style=flat-square)
 
-Kamaji is a tool aimed to build and operate a Managed Kubernetes Service with a fraction of the operational burden. With Kamaji, you can deploy and operate hundreds of Kubernetes clusters as a hyper-scaler.
+Kamaji deploys and operates Kubernetes at scale with a fraction of the operational burden. Kamaji turns any Kubernetes cluster into an “admin cluster” to orchestrate other Kubernetes clusters called “tenant clusters”. Kamaji is special because the Control Plane components are running in a single pod instead of dedicated machines. This solution makes running multiple Control Planes cheaper and easier to deploy and operate.
 
 ## Maintainers
 
@@ -11,7 +11,6 @@ Kamaji is a tool aimed to build and operate a Managed Kubernetes Service with a
 | Dario Tranchitella | <dario@tranchitella.eu> |  |
 | Massimiliano Giovagnoli | <me@maxgio.it> |  |
 | Adriano Pezzuto | <me@bsctl.io> |  |
-| Gonzalo Gabriel Jiménez Fuentes | <iam@mendrugory.com> |  |
 
 ## Source Code
 
@@ -99,6 +98,7 @@ Here the values you can override:
 | etcd.overrides.endpoints | object | `{"etcd-0":"etcd-0.etcd.kamaji-system.svc.cluster.local","etcd-1":"etcd-1.etcd.kamaji-system.svc.cluster.local","etcd-2":"etcd-2.etcd.kamaji-system.svc.cluster.local"}` | (map) Dictionary of the endpoints for the etcd cluster's members, key is the name of the etcd server. Don't define the protocol (TLS is automatically inflected), or any port, inflected from .etcd.peerApiPort value. |
 | etcd.peerApiPort | int | `2380` | The peer API port which servers are listening to. |
 | etcd.persistence.accessModes[0] | string | `"ReadWriteOnce"` |  |
+| etcd.persistence.customAnnotations | object | `{}` | The custom annotations to add to the PVC |
 | etcd.persistence.size | string | `"10Gi"` |  |
 | etcd.persistence.storageClass | string | `""` |  |
 | etcd.port | int | `2379` | The client request port. |
@@ -128,6 +128,7 @@ Here the values you can override:
 | serviceAccount.annotations | object | `{}` |  |
 | serviceAccount.create | bool | `true` |  |
 | serviceAccount.name | string | `"kamaji-controller-manager"` |  |
+| serviceMonitor.enabled | bool | `false` | Toggle the ServiceMonitor true if you have Prometheus Operator installed and configured |
 | temporaryDirectoryPath | string | `"/tmp/kamaji"` | Directory which will be used to work with temporary files. (default "/tmp/kamaji") |
 | tolerations | list | `[]` | Kubernetes node taints that the Kamaji controller pods would tolerate |
 

charts/kamaji/app-readme.md

@@ -1,30 +1,12 @@
-# Kamaji - Managed Kubernetes Service
+# Kamaji
 
-Kamaji is a tool aimed to build and operate a Managed Kubernetes Service with a fraction of the operational burden.
+Kamaji deploys and operates Kubernetes at scale with a fraction of the operational burden.
 
 Useful links:
 - [Kamaji Github repository](https://github.com/clastix/kamaji)
-- [Kamaji Documentation](https://github.com/clastix/kamaji/docs/)
+- [Kamaji Documentation](https://kamaji.clastix.io)
 
 ## Requirements
 
 * Kubernetes v1.22+
-* Helm v3
-
-# Installation
-
-To install the Chart with the release name `kamaji`:
-
-        helm upgrade --install --namespace kamaji-system --create-namespace clastix/kamaji
-
-Show the status:
-
-        helm status kamaji -n kamaji-system
-
-Upgrade the Chart
-
-        helm upgrade kamaji -n kamaji-system clastix/kamaji
-
-Uninstall the Chart
-
-        helm uninstall kamaji -n kamaji-system
+* Helm v3

charts/kamaji/crds/datastore.yaml

@@ -4,7 +4,7 @@ kind: CustomResourceDefinition
 metadata:
   annotations:
     cert-manager.io/inject-ca-from: kamaji-system/kamaji-serving-cert
-    controller-gen.kubebuilder.io/version: v0.9.2
+    controller-gen.kubebuilder.io/version: v0.11.4
   name: datastores.kamaji.clastix.io
 spec:
   group: kamaji.clastix.io

charts/kamaji/templates/_helpers.tpl

@@ -46,9 +46,9 @@ app.kubernetes.io/managed-by: {{ .Release.Service }}
 Selector labels
 */}}
 {{- define "kamaji.selectorLabels" -}}
-app.kubernetes.io/name: {{ include "kamaji.name" . }}
-app.kubernetes.io/instance: {{ .Release.Name }}
-app.kubernetes.io/component: controller-manager
+app.kubernetes.io/name: {{ default (include "kamaji.name" .) .name }}
+app.kubernetes.io/instance: {{ default .Release.Name .instance }}
+app.kubernetes.io/component: {{ default "controller-manager" .component }}
 {{- end }}
 
 {{/*
@@ -69,6 +69,13 @@ Create the name of the Service to user for webhooks
 {{- printf "%s-webhook-service" (include "kamaji.fullname" .) }}
 {{- end }}
 
+{{/*
+Create the name of the Service to user for metrics
+*/}}
+{{- define "kamaji.metricsServiceName" -}}
+{{- printf "%s-metrics-service" (include "kamaji.fullname" .) }}
+{{- end }}
+
 {{/*
 Create the name of the cert-manager secret
 */}}

charts/kamaji/templates/certmanager_certificate.yaml

@@ -2,8 +2,8 @@ apiVersion: cert-manager.io/v1
 kind: Certificate
 metadata:
   labels:
-    {{- include "kamaji.labels" . | nindent 4 }}
-    app.kubernetes.io/component: certificate
+    {{- $data := . | mustMergeOverwrite (dict "component" "certificate") -}}
+    {{- include "kamaji.labels" $data | nindent 4 }}
   name: {{ include "kamaji.certificateName" . }}
   namespace: {{ .Release.Namespace }}
 spec:

charts/kamaji/templates/certmanager_issuer.yaml

@@ -2,8 +2,8 @@ apiVersion: cert-manager.io/v1
 kind: Issuer
 metadata:
   labels:
-    {{- include "kamaji.labels" . | nindent 4 }}
-    app.kubernetes.io/component: issuer
+    {{- $data := . | mustMergeOverwrite (dict "component" "issuer") -}}
+    {{- include "kamaji.labels" $data | nindent 4 }}
   name: kamaji-selfsigned-issuer
   namespace: {{ .Release.Namespace }}
 spec:

charts/kamaji/templates/controller.yaml

@@ -62,6 +62,9 @@ spec:
         - containerPort: 9443
           name: webhook-server
           protocol: TCP
+        - containerPort: 8080
+          name: metrics
+          protocol: TCP
         - containerPort: 8081
           name: healthcheck
           protocol: TCP
@@ -74,11 +77,16 @@ spec:
         securityContext:
           {{- toYaml .Values.securityContext | nindent 12 }}
         volumeMounts:
+          - mountPath: /tmp
+            name: tmp
           - mountPath: /tmp/k8s-webhook-server/serving-certs
             name: cert
             readOnly: true
       terminationGracePeriodSeconds: 10
       volumes:
+        - name: tmp
+          emptyDir:
+            medium: Memory
         - name: cert
           secret:
             defaultMode: 420

charts/kamaji/templates/datastore.yaml

@@ -12,7 +12,12 @@ spec:
     {{- include "datastore.endpoints" . | indent 4 }}
 {{- if (and .Values.datastore.basicAuth.usernameSecret.name .Values.datastore.basicAuth.passwordSecret.name) }}
   basicAuth:
-    {{- .Values.datastore.basicAuth | toYaml | nindent 4 }}
+    username:
+      secretReference:
+        {{- .Values.datastore.basicAuth.usernameSecret | toYaml | nindent 8 }}
+    password:
+      secretReference:
+        {{- .Values.datastore.basicAuth.passwordSecret | toYaml | nindent 8 }}
 {{- end }}
   tlsConfig:
     certificateAuthority:

charts/kamaji/templates/etcd_job_postdelete.yaml

@@ -28,4 +28,8 @@ spec:
             - --ignore-not-found=true
             - {{ include "etcd.caSecretName" . }}
             - {{ include "etcd.clientSecretName" . }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
 {{- end }}

charts/kamaji/templates/etcd_job_postinstall.yaml

@@ -63,4 +63,8 @@ spec:
         - name: certs
           secret:
             secretName: {{ include "etcd.caSecretName" . }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
 {{- end }}

charts/kamaji/templates/etcd_job_preinstall.yaml

@@ -57,4 +57,8 @@ spec:
             name: {{ include "etcd.csrConfigMapName" . }}
         - name: certs
           emptyDir: {}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
 {{- end }}

charts/kamaji/templates/etcd_sts.yaml

@@ -81,6 +81,10 @@ spec:
   volumeClaimTemplates:
   - metadata:
       name: data
+      {{- with .Values.etcd.persistence.customAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
     spec:
       storageClassName: {{ .Values.etcd.persistence.storageClassName }}
       accessModes:

charts/kamaji/templates/mutatingwebhookconfiguration.yaml

@@ -4,30 +4,10 @@ metadata:
   annotations:
     cert-manager.io/inject-ca-from: {{ .Release.Namespace }}/{{ include "kamaji.certificateName" . }}
   labels:
-    {{- include "kamaji.labels" . | nindent 4 }}
-    app.kubernetes.io/instance: mutating-webhook-configuration
+    {{- $data := . | mustMergeOverwrite (dict "instance" "mutating-webhook-configuration") -}}
+    {{- include "kamaji.labels" $data | nindent 4 }}
   name: kamaji-mutating-webhook-configuration
 webhooks:
-  - admissionReviewVersions:
-      - v1
-    clientConfig:
-      service:
-        name: {{ include "kamaji.webhookServiceName" . }}
-        namespace: {{ .Release.Namespace }}
-        path: /mutate-kamaji-clastix-io-v1alpha1-datastore
-    failurePolicy: Fail
-    name: mdatastore.kb.io
-    rules:
-      - apiGroups:
-          - kamaji.clastix.io
-        apiVersions:
-          - v1alpha1
-        operations:
-          - CREATE
-          - UPDATE
-        resources:
-          - datastores
-    sideEffects: None
   - admissionReviewVersions:
       - v1
     clientConfig:

charts/kamaji/templates/service_metrics.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    {{- $data := . | mustMergeOverwrite (dict "component" "metrics") -}}
+    {{- include "kamaji.labels" $data | nindent 4 }}
+  name: {{ include "kamaji.metricsServiceName" . }}
+  namespace: {{ .Release.Namespace }}
+spec:
+  ports:
+    - port: 8080
+      name: metrics
+      protocol: TCP
+      targetPort: metrics
+  selector:
+    {{- include "kamaji.selectorLabels" . | nindent 4 }}

charts/kamaji/templates/service_webhook.yaml

@@ -2,15 +2,15 @@ apiVersion: v1
 kind: Service
 metadata:
   labels:
-    {{- include "kamaji.labels" . | nindent 4 }}
-    app.kubernetes.io/component: webhook
-    app.kubernetes.io/instance: webhook-service
+    {{- $data := . | mustMergeOverwrite (dict "component" "webhook" "instance" "webhook-service") -}}
+    {{- include "kamaji.labels" $data | nindent 4 }}
   name: {{ include "kamaji.webhookServiceName" . }}
   namespace: {{ .Release.Namespace }}
 spec:
   ports:
     - port: 443
       protocol: TCP
+      name: webhook-server
       targetPort: webhook-server
   selector:
     {{- include "kamaji.selectorLabels" . | nindent 4 }}

charts/kamaji/templates/servicemonitor.yaml

@@ -0,0 +1,21 @@
+{{- if and (.Capabilities.APIVersions.Has "monitoring.coreos.com/v1") .Values.serviceMonitor.enabled }}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  labels:
+    {{- $data := . | mustMergeOverwrite (dict "component" "servicemonitor") -}}
+    {{- include "kamaji.labels" $data | nindent 4 }}
+  name: {{ include "kamaji.fullname" . }}
+  namespace: {{ .Release.Namespace }}
+spec:
+  endpoints:
+    - path: /metrics
+      port: metrics
+      scheme: http
+  namespaceSelector:
+    matchNames:
+      - {{ .Release.Namespace }}
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: {{ include "kamaji.name" . }}
+{{- end }}

charts/kamaji/templates/validatingwebhookconfiguration.yaml

@@ -4,8 +4,8 @@ metadata:
   annotations:
     cert-manager.io/inject-ca-from: {{ .Release.Namespace }}/{{ include "kamaji.certificateName" . }}
   labels:
-    {{- include "kamaji.labels" . | nindent 4 }}
-    app.kubernetes.io/instance: validating-webhook-configuration
+    {{- $data := . | mustMergeOverwrite (dict "instance" "validating-webhook-configuration") -}}
+    {{- include "kamaji.labels" $data | nindent 4 }}
   name: kamaji-validating-webhook-configuration
 webhooks:
   - admissionReviewVersions:

charts/kamaji/values.yaml

@@ -15,6 +15,11 @@ image:
 # -- A list of extra arguments to add to the kamaji controller default ones
 extraArgs: []
 
+
+serviceMonitor:
+  # -- Toggle the ServiceMonitor true if you have Prometheus Operator installed and configured
+  enabled: false
+
 etcd:
   # -- Install an etcd with enabled multi-tenancy along with Kamaji
   deploy: true
@@ -52,6 +57,9 @@ etcd:
     storageClass: ""
     accessModes:
     - ReadWriteOnce
+    # -- The custom annotations to add to the PVC
+    customAnnotations: {}
+    #  volumeType: local
 
   overrides:
     caSecret:

cmd/manager/cmd.go

@@ -23,8 +23,11 @@ import (
 	"github.com/clastix/kamaji/controllers"
 	"github.com/clastix/kamaji/controllers/soot"
 	"github.com/clastix/kamaji/internal"
+	"github.com/clastix/kamaji/internal/builders/controlplane"
 	datastoreutils "github.com/clastix/kamaji/internal/datastore/utils"
 	"github.com/clastix/kamaji/internal/webhook"
+	"github.com/clastix/kamaji/internal/webhook/handlers"
+	"github.com/clastix/kamaji/internal/webhook/routes"
 )
 
 func NewCmd(scheme *runtime.Scheme) *cobra.Command {
@@ -41,6 +44,7 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command {
 		managerServiceName        string
 		webhookCABundle           []byte
 		migrateJobImage           string
+		maxConcurrentReconciles   int
 
 		webhookCAPath string
 	)
@@ -111,11 +115,12 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command {
 					KineContainerImage:   kineImage,
 					TmpBaseDirectory:     tmpDirectory,
 				},
-				TriggerChan:          tcpChannel,
-				KamajiNamespace:      managerNamespace,
-				KamajiServiceAccount: managerServiceAccountName,
-				KamajiService:        managerServiceName,
-				KamajiMigrateImage:   migrateJobImage,
+				TriggerChan:             tcpChannel,
+				KamajiNamespace:         managerNamespace,
+				KamajiServiceAccount:    managerServiceAccountName,
+				KamajiService:           managerServiceName,
+				KamajiMigrateImage:      migrateJobImage,
+				MaxConcurrentReconciles: maxConcurrentReconciles,
 			}
 
 			if err = reconciler.SetupWithManager(mgr); err != nil {
@@ -124,12 +129,6 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command {
 				return err
 			}
 
-			if err = (&webhook.Freeze{}).SetupWithManager(mgr); err != nil {
-				setupLog.Error(err, "unable to register webhook", "webhook", "Freeze")
-
-				return err
-			}
-
 			if err = (&kamajiv1alpha1.DatastoreUsedSecret{}).SetupWithManager(ctx, mgr); err != nil {
 				setupLog.Error(err, "unable to create indexer", "indexer", "DatastoreUsedSecret")
 
@@ -142,13 +141,38 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command {
 				return err
 			}
 
-			if err = (&kamajiv1alpha1.TenantControlPlane{}).SetupWebhookWithManager(mgr, datastore); err != nil {
-				setupLog.Error(err, "unable to create webhook", "webhook", "TenantControlPlane")
-
-				return err
-			}
-			if err = (&kamajiv1alpha1.DataStore{}).SetupWebhookWithManager(mgr); err != nil {
-				setupLog.Error(err, "unable to create webhook", "webhook", "DataStore")
+			err = webhook.Register(mgr, map[routes.Route][]handlers.Handler{
+				routes.TenantControlPlaneMigrate{}: {
+					handlers.Freeze{},
+				},
+				routes.TenantControlPlaneDefaults{}: {
+					handlers.TenantControlPlaneDefaults{DefaultDatastore: datastore},
+				},
+				routes.TenantControlPlaneValidate{}: {
+					handlers.TenantControlPlaneName{},
+					handlers.TenantControlPlaneVersion{},
+					handlers.TenantControlPlaneKubeletAddresses{},
+					handlers.TenantControlPlaneDataStore{Client: mgr.GetClient()},
+					handlers.TenantControlPlaneDeployment{
+						Client: mgr.GetClient(),
+						DeploymentBuilder: controlplane.Deployment{
+							Client:             mgr.GetClient(),
+							KineContainerImage: kineImage,
+						},
+						KonnectivityBuilder: controlplane.Konnectivity{
+							Scheme: *mgr.GetScheme(),
+						},
+					},
+				},
+				routes.DataStoreValidate{}: {
+					handlers.DataStoreValidation{Client: mgr.GetClient()},
+				},
+				routes.DataStoreSecrets{}: {
+					handlers.DataStoreSecretValidation{Client: mgr.GetClient()},
+				},
+			})
+			if err != nil {
+				setupLog.Error(err, "unable to create webhook")
 
 				return err
 			}
@@ -185,6 +209,7 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command {
 			return nil
 		},
 	}
+
 	// Setting zap logger
 	zapfs := flag.NewFlagSet("zap", flag.ExitOnError)
 	opts := zap.Options{
@@ -201,6 +226,7 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command {
 	cmd.Flags().StringVar(&kineImage, "kine-image", "rancher/kine:v0.9.2-amd64", "Container image along with tag to use for the Kine sidecar container (used only if etcd-storage-type is set to one of kine strategies).")
 	cmd.Flags().StringVar(&datastore, "datastore", "etcd", "The default DataStore that should be used by Kamaji to setup the required storage.")
 	cmd.Flags().StringVar(&migrateJobImage, "migrate-image", fmt.Sprintf("clastix/kamaji:v%s", internal.GitTag), "Specify the container image to launch when a TenantControlPlane is migrated to a new datastore.")
+	cmd.Flags().IntVar(&maxConcurrentReconciles, "max-concurrent-tcp-reconciles", 1, "Specify the number of workers for the Tenant Control Plane controller (beware of CPU consumption)")
 	cmd.Flags().StringVar(&managerNamespace, "pod-namespace", os.Getenv("POD_NAMESPACE"), "The Kubernetes Namespace on which the Operator is running in, required for the TenantControlPlane migration jobs.")
 	cmd.Flags().StringVar(&managerServiceName, "webhook-service-name", "kamaji-webhook-service", "The Kamaji webhook server Service name which is used to get validation webhooks, required for the TenantControlPlane migration jobs.")
 	cmd.Flags().StringVar(&managerServiceAccountName, "serviceaccount-name", os.Getenv("SERVICE_ACCOUNT"), "The Kubernetes Namespace on which the Operator is running in, required for the TenantControlPlane migration jobs.")

cmd/root.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	"github.com/spf13/cobra"
+	_ "go.uber.org/automaxprocs" // Automatically set `GOMAXPROCS` to match Linux container CPU quota.
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"

config/crd/bases/kamaji.clastix.io_datastores.yaml

@@ -3,8 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.9.2
-  creationTimestamp: null
+    controller-gen.kubebuilder.io/version: v0.11.4
   name: datastores.kamaji.clastix.io
 spec:
   group: kamaji.clastix.io

config/default/kustomization.yaml

@@ -19,8 +19,7 @@ bases:
 - ../samples
 - ../webhook
 - ../certmanager
-# [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'.
-#- ../prometheus
+- ../prometheus
 
 patchesStrategicMerge:
 # Mount the controller config file for loading manager configurations

config/manager/kustomization.yaml

@@ -13,4 +13,4 @@ kind: Kustomization
 images:
 - name: controller
   newName: clastix/kamaji
-  newTag: v0.1.1
+  newTag: v0.3.1

config/manager/manager.yaml

@@ -42,6 +42,10 @@ spec:
         image: controller:latest
         imagePullPolicy: Always
         name: manager
+        ports:
+          - containerPort: 8080
+            name: metrics
+            protocol: TCP
         securityContext:
           allowPrivilegeEscalation: false
         livenessProbe:

config/prometheus/kustomization.yaml

@@ -1,2 +1,5 @@
 resources:
 - monitor.yaml
+
+configurations:
+- kustomizeconfig.yaml

config/prometheus/kustomizeconfig.yaml

@@ -0,0 +1,4 @@
+varReference:
+  - kind: ServiceMonitor
+    group: monitoring.coreos.com
+    path: spec/namespaceSelector/matchNames

config/prometheus/monitor.yaml

@@ -1,4 +1,3 @@
-
 # Prometheus Monitor Service (Metrics)
 apiVersion: monitoring.coreos.com/v1
 kind: ServiceMonitor
@@ -10,11 +9,11 @@ metadata:
 spec:
   endpoints:
     - path: /metrics
-      port: https
-      scheme: https
-      bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
-      tlsConfig:
-        insecureSkipVerify: true
+      port: metrics
+      scheme: http
+  namespaceSelector:
+    matchNames:
+      - $(SERVICE_NAMESPACE)
   selector:
     matchLabels:
       control-plane: controller-manager

config/rbac/role.yaml

@@ -2,7 +2,6 @@
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
-  creationTimestamp: null
   name: manager-role
 rules:
 - apiGroups:

config/samples/kamaji_v1alpha1_tenantcontrolplane.yaml

@@ -1,22 +1,22 @@
 apiVersion: kamaji.clastix.io/v1alpha1
 kind: TenantControlPlane
 metadata:
-  name: test
+  name: 126-k8s
 spec:
   controlPlane:
     deployment:
-      replicas: 1
+      replicas: 2
     service:
       serviceType: LoadBalancer
   kubernetes:
-    version: "v1.25.4"
+    version: "v1.26.0"
     kubelet:
-      cgroupfs: cgroupfs
-    admissionControllers:
-      - ResourceQuota
-      - LimitRanger
+      cgroupfs: systemd
   networkProfile:
     port: 6443
   addons:
     coreDNS: {}
     kubeProxy: {}
+    konnectivity:
+      server:
+        port: 8132

config/samples/kamaji_v1alpha1_tenantcontrolplane_additionalcontainers.yaml

@@ -0,0 +1,30 @@
+apiVersion: kamaji.clastix.io/v1alpha1
+kind: TenantControlPlane
+metadata:
+  name: additionalcontainers
+spec:
+  dataStore: postgresql-bronze
+  controlPlane:
+    deployment:
+      replicas: 1
+      additionalInitContainers:
+        - name: init
+          image: registry.k8s.io/e2e-test-images/busybox:1.29-4
+          command:
+            - /bin/sh
+            - -c
+            - echo hello world
+      additionalContainers:
+        - name: nginx
+          image: registry.k8s.io/e2e-test-images/nginx:1.15-4
+    service:
+      serviceType: LoadBalancer
+  kubernetes:
+    version: "v1.26.0"
+    kubelet:
+      cgroupfs: systemd
+  networkProfile:
+    port: 6443
+  addons:
+    coreDNS: {}
+    kubeProxy: {}

config/samples/kamaji_v1alpha1_tenantcontrolplane_additionalvolumes.yaml

@@ -0,0 +1,60 @@
+apiVersion: kamaji.clastix.io/v1alpha1
+kind: TenantControlPlane
+metadata:
+  name: additional-volumes
+spec:
+  controlPlane:
+    deployment:
+      replicas: 1
+      additionalVolumes:
+        - name: api-server-volume
+          configMap:
+            name: api-server-extra-cm
+        - name: controller-manager-volume
+          configMap:
+            name: controller-manager-extra-cm
+        - name: scheduler-volume
+          configMap:
+            name: scheduler-extra-cm
+      additionalVolumeMounts:
+        apiServer:
+          - name: api-server-volume
+            mountPath: "/tmp/api-server"
+        controllerManager:
+          - name: controller-manager-volume
+            mountPath: "/tmp/controller-manager"
+        scheduler:
+          - name: scheduler-volume
+            mountPath: "/tmp/scheduler"
+    service:
+      serviceType: LoadBalancer
+  kubernetes:
+    version: "v1.26.0"
+    kubelet:
+      cgroupfs: systemd
+  networkProfile:
+    port: 6443
+  addons:
+    coreDNS: {}
+    kubeProxy: {}
+---
+apiVersion: v1
+data:
+  api-server: "This is an API Server volume"
+kind: ConfigMap
+metadata:
+  name: api-server-extra-cm
+---
+apiVersion: v1
+data:
+  controller-manager: "This is a Controller Manager volume"
+kind: ConfigMap
+metadata:
+  name: controller-manager-extra-cm
+---
+apiVersion: v1
+data:
+  controller-manager: "This is a Scheduler volume"
+kind: ConfigMap
+metadata:
+  name: scheduler-extra-cm

config/samples/kamaji_v1alpha1_tenantcontrolplane_kine.yaml

@@ -0,0 +1,18 @@
+apiVersion: kamaji.clastix.io/v1alpha1
+kind: TenantControlPlane
+metadata:
+  name: kine
+spec:
+  addons:
+    coreDNS: {}
+    kubeProxy: {}
+  controlPlane:
+    deployment:
+      replicas: 1
+    service:
+      serviceType: LoadBalancer
+  dataStore: postgresql-bronze
+  kubernetes:
+    kubelet:
+      cgroupfs: systemd
+    version: v1.26.0

config/samples/kamaji_v1alpha1_tenantcontrolplane_konnectivity.yaml

@@ -0,0 +1,21 @@
+apiVersion: kamaji.clastix.io/v1alpha1
+kind: TenantControlPlane
+metadata:
+  name: konnectivity-addon
+spec:
+  deployment:
+    replicas: 2
+  service:
+    serviceType: LoadBalancer
+  kubernetes:
+    version: "v1.26.0"
+    kubelet:
+      cgroupfs: systemd
+  networkProfile:
+    port: 6443
+  addons:
+    coreDNS: {}
+    kubeProxy: {}
+    konnectivity:
+      server:
+        port: 8132

config/webhook/manifests.yaml

@@ -2,29 +2,8 @@
 apiVersion: admissionregistration.k8s.io/v1
 kind: MutatingWebhookConfiguration
 metadata:
-  creationTimestamp: null
   name: mutating-webhook-configuration
 webhooks:
-- admissionReviewVersions:
-  - v1
-  clientConfig:
-    service:
-      name: webhook-service
-      namespace: system
-      path: /mutate-kamaji-clastix-io-v1alpha1-datastore
-  failurePolicy: Fail
-  name: mdatastore.kb.io
-  rules:
-  - apiGroups:
-    - kamaji.clastix.io
-    apiVersions:
-    - v1alpha1
-    operations:
-    - CREATE
-    - UPDATE
-    resources:
-    - datastores
-  sideEffects: None
 - admissionReviewVersions:
   - v1
   clientConfig:
@@ -49,7 +28,6 @@ webhooks:
 apiVersion: admissionregistration.k8s.io/v1
 kind: ValidatingWebhookConfiguration
 metadata:
-  creationTimestamp: null
   name: validating-webhook-configuration
 webhooks:
 - admissionReviewVersions:

controllers/resources.go

@@ -14,6 +14,7 @@ import (
 
 	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
 	"github.com/clastix/kamaji/controllers/finalizers"
+	builder "github.com/clastix/kamaji/internal/builders/controlplane"
 	"github.com/clastix/kamaji/internal/datastore"
 	"github.com/clastix/kamaji/internal/resources"
 	ds "github.com/clastix/kamaji/internal/resources/datastore"
@@ -245,7 +246,7 @@ func getKonnectivityServerRequirementsResources(c client.Client) []resources.Res
 
 func getKonnectivityServerPatchResources(c client.Client) []resources.Resource {
 	return []resources.Resource{
-		&konnectivity.KubernetesDeploymentResource{Client: c},
+		&konnectivity.KubernetesDeploymentResource{Builder: builder.Konnectivity{Scheme: *c.Scheme()}, Client: c},
 		&konnectivity.ServiceResource{Client: c},
 	}
 }

controllers/soot/controllers/coredns.go

@@ -60,7 +60,7 @@ func (c *CoreDNS) Reconcile(ctx context.Context, request reconcile.Request) (rec
 		return reconcile.Result{}, nil
 	}
 
-	if err = utils.UpdateStatus(ctx, c.AdminClient, c.GetTenantControlPlaneFunc, resource); err != nil {
+	if err = utils.UpdateStatus(ctx, c.AdminClient, tcp, resource); err != nil {
 		c.logger.Error(err, "update status failed", "resource", resource.GetName())
 
 		return reconcile.Result{}, err

controllers/soot/controllers/konnectivity.go

@@ -60,7 +60,7 @@ func (k *KonnectivityAgent) Reconcile(ctx context.Context, _ reconcile.Request)
 			continue
 		}
 
-		if err = utils.UpdateStatus(ctx, k.AdminClient, k.GetTenantControlPlaneFunc, resource); err != nil {
+		if err = utils.UpdateStatus(ctx, k.AdminClient, tcp, resource); err != nil {
 			k.logger.Error(err, "update status failed", "resource", resource.GetName())
 
 			return reconcile.Result{}, err

controllers/soot/controllers/kubeadm_phase.go

@@ -50,7 +50,7 @@ func (k *KubeadmPhase) Reconcile(ctx context.Context, _ reconcile.Request) (reco
 		return reconcile.Result{}, nil
 	}
 
-	if err = utils.UpdateStatus(ctx, k.Phase.GetClient(), k.GetTenantControlPlaneFunc, k.Phase); err != nil {
+	if err = utils.UpdateStatus(ctx, k.Phase.GetClient(), tcp, k.Phase); err != nil {
 		k.logger.Error(err, "update status failed")
 
 		return reconcile.Result{}, err

controllers/soot/controllers/kubeproxy.go

@@ -60,7 +60,7 @@ func (k *KubeProxy) Reconcile(ctx context.Context, _ reconcile.Request) (reconci
 		return reconcile.Result{}, nil
 	}
 
-	if err = utils.UpdateStatus(ctx, k.AdminClient, k.GetTenantControlPlaneFunc, resource); err != nil {
+	if err = utils.UpdateStatus(ctx, k.AdminClient, tcp, resource); err != nil {
 		k.logger.Error(err, "update status failed")
 
 		return reconcile.Result{}, err

controllers/soot/manager.go

@@ -123,22 +123,27 @@ func (m *Manager) Reconcile(ctx context.Context, request reconcile.Request) (res
 	// the soot manager if this is already registered.
 	v, ok := m.sootMap[request.String()]
 	if ok {
-		// The TenantControlPlane is in non-ready mode, or marked for deletion:
-		// we don't want to pollute with messages due to broken connection.
-		// Once the TCP will be ready again, the event will be intercepted and the manager started back.
-		if tcpStatus == kamajiv1alpha1.VersionNotReady {
+		switch {
+		case tcpStatus == kamajiv1alpha1.VersionCARotating:
+			// The TenantControlPlane CA has been rotated, it means the running manager
+			// must be restarted to avoid certificate signed by unknown authority errors.
 			return reconcile.Result{}, m.cleanup(ctx, request, tcp)
-		}
-
-		for _, trigger := range v.triggers {
-			trigger <- event.GenericEvent{Object: tcp}
+		case tcpStatus == kamajiv1alpha1.VersionNotReady:
+			// The TenantControlPlane is in non-ready mode, or marked for deletion:
+			// we don't want to pollute with messages due to broken connection.
+			// Once the TCP will be ready again, the event will be intercepted and the manager started back.
+			return reconcile.Result{}, m.cleanup(ctx, request, tcp)
+		default:
+			for _, trigger := range v.triggers {
+				trigger <- event.GenericEvent{Object: tcp}
+			}
 		}
 
 		return reconcile.Result{}, nil
 	}
 	// No need to start a soot manager if the TenantControlPlane is not ready:
 	// enqueuing back is not required since we're going to get that event once ready.
-	if tcpStatus == kamajiv1alpha1.VersionNotReady {
+	if tcpStatus == kamajiv1alpha1.VersionNotReady || tcpStatus == kamajiv1alpha1.VersionCARotating {
 		log.FromContext(ctx).Info("skipping start of the soot manager for a not ready instance")
 
 		return reconcile.Result{}, nil

controllers/tenantcontrolplane_controller.go

@@ -6,18 +6,23 @@ package controllers
 import (
 	"context"
 	"fmt"
+	"strings"
+	"time"
 
+	"github.com/juju/mutex/v2"
 	"github.com/pkg/errors"
 	appsv1 "k8s.io/api/apps/v1"
 	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
 	networkingv1 "k8s.io/api/networking/v1"
-	errors2 "k8s.io/apimachinery/pkg/api/errors"
+	apimachineryerrors "k8s.io/apimachinery/pkg/api/errors"
 	k8stypes "k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/util/workqueue"
+	"k8s.io/utils/clock"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/builder"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 	"sigs.k8s.io/controller-runtime/pkg/event"
 	"sigs.k8s.io/controller-runtime/pkg/handler"
@@ -36,14 +41,17 @@ import (
 
 // TenantControlPlaneReconciler reconciles a TenantControlPlane object.
 type TenantControlPlaneReconciler struct {
-	Client               client.Client
-	APIReader            client.Reader
-	Config               TenantControlPlaneReconcilerConfig
-	TriggerChan          TenantControlPlaneChannel
-	KamajiNamespace      string
-	KamajiServiceAccount string
-	KamajiService        string
-	KamajiMigrateImage   string
+	Client                  client.Client
+	APIReader               client.Reader
+	Config                  TenantControlPlaneReconcilerConfig
+	TriggerChan             TenantControlPlaneChannel
+	KamajiNamespace         string
+	KamajiServiceAccount    string
+	KamajiService           string
+	KamajiMigrateImage      string
+	MaxConcurrentReconciles int
+
+	clock mutex.Clock
 }
 
 // TenantControlPlaneReconcilerConfig gives the necessary configuration for TenantControlPlaneReconciler.
@@ -68,7 +76,7 @@ func (r *TenantControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R
 
 	tenantControlPlane, err := r.getTenantControlPlane(ctx, req.NamespacedName)()
 	if err != nil {
-		if errors2.IsNotFound(err) {
+		if apimachineryerrors.IsNotFound(err) {
 			log.Info("resource may have been deleted, skipping")
 
 			return ctrl.Result{}, nil
@@ -79,6 +87,25 @@ func (r *TenantControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R
 		return ctrl.Result{}, err
 	}
 
+	releaser, err := mutex.Acquire(r.mutexSpec(tenantControlPlane))
+	if err != nil {
+		switch {
+		case errors.As(err, &mutex.ErrTimeout):
+			log.Info("acquire timed out, current process is blocked by another reconciliation")
+
+			return ctrl.Result{Requeue: true}, nil
+		case errors.As(err, &mutex.ErrCancelled):
+			log.Info("acquire cancelled")
+
+			return ctrl.Result{Requeue: true}, nil
+		default:
+			log.Error(err, "acquire failed")
+
+			return ctrl.Result{}, err
+		}
+	}
+	defer releaser.Release()
+
 	markedToBeDeleted := tenantControlPlane.GetDeletionTimestamp() != nil
 
 	if markedToBeDeleted && !controllerutil.ContainsFinalizer(tenantControlPlane, finalizers.DatastoreFinalizer) {
@@ -156,7 +183,7 @@ func (r *TenantControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R
 			continue
 		}
 
-		if err = utils.UpdateStatus(ctx, r.Client, r.getTenantControlPlane(ctx, req.NamespacedName), resource); err != nil {
+		if err = utils.UpdateStatus(ctx, r.Client, tenantControlPlane, resource); err != nil {
 			log.Error(err, "update of the resource failed", "resource", resource.GetName())
 
 			return ctrl.Result{}, err
@@ -164,7 +191,11 @@ func (r *TenantControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R
 
 		log.Info(fmt.Sprintf("%s has been configured", resource.GetName()))
 
-		return ctrl.Result{}, nil
+		if result == resources.OperationResultEnqueueBack {
+			log.Info("requested enqueuing back", "resources", resource.GetName())
+
+			return ctrl.Result{Requeue: true}, nil
+		}
 	}
 
 	log.Info(fmt.Sprintf("%s has been reconciled", tenantControlPlane.GetName()))
@@ -172,8 +203,20 @@ func (r *TenantControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R
 	return ctrl.Result{}, nil
 }
 
+func (r *TenantControlPlaneReconciler) mutexSpec(obj client.Object) mutex.Spec {
+	return mutex.Spec{
+		Name:    strings.ReplaceAll(fmt.Sprintf("kamaji%s", obj.GetUID()), "-", ""),
+		Clock:   r.clock,
+		Delay:   10 * time.Millisecond,
+		Timeout: time.Second,
+		Cancel:  nil,
+	}
+}
+
 // SetupWithManager sets up the controller with the Manager.
 func (r *TenantControlPlaneReconciler) SetupWithManager(mgr ctrl.Manager) error {
+	r.clock = clock.RealClock{}
+
 	return ctrl.NewControllerManagedBy(mgr).
 		Watches(&source.Channel{Source: r.TriggerChan}, handler.Funcs{GenericFunc: func(genericEvent event.GenericEvent, limitingInterface workqueue.RateLimitingInterface) {
 			limitingInterface.AddRateLimited(ctrl.Request{
@@ -217,6 +260,9 @@ func (r *TenantControlPlaneReconciler) SetupWithManager(mgr ctrl.Manager) error
 
 			return ok && v == "migrate"
 		}))).
+		WithOptions(controller.Options{
+			MaxConcurrentReconciles: r.MaxConcurrentReconciles,
+		}).
 		Complete(r)
 }
 

controllers/utils/update_status.go

@@ -7,24 +7,27 @@ import (
 	"context"
 	"fmt"
 
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/util/retry"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
 	"github.com/clastix/kamaji/internal/resources"
 )
 
-func UpdateStatus(ctx context.Context, client client.Client, tcpRetrieval TenantControlPlaneRetrievalFn, resource resources.Resource) error {
-	updateErr := retry.RetryOnConflict(retry.DefaultRetry, func() error {
-		tenantControlPlane, err := tcpRetrieval()
-		if err != nil {
-			return err
-		}
+func UpdateStatus(ctx context.Context, client client.Client, tcp *kamajiv1alpha1.TenantControlPlane, resource resources.Resource) error {
+	updateErr := retry.RetryOnConflict(retry.DefaultRetry, func() (err error) {
+		defer func() {
+			if err != nil {
+				_ = client.Get(ctx, types.NamespacedName{Name: tcp.Name, Namespace: tcp.Namespace}, tcp)
+			}
+		}()
 
-		if err = resource.UpdateTenantControlPlaneStatus(ctx, tenantControlPlane); err != nil {
+		if err = resource.UpdateTenantControlPlaneStatus(ctx, tcp); err != nil {
 			return fmt.Errorf("error applying TenantcontrolPlane status: %w", err)
 		}
 
-		if err = client.Status().Update(ctx, tenantControlPlane); err != nil {
+		if err = client.Status().Update(ctx, tcp); err != nil {
 			return fmt.Errorf("error updating tenantControlPlane status: %w", err)
 		}
 

deploy/kamaji-azure.env

@@ -15,7 +15,7 @@ export KAMAJI_NAMESPACE=kamaji-system
 export TENANT_NAMESPACE=default
 export TENANT_NAME=tenant-00
 export TENANT_DOMAIN=$KAMAJI_REGION.cloudapp.azure.com
-export TENANT_VERSION=v1.25.0
+export TENANT_VERSION=v1.26.0
 export TENANT_PORT=6443 # port used to expose the tenant api server
 export TENANT_PROXY_PORT=8132 # port used to expose the konnectivity server
 export TENANT_POD_CIDR=10.36.0.0/16

deploy/kamaji.env

@@ -5,7 +5,7 @@ export KAMAJI_NAMESPACE=kamaji-system
 export TENANT_NAMESPACE=default
 export TENANT_NAME=tenant-00
 export TENANT_DOMAIN=clastix.labs
-export TENANT_VERSION=v1.25.0
+export TENANT_VERSION=v1.26.0
 export TENANT_PORT=6443 # port used to expose the tenant api server
 export TENANT_PROXY_PORT=8132 # port used to expose the konnectivity server
 export TENANT_POD_CIDR=10.36.0.0/16

deploy/kind/Makefile

@@ -11,13 +11,13 @@ prometheus-stack:
 	helm repo update
 	helm install prometheus-stack --create-namespace -n monitoring prometheus-community/kube-prometheus-stack
 
-reqs: kind ingress-nginx etcd-cluster cert-manager
+reqs: kind ingress-nginx cert-manager
 
 cert-manager:
 	@kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.10.1/cert-manager.yaml
 
 kamaji: reqs
-	@kubectl apply -f $(kind_path)/../../config/install.yaml
+	helm install kamaji --create-namespace -n kamaji-system $(kind_path)/../../charts/kamaji
 
 destroy: kind/destroy etcd-certificates/cleanup
 

deploy/kind/join-node.bash

@@ -7,7 +7,7 @@ export DOCKER_IMAGE_NAME="kindest/node"
 export DOCKER_NETWORK="kind"
 
 # Variables
-export KUBERNETES_VERSION=${1:-v1.23.5}
+export KUBERNETES_VERSION=${1:-v1.23.4}
 export KUBECONFIG="${KUBECONFIG:-/tmp/kubeconfig}"
 
 if [ -z $2 ]

docs/content/concepts.md

@@ -11,23 +11,25 @@ These are requirements of the design behind Kamaji:
 Goals and scope may vary as the project evolves.
 
 ## Tenant Control Plane
-What makes Kamaji special is that the Control Plane of a _“tenant cluster”_ is just one or more regular pods running in a namespace of the _“admin cluster”_ instead of a dedicated set of Virtual Machines. This solution makes running control planes at scale cheaper and easier to deploy and operate. The Tenant Control Plane components are packaged in the same way they are running in bare metal or virtual nodes. We leverage the `kubeadm` code to set up the control plane components as they were running on their own server. The unchanged images of upstream `kube-apiserver`, `kube-scheduler`, and `kube-controller-manager` are used.
+Kamaji is special because the Control Planes of the _“tenant cluster”_ are regular pods running in a namespace of the _“admin cluster”_ instead of a dedicated set of Virtual Machines. This solution makes running Control Planes at scale cheaper and easier to deploy and operate. The Tenant Control Plane components are packaged in the same way they are running in bare metal or virtual nodes. We leverage the `kubeadm` code to set up the control plane components as they were running on their own server. The unchanged images of upstream `kube-apiserver`, `kube-scheduler`, and `kube-controller-manager` are used.
 
-High Availability and rolling updates of the Tenant Control Plane pods are provided by a regular Deployment. Autoscaling based on the metrics is available. A Service is used to espose the Tenant Control Plane outside of the _“admin cluster”_. The `LoadBalancer` service type is used, `NodePort` and `ClusterIP` with an Ingress Controller are still viable options, depending on the case. 
+High Availability and rolling updates of the Tenant Control Plane pods are provided by a regular Deployment. Autoscaling based on the metrics is available. A Service is used to espose the Tenant Control Plane outside of the _“admin cluster”_. The `LoadBalancer` service type is used, `NodePort` and `ClusterIP` are other viable options, depending on the case.
 
 Kamaji offers a [Custom Resource Definition](https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/) to provide a declarative approach of managing a Tenant Control Plane. This *CRD* is called `TenantControlPlane`, or `tcp` in short.
 
 All the _“tenant clusters”_ built with Kamaji are fully compliant CNCF Kubernetes clusters and are compatible with the standard Kubernetes toolchains everybody knows and loves. See [CNCF compliance](reference/conformance.md).
 
 ## Tenant worker nodes
-And what about the tenant worker nodes? They are just _"worker nodes"_, i.e. regular virtual or bare metal machines, connecting to the APIs server of the Tenant Control Plane. Kamaji's goal is to manage the lifecycle of hundreds of these _“tenant clusters”_, not only one, so how to add another tenant cluster to Kamaji? As you could expect, you have just deploys a new Tenant Control Plane in one of the _“admin cluster”_ namespace, and then joins the tenant worker nodes to it.
 
-We have in roadmap, the Cluster APIs support as well as a Terraform provider so that you can create _“tenant clusters”_ in a declarative way.
+And what about the tenant worker nodes?
+They are just _"worker nodes"_, i.e. regular virtual or bare metal machines, connecting to the APIs server of the Tenant Control Plane.
+Kamaji's goal is to manage the lifecycle of hundreds of these _“tenant clusters”_, not only one, so how to add another tenant cluster to Kamaji?
+As you could expect, you have just deploys a new Tenant Control Plane in one of the _“admin cluster”_ namespace, and then joins the tenant worker nodes to it.
+
+A [Cluster API ControlPlane provider](https://github.com/clastix/cluster-api-control-plane-provider-kamaji) has been released, allowing to offer a Cluster API-native declarative lifecycle, by automating the worker nodes join.
 
 ## Datastores
-Putting the Tenant Control Plane in a pod is the easiest part. Also, we have to make sure each tenant cluster saves the state to be able to store and retrieve data. A dedicated `etcd` cluster for each tenant cluster doesn’t scale well for a managed service because `etcd` data persistence can be cumbersome at scale, rising the operational effort to mitigate it. So we have to find an alternative keeping in mind our goal for a resilient and cost-optimized solution at the same time.
-
-As we can deploy any Kubernetes cluster with an external `etcd` cluster, we explored this option for the tenant control planes. On the admin cluster, we can deploy a multi-tenant `etcd` datastore to save the state of multiple tenant clusters. Kamaji offers a Custom Resource Definition called `DataStore` to provide a declarative approach of managing Tenant datastores. With this solution, the resiliency is guaranteed by the usual `etcd` mechanism, and the pods' count remains under control, so it solves the main goal of resiliency and costs optimization. The trade-off here is that we have to operate an external datastore, in addition to `etcd` of the _“admin cluster”_ and manage the access to be sure that each _“tenant cluster”_ uses only its data.
+Putting the Tenant Control Plane in a pod is the easiest part. Also, we have to make sure each tenant cluster saves the state to be able to store and retrieve data. As we can deploy a Kubernetes cluster with an external `etcd` cluster, we explored this option for the Tenant Control Planes. On the admin cluster, you can deploy one or multi-tenant `etcd` to save the state of multiple tenant clusters. Kamaji offers a Custom Resource Definition called `DataStore` to provide a declarative approach of managing multiple datastores. By sharing the datastore between multiple tenants, the resiliency is still guaranteed and the pods' count remains under control, so it solves the main goal of resiliency and costs optimization. The trade-off here is that you have to operate external datastores, in addition to `etcd` of the _“admin cluster”_ and manage the access to be sure that each _“tenant cluster”_ uses only its data.
 
 ### Other storage drivers
 Kamaji offers the option of using a more capable datastore than `etcd` to save the state of multiple tenants' clusters. Thanks to the native [kine](https://github.com/k3s-io/kine) integration, you can run _MySQL_ or _PostgreSQL_ compatible databases as datastore for _“tenant clusters”_.
@@ -35,6 +37,11 @@ Kamaji offers the option of using a more capable datastore than `etcd` to save t
 ### Pooling
 By default, Kamaji is expecting to persist all the _“tenant clusters”_ data in a unique datastore that could be backed by different drivers. However, you can pick a different datastore for a specific set of _“tenant clusters”_ that could have different resources assigned or a different tiering. Pooling of multiple datastore is an option you can leverage for a very large set of _“tenant clusters”_ so you can distribute the load properly. As future improvements, we have a _datastore scheduler_ feature in roadmap so that Kamaji itself can assign automatically a _“tenant cluster”_ to the best datastore in the pool.
 
+### Migration
+In order to simplify Day2 Operations and reduce the operational burden, Kamaji provides the capability to live migrate data from a datastore to another one of the same driver without manual and error prone backup and restore operations.
+
+> Currently, live data migration is only available between datastores having the same driver.
+
 ## Konnectivity
 
 In addition to the standard control plane containers, Kamaji creates an instance of [konnectivity-server](https://kubernetes.io/docs/concepts/architecture/control-plane-node-communication/) running as sidecar container in the `tcp` pod and exposed on port `8132` of the `tcp` service.

docs/content/contribute.md

@@ -41,11 +41,7 @@ Please, split changes into several and documented small commits: this will help
 ## Code convention
 
 Kamaji is written in Golang. The changes must follow the Pull Request method where a _GitHub Action_ will
-check the `golangci-lint`, so ensure your changes respect the coding standard.
-
-### golint
-
-You can easily check them issuing the _Make_ recipe `golint`.
+check the `golangci-lint`, so ensure your changes respect the coding standard. You can easily check them issuing the _Make_ recipe `golint`.
 
 ```
 # make golint
@@ -54,10 +50,10 @@ golangci-lint run -c .golangci.yml
 
 > Enabled linters and related options are defined in the [.golanci.yml file](https://github.com/clastix/Kamaji/blob/master/.golangci.yml)
 
+Please, add a new single line at end of any file as the current coding style.
+
 ## Finding contributions to work on
-Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the
-default GitHub issue labels (enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any 'help wanted'
-and 'good first issue' issues are a great place to start.
+Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels (enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any 'help wanted' and 'good first issue' issues are a great place to start.
 
 ## Design Docs
 
@@ -74,10 +70,31 @@ When filing an issue, please check existing open, or recently closed, issues to
 * Any modifications you've made relevant to the bug
 * Anything unusual about your environment or deployment
 
-## Miscellanea
+## Governance
+
+This document lays out the guidelines under which the Kamaji project will be governed.
+The goal is to make sure that the roles and responsibilities are well defined and clarify how decisions are made.
+
+### Roles
+
+In the context of Kamaji project, we consider the following roles:
+
+* __Users__: everyone using Kamaji, typically willing to provide feedback by proposing features and/or filing issues.
+
+* __Contributors__: everyone contributing code, documentation, examples, tests, and participating in feature proposals as well as design discussions.
+
+* __Maintainers__: are responsible for engaging with and assisting contributors to iterate on the contributions until it reaches acceptable quality. Maintainers can decide whether the contributions can be accepted into the project or rejected.
+
+### Release Management
+
+The release process will be governed by Maintainers.
+
+### Roadmap Planning
+
+Maintainers will share roadmap and release versions as milestones in GitHub [project's page](https://github.com/clastix/kamaji). 
 
-Please, add a new single line at end of any file as the current coding style.
 
 ## Licensing
 
-See the [LICENSE](https://github.com/clastix/Kamaji/blob/master/LICENSE) file for our project's licensing. We can ask you to confirm the licensing of your contribution.
+See the [LICENSE](https://github.com/clastix/Kamaji/blob/master/LICENSE) file for our project's licensing. We can ask you to confirm the licensing of your contribution.
+

docs/content/contribute/governance.md

@@ -1,22 +0,0 @@
-# Governance
-
-This document lays out the guidelines under which the Kamaji project will be governed.
-The goal is to make sure that the roles and responsibilities are well defined and clarify how decisions are made.
-
-## Roles
-
-In the context of Kamaji project, we consider the following roles:
-
-* __Users__: everyone using Kamaji, typically willing to provide feedback by proposing features and/or filing issues.
-
-* __Contributors__: everyone contributing code, documentation, examples, tests, and participating in feature proposals as well as design discussions.
-
-* __Maintainers__: are responsible for engaging with and assisting contributors to iterate on the contributions until it reaches acceptable quality. Maintainers can decide whether the contributions can be accepted into the project or rejected.
-
-## Release Management
-
-The release process will be governed by Maintainers.
-
-## Roadmap Planning
-
-Maintainers will share roadmap and release versions as milestones in GitHub [project's page](https://github.com/clastix/kamaji). 

docs/content/contribute/index.md

@@ -1,2 +0,0 @@
-# Guidelines
-Guidelines for community contributions.

docs/content/getting-started.md

@@ -1,207 +1,344 @@
-# Getting started
+# Getting started with Kamaji
+This guide will lead you through the process of creating a working Kamaji setup on a generic infrastructure.
 
-This document explains how to deploy a minimal Kamaji setup on [KinD](https://kind.sigs.k8s.io/) for development scopes. Please refer to the [Kamaji documentation](concepts.md) for understanding all the terms used in this guide, as for example: `admin cluster`, `tenant cluster`, and `tenant control plane`.
+!!! warning ""
+    The material here is relatively dense. We strongly encourage you to dedicate time to walk through these instructions, with a mind to learning. We do NOT provide any "one-click" deployment here. However, once you've understood the components involved it is encouraged that you build suitable, auditable GitOps deployment processes around your final infrastructure.
 
-## Pre-requisites
+The guide requires:
 
-We assume you have installed on your workstation:
+- a bootstrap machine
+- a Kubernetes cluster to run the Admin and Tenant Control Planes
+- an arbitrary number of machines to host `Tenant`s' workloads
 
-- [Docker](https://docker.com)
-- [KinD](https://kind.sigs.k8s.io/)
-- [kubectl@v1.25.0](https://kubernetes.io/docs/tasks/tools/#kubectl)
-- [kubeadm@v1.25.0](https://kubernetes.io/docs/tasks/tools/#kubeadm)
+## Summary
+
+  * [Prepare the bootstrap workspace](#prepare-the-bootstrap-workspace)
+  * [Access Admin cluster](#access-admin-cluster)
+  * [Install Cert Manager](#install-cert-manager)
+  * [Install Kamaji controller](#install-kamaji-controller)
+  * [Create Tenant Cluster](#create-tenant-cluster)
+  * [Cleanup](#cleanup)
+
+## Prepare the bootstrap workspace
+On the bootstrap machine, clone the repo and prepare the workspace directory:
+
+```bash
+git clone https://github.com/clastix/kamaji
+cd kamaji/deploy
+```
+
+We assume you have installed on the bootstrap machine:
+
+- [kubectl](https://kubernetes.io/docs/tasks/tools/#kubectl)
+- [kubeadm](https://kubernetes.io/docs/tasks/tools/#kubeadm)
+- [helm](https://helm.sh/docs/intro/install/)
 - [jq](https://stedolan.github.io/jq/)
-- [openssl](https://www.openssl.org/)
-- [cfssl/cfssljson](https://github.com/cloudflare/cfssl)
 
+## Access Admin cluster
+In Kamaji, an Admin Cluster is a regular Kubernetes cluster which hosts zero to many Tenant Cluster Control Planes. The admin cluster acts as management cluster for all the Tenant clusters and hosts monitoring, logging, and governance of Kamaji setup, including all Tenant clusters. 
 
-> Starting from Kamaji v0.0.2, `kubectl` and `kubeadm` need to meet at least minimum version to `v1.25.0`:
-> this is required due to the latest changes addressed from the release Kubernetes 1.25 release regarding the `kubelet-config` ConfigMap required for the node join.
-
-## Setup Kamaji on KinD
-
-The instance of Kamaji is made of a single node hosting:
-
-- admin control-plane
-- admin worker
-- multi-tenant datastore
-
-### Standard installation
-
-You can install your KinD cluster, ETCD multi-tenant cluster and Kamaji operator with a **single command**:
+Throughout the following instructions, shell variables are used to indicate values that you should adjust to your environment:
 
 ```bash
-$ make -C deploy/kind
+source kamaji.env
 ```
 
-Now you can [create your first `TenantControlPlane`](#deploy-tenant-control-plane).
+Any regular and conformant Kubernetes v1.22+ cluster can be turned into a Kamaji setup. To work properly, the admin cluster should provide:
 
-### Data store-specific
+- CNI module installed, eg. [Calico](https://github.com/projectcalico/calico), [Cilium](https://github.com/cilium/cilium).
+- CSI module installed with a Storage Class for the Tenant datastores. Local Persistent Volumes are an option.
+- Support for LoadBalancer service type, eg. [MetalLB](https://metallb.universe.tf/), or a Cloud based controller.
+- Optionally, a Monitoring Stack installed, eg. [Prometheus](https://github.com/prometheus-community).
 
-Kamaji offers the possibility of using a different storage system than `ETCD` for the tenants, like `MySQL` or `PostgreSQL` compatible databases.
-
-First, setup a KinD cluster:
+Make sure you have a `kubeconfig` file with admin permissions on the cluster you want to turn into Kamaji Admin Cluster and check you can access:
 
 ```bash
-$ make -C deploy/kind kind
+kubectl cluster-info
 ```
 
-#### ETCD
+## Install Cert Manager
 
-Deploy a multi-tenant `ETCD` cluster into the Kamaji node:
+Kamaji takes advantage of the [dynamic admission control](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/), such as validating and mutating webhook configurations. These webhooks are secured by a TLS communication, and the certificates are managed by [`cert-manager`](https://cert-manager.io/), making it a prerequisite that must be installed:
 
 ```bash
-$ make -C deploy/kind etcd-cluster
+helm repo add jetstack https://charts.jetstack.io
+helm repo update
+helm install \
+  cert-manager jetstack/cert-manager \
+  --namespace cert-manager \
+  --create-namespace \
+  --version v1.11.0 \
+  --set installCRDs=true
 ```
 
-Now you're ready to [install Kamaji operator](#install-kamaji).
+## Install Kamaji Controller
 
-#### MySQL
+Installing Kamaji via Helm charts is the preferred way. The Kamaji controller needs to access a Datastore in order to save data of the tenants' clusters. The Kamaji Helm Chart provides the installation of a basic unamanaged `etcd` as datastore, out of box. 
 
-Deploy a MySQL/MariaDB backend into the Kamaji node:
+Install Kamaji with `helm` using an unmanaged `etcd` as default datastore:
 
 ```bash
-$ make -C deploy/kine/mysql mariadb
+helm repo add clastix https://clastix.github.io/charts
+helm repo update
+helm install kamaji clastix/kamaji -n kamaji-system --create-namespace
 ```
 
-Adjust the Kamaji install manifest `config/install.yaml` according to the example of a MySQL DataStore `config/samples/kamaji_v1alpha1_datastore_mysql.yaml` and make sure Kamaji uses the proper datastore name:
+!!! note "A managed datastore is highly recommended in production"
+     The [kamaji-etcd](https://github.com/clastix/kamaji-etcd) project provides the code to setup a multi-tenant `etcd` running as StatefulSet made of three replicas. Optionally, Kamaji offers support for a more robust storage system, as `MySQL` or `PostgreSQL` compatible database, thanks to the native [kine](https://github.com/k3s-io/kine) integration.
 
-```
---datastore={.metadata.name}
-```
+## Create Tenant Cluster
 
-Now you're ready to [install Kamaji operator](#install-kamaji).
+### Tenant Control Plane
 
-#### PostgreSQL
+A tenant control plane of example looks like:
 
-Deploy a PostgreSQL backend into the Kamaji node:
-
-```bash
-$ make -C deploy/kine/postgresql postgresql
-```
-
-Adjust the Kamaji install manifest `config/install.yaml` according to the example of a PostgreSQL DataStore `config/samples/kamaji_v1alpha1_datastore_postgresql.yaml` and make sure Kamaji uses the proper datastore name:
-
-```
---datastore={.metadata.name}
-```
-
-Now you're ready to [install Kamaji operator](#install-kamaji).
-
-### Install Kamaji
-
-Kamaji takes advantage of the [dynamic admission control](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/), such as validating and mutating webhook configurations.
-These webhooks are secured by a TLS communication, and the certificates are managed by [`cert-manager`](https://cert-manager.io/), making it a prerequisite that must be [installed](https://cert-manager.io/docs/installation/).
-
-```bash
-$ kubectl apply -f config/install.yaml
-```
-
-> Please note that this single YAML manifest is missing some required automations.
-> The preferred way to install Kamaji is using its Helm Chart.
-> Please, refer to the section [**Setup Kamaji on a generic infrastructure**.](/guides/kamaji-deployment-guide#install-kamaji-controller)
-
-### Deploy Tenant Control Plane
-
-Now it is the moment of deploying your first tenant control plane.
-
-```bash
-$ kubectl apply -f - <<EOF
+```yaml
+cat > ${TENANT_NAMESPACE}-${TENANT_NAME}-tcp.yaml <<EOF
 apiVersion: kamaji.clastix.io/v1alpha1
 kind: TenantControlPlane
 metadata:
-  name: tenant1
+  name: ${TENANT_NAME}
+  namespace: ${TENANT_NAMESPACE}
 spec:
+  dataStore: default
   controlPlane:
     deployment:
-      replicas: 2
+      replicas: 3
       additionalMetadata:
-        annotations:
-          environment.clastix.io: tenant1
-          tier.clastix.io: "0"
         labels:
-          tenant.clastix.io: tenant1
-          kind.clastix.io: deployment
+          tenant.clastix.io: ${TENANT_NAME}
+      extraArgs:
+        apiServer: []
+        controllerManager: []
+        scheduler: []
+      resources:
+        apiServer:
+          requests:
+            cpu: 250m
+            memory: 512Mi
+          limits: {}
+        controllerManager:
+          requests:
+            cpu: 125m
+            memory: 256Mi
+          limits: {}
+        scheduler:
+          requests:
+            cpu: 125m
+            memory: 256Mi
+          limits: {}
     service:
       additionalMetadata:
-        annotations:
-          environment.clastix.io: tenant1
-          tier.clastix.io: "0"
         labels:
-          tenant.clastix.io: tenant1
-          kind.clastix.io: service
-      serviceType: NodePort
+          tenant.clastix.io: ${TENANT_NAME}
+      serviceType: LoadBalancer
   kubernetes:
-    version: "v1.23.4"
+    version: ${TENANT_VERSION}
     kubelet:
-      cgroupfs: cgroupfs
+      cgroupfs: systemd
     admissionControllers:
-    - LimitRanger
-    - ResourceQuota
+      - ResourceQuota
+      - LimitRanger
   networkProfile:
-    address: "172.18.0.2"
-    port: 31443
+    port: ${TENANT_PORT}
     certSANs:
-    - "test.clastixlabs.io"
-    serviceCidr: "10.96.0.0/16"
-    podCidr: "10.244.0.0/16"
-    dnsServiceIPs: 
-    - "10.96.0.10"
+    - ${TENANT_NAME}.${TENANT_DOMAIN}
+    serviceCidr: ${TENANT_SVC_CIDR}
+    podCidr: ${TENANT_POD_CIDR}
+    dnsServiceIPs:
+    - ${TENANT_DNS_SERVICE}
   addons:
     coreDNS: {}
     kubeProxy: {}
+    konnectivity:
+      server:
+        port: ${TENANT_PROXY_PORT}
+        resources:
+          requests:
+            cpu: 100m
+            memory: 128Mi
+          limits: {}
 EOF
+
+kubectl -n ${TENANT_NAMESPACE} apply -f ${TENANT_NAMESPACE}-${TENANT_NAME}-tcp.yaml
 ```
 
-> Check networkProfile fields according to your installation
-> To let Kamaji works in kind, you have indicate that the service must be [NodePort](https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport)
+After a few seconds, check the created resources in the tenants namespace and when ready it will look similar to the following:
 
-### Get Kubeconfig
+```command
+kubectl -n ${TENANT_NAMESPACE} get tcp,deploy,pods,svc
 
-Let's retrieve kubeconfig and store in `/tmp/kubeconfig`
+NAME                           VERSION   STATUS   CONTROL-PLANE ENDPOINT   KUBECONFIG                   DATASTORE   AGE
+tenantcontrolplane/tenant-00   v1.25.2   Ready    192.168.32.240:6443      tenant-00-admin-kubeconfig   default     2m20s
 
-```bash
-$ kubectl get secrets tenant1-admin-kubeconfig -o json \
- | jq -r '.data["admin.conf"]' \
- | base64 -d > /tmp/kubeconfig
- ```
+NAME                        READY   UP-TO-DATE   AVAILABLE   AGE
+deployment.apps/tenant-00   3/3     3            3           118s
 
-It can be export it, to facilitate the next tasks:
+NAME                             READY   STATUS    RESTARTS   AGE
+pod/tenant-00-58847c8cdd-7hc4n   4/4     Running   0          82s
+pod/tenant-00-58847c8cdd-ft5xt   4/4     Running   0          82s
+pod/tenant-00-58847c8cdd-shc7t   4/4     Running   0          82s
 
-```bash
-$ export KUBECONFIG=/tmp/kubeconfig
+NAME                TYPE           CLUSTER-IP      EXTERNAL-IP      PORT(S)                         AGE
+service/tenant-00   LoadBalancer   10.32.132.241   192.168.32.240   6443:32152/TCP,8132:32713/TCP   2m20s
 ```
 
-### Install CNI
+The regular Tenant Control Plane containers: `kube-apiserver`, `kube-controller-manager`, `kube-scheduler` are running unchanged in the `tcp` pods instead of dedicated machines and they are exposed through a service on the port `6443` of worker nodes in the admin cluster.
 
-We highly recommend to install [kindnet](https://github.com/aojea/kindnet) as CNI for your kamaji TCP.
+The `LoadBalancer` service type is used to expose the Tenant Control Plane on the assigned `loadBalancerIP` acting as `ControlPlaneEndpoint` for the worker nodes and other clients as, for example, `kubectl`. Service types `NodePort` and `ClusterIP` are still viable options to expose the Tenant Control Plane, depending on the case. High Availability and rolling updates of the Tenant Control Planes are provided by the `tcp` Deployment and all the resources reconcilied by the Kamaji controller.
+
+### Working with Tenant Control Plane
+
+Collect the external IP address of the `tcp` service:
 
 ```bash
-$ kubectl create -f https://raw.githubusercontent.com/aojea/kindnet/master/install-kindnet.yaml
+TENANT_ADDR=$(kubectl -n ${TENANT_NAMESPACE} get svc ${TENANT_NAME} -o json | jq -r ."spec.loadBalancerIP")
+```
+
+and check it out:
+
+```bash
+curl -k https://${TENANT_ADDR}:${TENANT_PORT}/healthz
+curl -k https://${TENANT_ADDR}:${TENANT_PORT}/version
+```
+
+The `kubeconfig` required to access the Tenant Control Plane is stored in a secret:
+
+```bash
+kubectl get secrets -n ${TENANT_NAMESPACE} ${TENANT_NAME}-admin-kubeconfig -o json \
+  | jq -r '.data["admin.conf"]' \
+  | base64 --decode \
+  > ${TENANT_NAMESPACE}-${TENANT_NAME}.kubeconfig
+```
+
+and let's check it out:
+
+```bash
+kubectl --kubeconfig=${TENANT_NAMESPACE}-${TENANT_NAME}.kubeconfig cluster-info
+
+Kubernetes control plane is running at https://192.168.32.240:6443
+CoreDNS is running at https://192.168.32.240:6443/api/v1/namespaces/kube-system/services/kube-dns:dns/proxy
+```
+
+Check out how the Tenant control Plane advertises itself to workloads:
+
+```bash
+kubectl --kubeconfig=${TENANT_NAMESPACE}-${TENANT_NAME}.kubeconfig get svc
+
+NAMESPACE     NAME         TYPE        CLUSTER-IP   EXTERNAL-IP   PORT(S)   AGE
+default       kubernetes   ClusterIP   10.32.0.1    <none>        443/TCP   6m
+```
+
+```bash
+kubectl --kubeconfig=${TENANT_NAMESPACE}-${TENANT_NAME}.kubeconfig get ep
+
+NAME         ENDPOINTS             AGE
+kubernetes   192.168.32.240:6443   18m
+```
+
+And make sure it is `${TENANT_ADDR}:${TENANT_PORT}`.
+
+### Prepare worker nodes to join
+
+Currently, Kamaji does not provide any helper for creation of tenant worker nodes.
+You should get a set of machines from your infrastructure provider, turn them into worker nodes, and then join to the tenant control plane with the `kubeadm`.
+
+Kamaji is sticking to the [Cluster Management API](https://github.com/kubernetes-sigs/cluster-api) project contracts by providing a `ControlPlane` provider.
+Please, refer to the [official repository](https://github.com/clastix/cluster-api-control-plane-provider-kamaji) to learn more about it.
+
+You can use the provided helper script `/deploy/nodes-prerequisites.sh`, in order to install the dependencies on all the worker nodes:
+
+- Install `containerd` as container runtime
+- Install `crictl`, the command line for working with `containerd`
+- Install `kubectl`, `kubelet`, and `kubeadm` in the desired version
+
+!!! warning ""
+    The provided script is just a facility: it assumes all worker nodes are running `Ubuntu 20.04`. Make sure to adapt the script if you're using a different distribution.
+
+Run the script:
+
+```bash
+HOSTS=(${WORKER0} ${WORKER1} ${WORKER2})
+./nodes-prerequisites.sh ${TENANT_VERSION:1} ${HOSTS[@]}
 ```
 
 ### Join worker nodes
+The current approach for joining nodes is to use `kubeadm` and therefore, we will create a bootstrap token to perform the action. In order to facilitate the step, we will store the entire command of joining in a variable:
 
 ```bash
-$ make -C deploy/kind kamaji-kind-worker-join
+JOIN_CMD=$(echo "sudo ")$(kubeadm --kubeconfig=${TENANT_NAMESPACE}-${TENANT_NAME}.kubeconfig token create --print-join-command)
 ```
 
-> To add more worker nodes, run again the command above.
-
-Check out the node:
+A bash loop will be used to join all the available nodes.
 
 ```bash
-$ kubectl get nodes
-NAME           STATUS   ROLES    AGE   VERSION
-d2d4b468c9de   Ready    <none>   44s   v1.23.4
+HOSTS=(${WORKER0} ${WORKER1} ${WORKER2})
+for i in "${!HOSTS[@]}"; do
+  HOST=${HOSTS[$i]}
+  ssh ${USER}@${HOST} -t ${JOIN_CMD};
+done
 ```
 
-> For more complex scenarios (exposing port, different version and so on), run `join-node.bash`.
+Checking the nodes:
 
-Tenant control plane provision has been finished in a minimal Kamaji setup based on KinD. Therefore, you could develop, test and make your own experiments with Kamaji.
+```bash
+kubectl --kubeconfig=${TENANT_NAMESPACE}-${TENANT_NAME}.kubeconfig get nodes 
+
+NAME                  STATUS     ROLES    AGE   VERSION
+tenant-00-worker-00   NotReady   <none>   25s   v1.25.0
+tenant-00-worker-01   NotReady   <none>   17s   v1.25.0
+tenant-00-worker-02   NotReady   <none>   9s    v1.25.0
+```
+
+The cluster needs a [CNI](https://kubernetes.io/docs/concepts/extend-kubernetes/compute-storage-net/network-plugins/) plugin to get the nodes ready. In this guide, we are going to install [calico](https://projectcalico.docs.tigera.io/about/about-calico), but feel free to use one of your taste.
+
+Download the latest stable Calico manifest:
+
+```bash
+curl https://raw.githubusercontent.com/projectcalico/calico/v3.24.1/manifests/calico.yaml -O
+```
+
+Before to apply the Calico manifest, you can customize it as necessary according to your preferences.
+
+Apply to the tenant cluster:
+
+```bash
+kubectl --kubeconfig=${TENANT_NAMESPACE}-${TENANT_NAME}.kubeconfig apply -f calico.yaml
+```
+
+And after a while, nodes will be ready
+
+```bash
+kubectl --kubeconfig=${TENANT_NAMESPACE}-${TENANT_NAME}.kubeconfig get nodes 
+NAME                  STATUS   ROLES    AGE     VERSION
+tenant-00-worker-00   Ready    <none>   2m48s   v1.25.0
+tenant-00-worker-01   Ready    <none>   2m40s   v1.25.0
+tenant-00-worker-02   Ready    <none>   2m32s   v1.25.0
+```
 
 ## Cleanup
+Remove the worker nodes joined the tenant control plane
 
 ```bash
-$ make -C deploy/kind destroy
+kubectl --kubeconfig=${TENANT_NAMESPACE}-${TENANT_NAME}.kubeconfig delete nodes --all 
 ```
+
+For each worker node, login and clean it
+
+```bash
+HOSTS=(${WORKER0} ${WORKER1} ${WORKER2})
+for i in "${!HOSTS[@]}"; do
+  HOST=${HOSTS[$i]}
+  ssh ${USER}@${HOST} -t 'sudo kubeadm reset -f';
+  ssh ${USER}@${HOST} -t 'sudo rm -rf /etc/cni/net.d';
+  ssh ${USER}@${HOST} -t 'sudo systemctl reboot';
+done
+```
+
+Delete the tenant control plane from kamaji
+
+```bash
+kubectl delete -f ${TENANT_NAMESPACE}-${TENANT_NAME}-tcp.yaml
+```
+
+That's all folks!

docs/content/guides/alternative-datastore.md

@@ -0,0 +1,63 @@
+# Use alternative datastores
+
+Kamaji offers the possibility of having a different storage system than `etcd` thanks to [kine](https://github.com/k3s-io/kine) integration. One of the implementations is [PostgreSQL](https://www.postgresql.org/).
+
+## Install the datastore
+
+On the admin cluster, install one of the alternative supported datastore:
+
+- **MySQL** install it with command:
+
+    `$ make -C deploy/kine/mysql mariadb`
+
+- **PostgreSQL** install it with command:
+
+    `$ make -C deploy/kine/postgresql postgresql`
+
+## Install Cert Manager
+
+As prerequisite for Kamaji, install the Cert Manager
+
+```bash
+helm repo add jetstack https://charts.jetstack.io
+helm repo update
+helm install \
+  cert-manager jetstack/cert-manager \
+  --namespace cert-manager \
+  --create-namespace \
+  --version v1.11.0 \
+  --set installCRDs=true
+```
+
+## Install Kamaji
+
+Use Helm to install the Kamaji Operator and make sure it uses a datastore with the proper driver `datastore.driver=<MySQL|PostgreSQL>`.
+
+For example, with a PostreSQL datastore installed:
+
+```bash
+helm install kamaji charts/kamaji -n kamaji-system --create-namespace \
+  --set etcd.deploy=false \
+  --set datastore.driver=PostgreSQL \
+  --set datastore.endpoints[0]=postgres-default-rw.kamaji-system.svc:5432 \
+  --set datastore.basicAuth.usernameSecret.name=postgres-default-superuser \
+  --set datastore.basicAuth.usernameSecret.namespace=kamaji-system \
+  --set datastore.basicAuth.usernameSecret.keyPath=username \
+  --set datastore.basicAuth.passwordSecret.name=postgres-default-superuser \
+  --set datastore.basicAuth.passwordSecret.namespace=kamaji-system \
+  --set datastore.basicAuth.passwordSecret.keyPath=password \
+  --set datastore.tlsConfig.certificateAuthority.certificate.name=postgres-default-ca \
+  --set datastore.tlsConfig.certificateAuthority.certificate.namespace=kamaji-system \
+  --set datastore.tlsConfig.certificateAuthority.certificate.keyPath=ca.crt \
+  --set datastore.tlsConfig.certificateAuthority.privateKey.name=postgres-default-ca \
+  --set datastore.tlsConfig.certificateAuthority.privateKey.namespace=kamaji-system \
+  --set datastore.tlsConfig.certificateAuthority.privateKey.keyPath=ca.key \
+  --set datastore.tlsConfig.clientCertificate.certificate.name=postgres-default-root-cert \
+  --set datastore.tlsConfig.clientCertificate.certificate.namespace=kamaji-system \
+  --set datastore.tlsConfig.clientCertificate.certificate.keyPath=tls.crt \
+  --set datastore.tlsConfig.clientCertificate.privateKey.name=postgres-default-root-cert \
+  --set datastore.tlsConfig.clientCertificate.privateKey.namespace=kamaji-system \
+  --set datastore.tlsConfig.clientCertificate.privateKey.keyPath=tls.key
+```
+
+Once installed, you will able to create Tenant Control Planes using an alternative datastore.

docs/content/guides/backup-and-restore.md

@@ -0,0 +1,73 @@
+# Backup and restore
+
+As mentioned in the introduction, Kamaji “tenant clusters” are just regular pods scheduled on top of a choosn admin cluster; as such, you can take advantage of the same backup and restore methods that you would use to maintain the standard workload.
+
+This guide will assist you in how to backup and restore TCP resources on the admin cluster using [Velero](https://tanzu.vmware.com/developer/guides/what-is-velero/).
+
+## Prerequisites
+
+Before proceeding with the next steps, we assume that the following prerequisites are met:
+
+- Working admin cluster
+- Working datastore resource
+- Working TCP resource
+- Velero binary installed on the operator VM
+- Velero installed on the admin cluster
+- Configured BackupStorageLocation for Velero
+
+## Backup step
+
+This example shows how to backup and restore a Tenant Control Plane called `tenant-00` and related resources using the `--include-namespaces` tag. Assume the Tenant Control Plane is deployed into the `tenant-00` namespace:
+
+```
+velero backup create tenant-00 --include-namespaces tenant-00
+```
+
+then, verify the backup job status:
+
+```
+velero backup get tenant-00
+NAME        STATUS     ERRORS   WARNINGS   CREATED                         EXPIRES   STORAGE LOCATION   SELECTOR
+tenant-00   Completed  0        0          2023-02-23 17:45:13 +0100 CET   27d       cloudian           <none>
+```
+
+in case of problems, you can get more information by running:
+
+```
+velero backup describe tenant-00
+```
+
+## Restore step
+
+>_WARNING_: this procedure will restore just the TCP resource.
+In the event that the related datastore has been lost, you MUST restore it BEFORE continue; to do this, refer to the backup and restore strategy of the datastore of your choice.
+
+---
+
+To restore just the desired TCP, simply execute:
+
+```
+velero restore create tenant-00 \
+    --from-backup tenant-00 \
+    --include-resources tcp,secret \
+    --status-include-resources tcp
+```
+
+verify the restore job status:
+
+```
+velero restore get
+
+NAME           BACKUP         STATUS      STARTED                         COMPLETED                       ERRORS   WARNINGS   CREATED                         SELECTOR
+tenant-00      tenant-00      Completed   2023-02-24 12:31:39 +0100 CET   2023-02-24 12:31:40 +0100 CET   0        0          2023-02-24 12:31:39 +0100 CET   <none>
+```
+
+In a bunch of seconds, the Kamaji controller will reconcile the TCP and its status will pass from Ready, to NotReady and, finally, Ready again:
+
+```
+kubectl get tcp -A
+
+NAMESPACE   NAME           VERSION   STATUS   CONTROL-PLANE ENDPOINT   KUBECONFIG                      DATASTORE   AGE
+tenant-00   solar-energy   v1.25.6   Ready    192.168.1.251:8443       solar-energy-admin-kubeconfig   dedicated   6m
+[...]
+```

docs/content/guides/datastore-migration.md

@@ -0,0 +1,171 @@
+# Datastore Migration
+
+On the admin cluster, you can deploy one or more multi-tenant datastores as `etcd`, `PostgreSQL`, and `MySQL` to save the state of the tenant clusters. A Tenant Control Plane can be migrated from a datastore to another one without service disruption or without complex and error prone backup & restore procedures.
+
+This guide will assist you to live migrate Tenant's data from a datastore to another one having the same `etcd` driver.
+
+## Prerequisites
+
+Assume you have a Tenant Control Plane using the default datastore:
+
+``` shell
+kubectl get tcp
+NAME        VERSION   STATUS   CONTROL-PLANE ENDPOINT   KUBECONFIG                   DATASTORE   AGE
+tenant-00   v1.25.2   Ready    192.168.32.200:6443      tenant-00-admin-kubeconfig   default     8d
+```
+
+You can check a custom resource called `DataStore` providing a declarative description of the `default` datastore:
+
+```yaml
+apiVersion: kamaji.clastix.io/v1alpha1
+kind: DataStore
+metadata:
+  annotations:
+  labels:
+  name: default
+spec:
+  driver: etcd
+  endpoints:
+  - etcd-0.etcd.kamaji-system.svc.cluster.local:2379
+  - etcd-1.etcd.kamaji-system.svc.cluster.local:2379
+  - etcd-2.etcd.kamaji-system.svc.cluster.local:2379
+  tlsConfig:
+    certificateAuthority:
+      certificate:
+        secretReference:
+          keyPath: ca.crt
+          name: etcd-certs
+          namespace: kamaji-system
+      privateKey:
+        secretReference:
+          keyPath: ca.key
+          name: etcd-certs
+          namespace: kamaji-system
+    clientCertificate:
+      certificate:
+        secretReference:
+          keyPath: tls.crt
+          name: etcd-root-client-certs
+          namespace: kamaji-system
+      privateKey:
+        secretReference:
+          keyPath: tls.key
+          name: etcd-root-client-certs
+          namespace: kamaji-system
+status:
+  usedBy:
+  - default/tenant-00
+```
+
+The `default` datastore is installed by Kamaji Helm chart in the same namespace hosting the controller:
+
+```shell
+kubectl -n kamaji-system get pods
+NAME                              READY   STATUS      RESTARTS   AGE
+etcd-0                            1/1     Running     0          23d
+etcd-1                            1/1     Running     0          23d
+etcd-2                            1/1     Running     0          23d
+kamaji-5d6cdfbbb9-bn27f           1/1     Running     0          2d19h
+```
+
+## Install a new datastore
+A managed datastore is highly recommended in production. The [kamaji-etcd](https://github.com/clastix/kamaji-etcd) project provides a viable option to setup a managed multi-tenant `etcd` running as StatefulSet made of three replicas:
+
+```bash
+helm repo add clastix https://clastix.github.io/charts
+helm repo update
+helm install dedicated clastix/kamaji-etcd -n dedicated --create-namespace --set datastore.enabled=true
+```
+
+You should end up with a new datastore `dedicated` provided by an `etcd` cluster:
+
+```yaml
+kubectl get datastore dedicated -o yaml
+apiVersion: kamaji.clastix.io/v1alpha1
+kind: DataStore
+metadata:
+  annotations:
+  labels:
+  name: dedicated
+spec:
+  driver: etcd
+  endpoints:
+  - dedicated-0.dedicated.dedicated.svc.cluster.local:2379
+  - dedicated-1.dedicated.dedicated.svc.cluster.local:2379
+  - dedicated-2.dedicated.dedicated.svc.cluster.local:2379
+  tlsConfig:
+    certificateAuthority:
+      certificate:
+        secretReference:
+          keyPath: ca.crt
+          name: dedicated-certs
+          namespace: dedicated
+      privateKey:
+        secretReference:
+          keyPath: ca.key
+          name: dedicated-certs
+          namespace: dedicated
+    clientCertificate:
+      certificate:
+        secretReference:
+          keyPath: tls.crt
+          name: dedicated-root-client-certs
+          namespace: dedicated
+      privateKey:
+        secretReference:
+          keyPath: tls.key
+          name: dedicated-root-client-certs
+          namespace: dedicated
+status: {}
+```
+
+Check the `etcd` cluster:
+
+```bash
+kubectl -n dedicated get sts,pods,pvc
+NAME                         READY   AGE
+statefulset.apps/dedicated   3/3     25h
+
+NAME                                  READY   STATUS      RESTARTS   AGE
+pod/dedicated-0                       1/1     Running     0          25h
+pod/dedicated-1                       1/1     Running     0          25h
+pod/dedicated-2                       1/1     Running     0          25h
+
+NAME                                     STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS   AGE
+persistentvolumeclaim/data-dedicated-0   Bound    pvc-a5c66737-ef78-4689-b863-037f8382ed78   10Gi       RWO            local-path     25h
+persistentvolumeclaim/data-dedicated-1   Bound    pvc-1e9f77eb-89f3-4256-9508-c18b71fca7ea   10Gi       RWO            local-path     25h
+persistentvolumeclaim/data-dedicated-2   Bound    pvc-957c4802-1e7c-4f37-ac01-b89ad1fa9fdb   10Gi       RWO            local-path     25h
+```
+
+## Migrate data
+To migrate data from current `default` datastore to the new dedicated one, patch the Tenant Control Plane `tenant-00` to use the new `dedicated` datastore:
+
+```shell
+kubectl patch --type merge tcp tenant-00 -p '{"spec": {"dataStore": "dedicated"}}'
+```
+
+and check the process happening in real time:
+
+```shell
+kubectl get tcp -w
+NAME        VERSION   STATUS      CONTROL-PLANE ENDPOINT   KUBECONFIG                   DATASTORE   AGE
+tenant-00   v1.25.2   Ready       192.168.32.200:6443      tenant-00-admin-kubeconfig   default     9d
+tenant-00   v1.25.2   Migrating   192.168.32.200:6443      tenant-00-admin-kubeconfig   default     9d
+tenant-00   v1.25.2   Migrating   192.168.32.200:6443      tenant-00-admin-kubeconfig   default     9d
+tenant-00   v1.25.2   Migrating   192.168.32.200:6443      tenant-00-admin-kubeconfig   dedicated   9d
+tenant-00   v1.25.2   Migrating   192.168.32.200:6443      tenant-00-admin-kubeconfig   dedicated   9d
+tenant-00   v1.25.2   Ready       192.168.32.200:6443      tenant-00-admin-kubeconfig   dedicated   9d
+```
+
+During the datastore migration, the Tenant Control Plane is put in read-only mode to avoid misalignments between source and destination datastores. If tenant users try to update the data, an admission controller denies the request with the following message:
+
+
+```shell
+Error from server (the current Control Plane is in freezing mode due to a maintenance mode,
+all the changes are blocked: removing the webhook may lead to an inconsistent state upon its completion):
+admission webhook "catchall.migrate.kamaji.clastix.io" denied the request
+```
+
+After a while, depending on the amount of data to migrate, the Tenant Control Plane is put back in full operating mode by the Kamaji controller.
+
+> Please, note the datastore migration leaves the data on the default datastore, so you have to remove it manually.

docs/content/guides/kamaji-azure-deployment.md

@@ -1,32 +1,33 @@
 # Setup Kamaji on Azure
 This guide will lead you through the process of creating a working Kamaji setup on on MS Azure.
 
-The material here is relatively dense. We strongly encourage you to dedicate time to walk through these instructions, with a mind to learning. We do NOT provide any "one-click" deployment here. However, once you've understood the components involved it is encouraged that you build suitable, auditable GitOps deployment processes around your final infrastructure.
+!!! warning ""
+    The material here is relatively dense. We strongly encourage you to dedicate time to walk through these instructions, with a mind to learning. We do NOT provide any "one-click" deployment here. However, once you've understood the components involved it is encouraged that you build suitable, auditable GitOps deployment processes around your final infrastructure.
 
 The guide requires:
 
-- one bootstrap workstation
-- an AKS Kubernetes cluster to run the Admin and Tenant Control Planes
-- an arbitrary number of Azure virtual machines to host `Tenant`s' workloads
+- a bootstrap machine
+- a Kubernetes cluster to run the Admin and Tenant Control Planes
+- an arbitrary number of machines to host `Tenant`s' workloads
 
 ## Summary
 
   * [Prepare the bootstrap workspace](#prepare-the-bootstrap-workspace)
   * [Access Admin cluster](#access-admin-cluster)
-  * [Install DataStore](#install-datastore)
+  * [Install Cert Manager](#install-cert-manager)
   * [Install Kamaji controller](#install-kamaji-controller)
   * [Create Tenant Cluster](#create-tenant-cluster)
   * [Cleanup](#cleanup)
 
 ## Prepare the bootstrap workspace
-This guide is supposed to be run from a remote or local bootstrap machine. First, clone the repo and prepare the workspace directory:
+On the bootstrap machine, clone the repo and prepare the workspace directory:
 
 ```bash
 git clone https://github.com/clastix/kamaji
 cd kamaji/deploy
 ```
 
-We assume you have installed on your workstation:
+We assume you have installed on the bootstrap machine:
 
 - [kubectl](https://kubernetes.io/docs/tasks/tools/#kubectl)
 - [kubeadm](https://kubernetes.io/docs/tasks/tools/#kubeadm)
@@ -40,10 +41,10 @@ Make sure you have a valid Azure subscription, and login to Azure:
 az account set --subscription "MySubscription"
 az login
 ```
-> Currently, the Kamaji setup, including Admin and Tenant clusters need to be deployed within the same Azure region. Cross-regions deployments are not supported.
+
 
 ## Access Admin cluster
-In Kamaji, an Admin Cluster is a regular Kubernetes cluster which hosts zero to many Tenant Cluster Control Planes. The admin cluster acts as management cluster for all the Tenant clusters and implements Monitoring, Logging, and Governance of all the Kamaji setup, including all Tenant clusters. For this guide, we're going to use an instance of Azure Kubernetes Service - AKS as the Admin Cluster.
+In Kamaji, an Admin Cluster is a regular Kubernetes cluster which hosts zero to many Tenant Cluster Control Planes. The admin cluster acts as management cluster for all the Tenant clusters and implements Monitoring, Logging, and Governance of all the Kamaji setup, including all Tenant clusters. For this guide, we're going to use an instance of Azure Kubernetes Service (AKS) as Admin Cluster.
 
 Throughout the following instructions, shell variables are used to indicate values that you should adjust to your own Azure environment:
 
@@ -96,21 +97,26 @@ And check you can access:
 kubectl cluster-info
 ```
 
-## Install datastore
-The Kamaji controller needs to access a multi-tenant datastore in order to save data of the tenants' clusters. The Kamaji Helm Chart provides the installation of an unamanaged `etcd`. However, a managed `etcd` is highly recommended in production.
+## Install Cert Manager
 
-As alternative, the [kamaji-etcd](https://github.com/clastix/kamaji-etcd) project provides a viable option to setup a manged multi-tenant `etcd` as 3 replicas StatefulSet with data persistence:
+Kamaji takes advantage of the [dynamic admission control](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/), such as validating and mutating webhook configurations. These webhooks are secured by a TLS communication, and the certificates are managed by [`cert-manager`](https://cert-manager.io/), making it a prerequisite that must be installed:
 
 ```bash
-helm repo add clastix https://clastix.github.io/charts
+helm repo add jetstack https://charts.jetstack.io
 helm repo update
-helm install etcd clastix/kamaji-etcd -n kamaji-system --create-namespace
+helm install \
+  cert-manager jetstack/cert-manager \
+  --namespace cert-manager \
+  --create-namespace \
+  --version v1.11.0 \
+  --set installCRDs=true
 ```
 
-Optionally, Kamaji offers the possibility of using a different storage system for the tenants' clusters, as MySQL or PostgreSQL compatible database, thanks to the native [kine](https://github.com/k3s-io/kine) integration.
-
 ## Install Kamaji Controller
-Install Kamaji with `helm` using an unmanaged `etcd` as datastore:
+
+Installing Kamaji via Helm charts is the preferred way. The Kamaji controller needs to access a Datastore in order to save data of the tenants' clusters. The Kamaji Helm Chart provides the installation of a basic unamanaged `etcd` as datastore, out of box. 
+
+Install Kamaji with `helm` using an unmanaged `etcd` as default datastore:
 
 ```bash
 helm repo add clastix https://clastix.github.io/charts
@@ -118,15 +124,8 @@ helm repo update
 helm install kamaji clastix/kamaji -n kamaji-system --create-namespace
 ```
 
-Alternatively, if you opted for a managed `etcd` datastore:
-
-```
-helm repo add clastix https://clastix.github.io/charts
-helm repo update
-helm install kamaji clastix/kamaji -n kamaji-system --create-namespace --set etcd.deploy=false    
-```
-
-Congratulations! You just turned your Azure Kubernetes AKS cluster into a Kamaji cluster capable to run multiple Tenant Control Planes.
+!!! note "A managed datastore is highly recommended in production"
+     The [kamaji-etcd](https://github.com/clastix/kamaji-etcd) project provides the code to setup a multi-tenant `etcd` running as StatefulSet made of three replicas. Optionally, Kamaji offers support for a more robust storage system, as `MySQL` or `PostgreSQL` compatible database, thanks to the native [kine](https://github.com/k3s-io/kine) integration.
 
 ## Create Tenant Cluster
 
@@ -146,6 +145,7 @@ metadata:
   name: ${TENANT_NAME}
   namespace: ${TENANT_NAMESPACE}
 spec:
+  dataStore: default
   controlPlane:
     deployment:
       replicas: 3
@@ -171,7 +171,7 @@ spec:
           requests:
             cpu: 125m
             memory: 256Mi
-          limits: {} 
+          limits: {}
     service:
       additionalMetadata:
         labels:
@@ -219,7 +219,7 @@ spec:
     protocol: TCP
     targetPort: ${TENANT_PORT}
   selector:
-    kamaji.clastix.io/soot: ${TENANT_NAME}
+    kamaji.clastix.io/name: ${TENANT_NAME}
   type: LoadBalancer
 EOF
 
@@ -273,7 +273,12 @@ kubernetes   10.240.0.100:6443   57m
 ```
 
 ### Prepare worker nodes to join
-Currently Kamaji does not provide any helper for creation of tenant worker nodes. You should get a set of machines from your infrastructure provider, turn them into worker nodes, and then join to the tenant control plane with the `kubeadm`. In the future, we'll provide integration with Cluster APIs and other tools, as for example, Terrform.
+
+Currently, Kamaji does not provide any helper for creation of tenant worker nodes.
+You should get a set of machines from your infrastructure provider, turn them into worker nodes, and then join to the tenant control plane with the `kubeadm`. 
+
+Kamaji is sticking to the [Cluster Management API](https://github.com/kubernetes-sigs/cluster-api) project contracts by providing a `ControlPlane` provider.
+An Azure-based cluster is not yet available: the available road-map is available on the [official repository](https://github.com/clastix/cluster-api-control-plane-provider-kamaji).
 
 Create an Azure VM Stateful Set to host worker nodes
 

docs/content/guides/kamaji-deployment-guide.md

@@ -1,334 +0,0 @@
-# Setup Kamaji on a generic infrastructure
-This guide will lead you through the process of creating a working Kamaji setup on a generic infrastructure, either virtual or bare metal.
-
-The material here is relatively dense. We strongly encourage you to dedicate time to walk through these instructions, with a mind to learning. We do NOT provide any "one-click" deployment here. However, once you've understood the components involved it is encouraged that you build suitable, auditable GitOps deployment processes around your final infrastructure.
-
-The guide requires:
-
-- one bootstrap workstation
-- a Kubernetes cluster to run the Admin and Tenant Control Planes
-- an arbitrary number of machines to host `Tenant`s' workloads
-
-## Summary
-
-  * [Prepare the bootstrap workspace](#prepare-the-bootstrap-workspace)
-  * [Access Admin cluster](#access-admin-cluster)
-  * [Install DataStore](#install-datastore)
-  * [Install Kamaji controller](#install-kamaji-controller)
-  * [Create Tenant Cluster](#create-tenant-cluster)
-  * [Cleanup](#cleanup)
-
-## Prepare the bootstrap workspace
-This guide is supposed to be run from a remote or local bootstrap machine. First, clone the repo and prepare the workspace directory:
-
-```bash
-git clone https://github.com/clastix/kamaji
-cd kamaji/deploy
-```
-
-We assume you have installed on your workstation:
-
-- [kubectl](https://kubernetes.io/docs/tasks/tools/#kubectl)
-- [kubeadm](https://kubernetes.io/docs/tasks/tools/#kubeadm)
-- [helm](https://helm.sh/docs/intro/install/)
-- [jq](https://stedolan.github.io/jq/)
-
-## Access Admin cluster
-In Kamaji, an Admin Cluster is a regular Kubernetes cluster which hosts zero to many Tenant Cluster Control Planes. The admin cluster acts as management cluster for all the Tenant clusters and implements Monitoring, Logging, and Governance of all the Kamaji setup, including all Tenant clusters. 
-
-Throughout the following instructions, shell variables are used to indicate values that you should adjust to your environment:
-
-```bash
-source kamaji.env
-```
-
-Any regular and conformant Kubernetes v1.22+ cluster can be turned into a Kamaji setup. To work properly, the admin cluster should provide at least:
-
-- CNI module installed, eg. [Calico](https://github.com/projectcalico/calico), [Cilium](https://github.com/cilium/cilium).
-- CSI module installed with a Storage Class for the Tenants' `etcd`. Local Persistent Volumes are an option.
-- Support for LoadBalancer Service Type, or alternatively, an Ingress Controller, eg. [ingress-nginx](https://github.com/kubernetes/ingress-nginx), [haproxy](https://github.com/haproxytech/kubernetes-ingress).
-- Monitoring Stack, eg. [Prometheus](https://github.com/prometheus-community).
-
-Make sure you have a `kubeconfig` file with admin permissions on the cluster you want to turn into Kamaji Admin Cluster.
-
-## Install datastore
-The Kamaji controller needs to access a multi-tenant datastore in order to save data of the tenants' clusters. The Kamaji Helm Chart provides the installation of an unamanaged `etcd`. However, a managed `etcd` is highly recommended in production.
-
-As alternative, the [kamaji-etcd](https://github.com/clastix/kamaji-etcd) project provides a viable option to setup a manged multi-tenant `etcd` as 3 replicas StatefulSet with data persistence:
-
-```bash
-helm repo add clastix https://clastix.github.io/charts
-helm repo update
-helm install etcd clastix/kamaji-etcd -n kamaji-system --create-namespace
-```
-
-Optionally, Kamaji offers the possibility of using a different storage system for the tenants' clusters, as MySQL or PostgreSQL compatible database, thanks to the native [kine](https://github.com/k3s-io/kine) integration.
-
-## Install Kamaji Controller
-Install Kamaji with `helm` using an unmanaged `etcd` as datastore:
-
-```bash
-helm repo add clastix https://clastix.github.io/charts
-helm repo update
-helm install kamaji clastix/kamaji -n kamaji-system --create-namespace
-```
-
-Alternatively, if you opted for a managed `etcd` datastore:
-
-```bash
-helm repo add clastix https://clastix.github.io/charts
-helm repo update
-helm install kamaji clastix/kamaji -n kamaji-system --create-namespace --set etcd.deploy=false    
-```
-
-Congratulations! You just turned your Kubernetes cluster into a Kamaji cluster capable to run multiple Tenant Control Planes.
-
-## Create Tenant Cluster
-
-### Tenant Control Plane
-
-A tenant control plane of example looks like:
-
-```yaml
-cat > ${TENANT_NAMESPACE}-${TENANT_NAME}-tcp.yaml <<EOF
-apiVersion: kamaji.clastix.io/v1alpha1
-kind: TenantControlPlane
-metadata:
-  name: ${TENANT_NAME}
-  namespace: ${TENANT_NAMESPACE}
-spec:
-  controlPlane:
-    deployment:
-      replicas: 3
-      additionalMetadata:
-        labels:
-          tenant.clastix.io: ${TENANT_NAME}
-      extraArgs:
-        apiServer: []
-        controllerManager: []
-        scheduler: []
-      resources:
-        apiServer:
-          requests:
-            cpu: 250m
-            memory: 512Mi
-          limits: {}
-        controllerManager:
-          requests:
-            cpu: 125m
-            memory: 256Mi
-          limits: {}
-        scheduler:
-          requests:
-            cpu: 125m
-            memory: 256Mi
-          limits: {}
-    service:
-      additionalMetadata:
-        labels:
-          tenant.clastix.io: ${TENANT_NAME}
-      serviceType: LoadBalancer
-  kubernetes:
-    version: ${TENANT_VERSION}
-    kubelet:
-      cgroupfs: systemd
-    admissionControllers:
-      - ResourceQuota
-      - LimitRanger
-  networkProfile:
-    port: ${TENANT_PORT}
-    certSANs:
-    - ${TENANT_NAME}.${TENANT_DOMAIN}
-    serviceCidr: ${TENANT_SVC_CIDR}
-    podCidr: ${TENANT_POD_CIDR}
-    dnsServiceIPs:
-    - ${TENANT_DNS_SERVICE}
-  addons:
-    coreDNS: {}
-    kubeProxy: {}
-    konnectivity:
-      server:
-        port: ${TENANT_PROXY_PORT}
-        resources:
-          requests:
-            cpu: 100m
-            memory: 128Mi
-          limits: {}
-EOF
-
-kubectl -n ${TENANT_NAMESPACE} apply -f ${TENANT_NAMESPACE}-${TENANT_NAME}-tcp.yaml
-```
-
-After a few minutes, check the created resources in the tenants namespace and when ready it will look similar to the following:
-
-```command
-kubectl -n tenants get tcp,deploy,pods,svc
-NAME                                             VERSION   STATUS   CONTROL-PLANE-ENDPOINT   KUBECONFIG                   AGE
-tenantcontrolplane.kamaji.clastix.io/tenant-00   v1.23.1   Ready    192.168.32.240:6443      tenant-00-admin-kubeconfig   2m20s
-
-NAME                        READY   UP-TO-DATE   AVAILABLE   AGE
-deployment.apps/tenant-00   3/3     3            3           118s
-
-NAME                             READY   STATUS    RESTARTS   AGE
-pod/tenant-00-58847c8cdd-7hc4n   4/4     Running   0          82s
-pod/tenant-00-58847c8cdd-ft5xt   4/4     Running   0          82s
-pod/tenant-00-58847c8cdd-shc7t   4/4     Running   0          82s
-
-NAME                TYPE           CLUSTER-IP      EXTERNAL-IP      PORT(S)                         AGE
-service/tenant-00   LoadBalancer   10.32.132.241   192.168.32.240   6443:32152/TCP,8132:32713/TCP   2m20s
-```
-
-The regular Tenant Control Plane containers: `kube-apiserver`, `kube-controller-manager`, `kube-scheduler` are running unchanged in the `tcp` pods instead of dedicated machines and they are exposed through a service on the port `6443` of worker nodes in the Admin cluster.
-
-The `LoadBalancer` service type is used to expose the Tenant Control Plane. However, `NodePort` and `ClusterIP` with an Ingress Controller are still viable options, depending on the case. High Availability and rolling updates of the Tenant Control Plane are provided by the `tcp` Deployment and all the resources reconcilied by the Kamaji controller.
-
-### Working with Tenant Control Plane
-
-Collect the external IP address of the `tcp` service:
-
-```bash
-TENANT_ADDR=$(kubectl -n ${TENANT_NAMESPACE} get svc ${TENANT_NAME} -o json | jq -r ."spec.loadBalancerIP")
-```
-
-and check it out:
-
-```bash
-curl -k https://${TENANT_ADDR}:${TENANT_PORT}/healthz
-curl -k https://${TENANT_ADDR}:${TENANT_PORT}/version
-```
-
-The `kubeconfig` required to access the Tenant Control Plane is stored in a secret:
-
-```bash
-kubectl get secrets -n ${TENANT_NAMESPACE} ${TENANT_NAME}-admin-kubeconfig -o json \
-  | jq -r '.data["admin.conf"]' \
-  | base64 --decode \
-  > ${TENANT_NAMESPACE}-${TENANT_NAME}.kubeconfig
-```
-
-and let's check it out:
-
-```bash
-kubectl --kubeconfig=${TENANT_NAMESPACE}-${TENANT_NAME}.kubeconfig cluster-info
-
-Kubernetes control plane is running at https://192.168.32.240:6443
-CoreDNS is running at https://192.168.32.240:6443/api/v1/namespaces/kube-system/services/kube-dns:dns/proxy
-```
-
-Check out how the Tenant control Plane advertises itself to workloads:
-
-```bash
-kubectl --kubeconfig=${TENANT_NAMESPACE}-${TENANT_NAME}.kubeconfig get svc
-
-NAMESPACE     NAME         TYPE        CLUSTER-IP   EXTERNAL-IP   PORT(S)   AGE
-default       kubernetes   ClusterIP   10.32.0.1    <none>        443/TCP   6m
-```
-
-```bash
-kubectl --kubeconfig=${TENANT_NAMESPACE}-${TENANT_NAME}.kubeconfig get ep
-
-NAME         ENDPOINTS             AGE
-kubernetes   192.168.32.240:6443   18m
-```
-
-And make sure it is `${TENANT_ADDR}:${TENANT_PORT}`.
-
-### Prepare worker nodes to join
-
-Currently Kamaji does not provide any helper for creation of tenant worker nodes. You should get a set of machines from your infrastructure provider, turn them into worker nodes, and then join to the tenant control plane with the `kubeadm`. In the future, we'll provide integration with Cluster APIs and other tools, as for example, Terraform.
-
-You can use the provided helper script `/deploy/nodes-prerequisites.sh`, in order to install the dependencies on all the worker nodes:
-
-- Install `containerd` as container runtime
-- Install `crictl`, the command line for working with `containerd`
-- Install `kubectl`, `kubelet`, and `kubeadm` in the desired version
-
-> Warning: the script assumes all worker nodes are running `Ubuntu 20.04`. Make sure to adapt the script if you're using a different distribution.
-
-Run the script:
-
-```bash
-HOSTS=(${WORKER0} ${WORKER1} ${WORKER2})
-./nodes-prerequisites.sh ${TENANT_VERSION:1} ${HOSTS[@]}
-```
-
-### Join worker nodes
-The current approach for joining nodes is to use `kubeadm` and therefore, we will create a bootstrap token to perform the action. In order to facilitate the step, we will store the entire command of joining in a variable:
-
-```bash
-JOIN_CMD=$(echo "sudo ")$(kubeadm --kubeconfig=${TENANT_NAMESPACE}-${TENANT_NAME}.kubeconfig token create --print-join-command)
-```
-
-A bash loop will be used to join all the available nodes.
-
-```bash
-HOSTS=(${WORKER0} ${WORKER1} ${WORKER2})
-for i in "${!HOSTS[@]}"; do
-  HOST=${HOSTS[$i]}
-  ssh ${USER}@${HOST} -t ${JOIN_CMD};
-done
-```
-
-Checking the nodes:
-
-```bash
-kubectl --kubeconfig=${TENANT_NAMESPACE}-${TENANT_NAME}.kubeconfig get nodes 
-
-NAME                  STATUS     ROLES    AGE   VERSION
-tenant-00-worker-00   NotReady   <none>   25s   v1.25.0
-tenant-00-worker-01   NotReady   <none>   17s   v1.25.0
-tenant-00-worker-02   NotReady   <none>   9s    v1.25.0
-```
-
-The cluster needs a [CNI](https://kubernetes.io/docs/concepts/extend-kubernetes/compute-storage-net/network-plugins/) plugin to get the nodes ready. In this guide, we are going to install [calico](https://projectcalico.docs.tigera.io/about/about-calico), but feel free to use one of your taste.
-
-Download the latest stable Calico manifest:
-
-```bash
-curl https://raw.githubusercontent.com/projectcalico/calico/v3.24.1/manifests/calico.yaml -O
-```
-
-Before to apply the Calico manifest, you can customize it as necessary according to your preferences.
-
-Apply to the tenant cluster:
-
-```bash
-kubectl --kubeconfig=${TENANT_NAMESPACE}-${TENANT_NAME}.kubeconfig apply -f calico.yaml
-```
-
-And after a while, nodes will be ready
-
-```bash
-kubectl --kubeconfig=${TENANT_NAMESPACE}-${TENANT_NAME}.kubeconfig get nodes 
-NAME                  STATUS   ROLES    AGE     VERSION
-tenant-00-worker-00   Ready    <none>   2m48s   v1.25.0
-tenant-00-worker-01   Ready    <none>   2m40s   v1.25.0
-tenant-00-worker-02   Ready    <none>   2m32s   v1.25.0
-```
-
-## Cleanup
-Remove the worker nodes joined the tenant control plane
-
-```bash
-kubectl --kubeconfig=${TENANT_NAMESPACE}-${TENANT_NAME}.kubeconfig delete nodes --all 
-```
-
-For each worker node, login and clean it
-
-```bash
-HOSTS=(${WORKER0} ${WORKER1} ${WORKER2})
-for i in "${!HOSTS[@]}"; do
-  HOST=${HOSTS[$i]}
-  ssh ${USER}@${HOST} -t 'sudo kubeadm reset -f';
-  ssh ${USER}@${HOST} -t 'sudo rm -rf /etc/cni/net.d';
-  ssh ${USER}@${HOST} -t 'sudo systemctl reboot';
-done
-```
-
-Delete the tenant control plane from kamaji
-
-```bash
-kubectl delete -f ${TENANT_NAMESPACE}-${TENANT_NAME}-tcp.yaml
-```
-
-That's all folks!

docs/content/guides/kamaji-gitops-flux.md

@@ -1,20 +1,16 @@
 # Manage tenant resources GitOps-way from the admin cluster
 
-In this guide, you can learn how to apply applications and resources in general, the GitOps-way, to the Tenant Control Planes.
+This guide describe a declarative way to deploy Kubernetes add-ons across multiple Tenant Clusters, the GitOps-way. An admin may need to apply a specific workload into Tenant Clusters and ensure is constantly reconciled, no matter what the tenants will do in their clusters. Examples include installing monitoring agents, ensuring specific policies, installing infrastructure operators like Cert Manager and so on.
 
-An admin may need to apply a specific workload into tenant control planes and ensure is constantly reconciled, no matter what the tenants will do in their clusters.
-
-Examples include installing monitoring agents, ensuring specific policies, installing infrastructure operators like Cert Manager and so on.
+This way the tenant resources can be ensured from a single pane of glass, from the *admin cluster*.
 
 ## Flux as the GitOps operator
 
-As GitOps ensures a constant reconciliation to a Git-versioned desired state, Flux can satisfy the requirement of those scenarios.
-
-In particular, the controllers that reconcile [resources](https://fluxcd.io/flux/concepts/#reconciliation) support communicating to external clusters.
+As GitOps ensures a constant reconciliation to a Git-versioned desired state, [Flux](https://fluxcd.io) can satisfy the requirement of those scenarios. In particular, the controllers that reconcile [resources](https://fluxcd.io/flux/concepts/#reconciliation) support communicating to external clusters.
 
 In this scenario the Flux toolkit would run in the *admin cluster*, with reconcile controllers reconciling resources into *tenant clusters*.
 
-<img src="../images/kamaji-flux.png" alt="kamaji-flux" width="720"/>
+![Architecture](../images/kamaji-flux.png)
 
 This is something possible as the Flux reconciliation Custom Resources specifications provide ability to specify `Secret` which contain a `kubeconfig` - here you can find the related documentation for both [`Kustomization`](https://fluxcd.io/flux/components/kustomize/kustomization/#remote-clusters--cluster-api) and [`HelmRelease`](https://fluxcd.io/flux/components/helm/helmreleases/#remote-clusters--cluster-api) CRs.
 
@@ -86,10 +82,6 @@ tenant1-cert-manager-cainjector   1/1     1            1           4m3s
 tenant1-cert-manager-webhook      1/1     1            1           4m3s
 ```
 
-## Conclusion
-
-This way tenant resources can be ensured from a single pane of glass, from the *admin cluster*.
-
 No matter what the tenant users will do on the *tenant cluster*, the Flux reconciliation controllers wirunning in the *admin cluster* will ensure the desired state declared by the reconciliation resources applied existing in the *admin cluster*, will be reconciled in the *tenant cluster*.
 
 Furthermore, this approach does not need to have in each tenant cluster nor Flux neither applied the related reconciliation Custom Resorces.

docs/content/guides/mysql-datastore.md

@@ -1,5 +0,0 @@
-# MySQL as Kubernetes Storage
-
-Kamaji offers the possibility of having a different storage system than `ETCD` thanks to [kine](https://github.com/k3s-io/kine) integration. One of the implementations is [MySQL](https://www.mysql.com/).
-
-> A detailed guide for production setup will be released soon. Please refer to [Getting Started Guide](../getting-started.md) for a demo setup with KinD.

docs/content/guides/postgresql-datastore.md

@@ -1,6 +0,0 @@
-# PostgreSQL as Kubernetes Storage
-
-Kamaji offers the possibility of having a different storage system than `etcd` thanks to [kine](https://github.com/k3s-io/kine) integration.
-One of the implementations is [PostgreSQL](https://www.postgresql.org/).
-
-> A detailed guide for production setup will be released soon. Please refer to [Getting Started Guide](../getting-started.md) for a demo setup with KinD.

docs/content/guides/upgrade.md

@@ -7,7 +7,29 @@ The process of upgrading a _“tenant cluster”_ consists in two steps:
 ## Upgrade of Tenant Control Plane
 You should patch the `TenantControlPlane.spec.kubernetes.version` custom resource with a new compatible value according to the [Version Skew Policy](https://kubernetes.io/releases/version-skew-policy/).
 
-> Note: during the upgrade, a new ReplicaSet of Tenant Control Plane pod will be created, so make sure you have at least two pods to avoid service disruption.
+During the upgrade, a new ReplicaSet of Tenant Control Plane pod will be created, so make sure you have enough replicas to avoid service disruption. Also make sure you have the Rolling Update strategy properly configured:
+
+```yaml
+apiVersion: kamaji.clastix.io/v1alpha1
+kind: TenantControlPlane
+metadata:
+  name: tenant-00
+spec:
+  controlPlane:
+    deployment:
+      replicas: 3
+      strategy:
+        rollingUpdate:
+          maxSurge: 1
+          maxUnavailable: 1
+        type: RollingUpdate
+...
+```
 
 ## Upgrade of Tenant Worker Nodes
-As currently Kamaji is not providing any helpers for Tenant Worker Nodes, you should make sure to upgrade them manually, for example, with the help of `kubeadm`. We have in roadmap, the Cluster APIs support so that you can upgrade _“tenant clusters”_ in a fully declarative way.
+
+As currently Kamaji is not providing any helpers for Tenant Worker Nodes, you should make sure to upgrade them manually, for example, with the help of `kubeadm`.
+Refer to the official [documentation](https://kubernetes.io/docs/tasks/administer-cluster/kubeadm/kubeadm-upgrade/#upgrade-worker-nodes).
+
+Kamaji is offering a [Cluster API Control Plane provider](https://github.com/clastix/cluster-api-control-plane-provider-kamaji), thus integrating with the Kubernetes clusters declarative management approach.
+You can refer to the official [Cluster API documentation](https://cluster-api.sigs.k8s.io/).

docs/content/index.md

@@ -1,51 +1,20 @@
 # Kamaji
+
 **Kamaji** deploys and operates Kubernetes at scale with a fraction of the operational burden. 
 
 ## How it works
-Kamaji turns any Kubernetes cluster into an _“admin cluster”_ to orchestrate other Kubernetes clusters called _“tenant clusters”_. What makes Kamaji special is that Control Planes of _“tenant clusters”_ are just regular pods running in the _“admin cluster”_ instead of dedicated Virtual Machines. This solution makes running control planes at scale cheaper and easier to deploy and operate. View [Concepts](concepts.md) for a deeper understanding of principles behind Kamaji's design.
+Kamaji turns any Kubernetes cluster into an _“admin cluster”_ to orchestrate other Kubernetes clusters called _“tenant clusters”_. Kamaji is special because the Control Plane components are running in a single pod instead of dedicated machines. This solution makes running multiple Control Planes cheaper and easier to deploy and operate. 
 
-![Architecture](images/kamaji-light.png#gh-light-mode-only)
-![Architecture](images/kamaji-dark.png#gh-dark-mode-only)
+<img src="images/architecture.png"  width="600">
 
-All the tenant clusters built with Kamaji are fully compliant [CNCF Certified Kubernetes](https://www.cncf.io/certification/software-conformance/) and are compatible with the standard toolchains everybody knows and loves.
+View [Concepts](concepts.md) for a deeper understanding of principles behind Kamaji's design.
 
-<p align="center" style="padding: 6px 6px">
-  <img src="https://raw.githubusercontent.com/cncf/artwork/master/projects/kubernetes/certified-kubernetes/versionless/color/certified-kubernetes-color.png" width="200" />
-</p>
-
-## Features
-
-- **Self Service Kubernetes:** leave users the freedom to self-provision their Kubernetes clusters according to the assigned boundaries.
-- **Multi-cluster Management:** centrally manage multiple tenant clusters from a single admin cluster. Happy SREs. 
-- **Cheaper Control Planes:** place multiple tenant control planes on a single node, instead of having three nodes for a single control plane.
-- **Stronger Multi-Tenancy:** leave tenants to access the control plane with admin permissions while keeping the tenant isolated at the infrastructure level.
-- **Kubernetes Inception:** use Kubernetes to manage Kubernetes by re-using all the Kubernetes goodies you already know and love.
-- **Full APIs compliant:** tenant clusters are fully CNCF compliant built with upstream Kubernetes binaries. A user does not see differences between a Kamaji provisioned cluster and a dedicated cluster.
+!!! info "CNCF Compliance"
+    All the tenant clusters built with Kamaji are fully compliant [CNCF Certified Kubernetes](https://www.cncf.io/certification/software-conformance/) and are compatible with the standard toolchains everybody knows and loves.
 
 ## Getting started
 
-Please refer to the [Getting Started guide](getting-started.md) to deploy a minimal setup of Kamaji on [KinD](https://kind.sigs.k8s.io/).
+Please refer to the [Getting Started guide](getting-started.md) to deploy a minimal setup of Kamaji.
 
-## Open Source
-Kamaji is Open Source with Apache 2 license and any contribution is welcome. Open an issue or suggest an enhancement on the GitHub [project's page](https://github.com/clastix/kamaji). Join the [Kubernetes Slack Workspace](https://slack.k8s.io/) and the [`#kamaji`](https://kubernetes.slack.com/archives/C03GLTTMWNN) channel to meet end-users and contributors.
 
-## FAQs
-Q. What does Kamaji mean?
 
-A. Kamaji is named as the character _Kamaji_ from the Japanese movie [_Spirited Away_](https://en.wikipedia.org/wiki/Spirited_Away).
-
-Q. Is Kamaji another Kubernetes distribution?
-
-A. No, Kamaji is a Kubernetes Operator you can install on top of any Kubernetes cluster to provide hundreds or thousands of managed Kubernetes clusters as a service. We tested Kamaji on vanilla Kubernetes 1.22+, KinD, and Azure AKS. We expect it to work smoothly on other Kubernetes distributions. The tenant clusters made with Kamaji are conformant CNCF Kubernetes clusters as we leverage [`kubeadm`](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/).
-
-Q. Is it safe to run Kubernetes control plane components in a pod instead of dedicated virtual machines?
-
-A. Yes, the tenant control plane components are packaged in the same way they are running in bare metal or virtual nodes. We leverage the `kubeadm` code to set up the control plane components as they were running on their own server. The unchanged images of upstream `kube-apiserver`, `kube-scheduler`, and `kube-controller-manager` are used.
-
-Q. You already provide a Kubernetes multi-tenancy solution with [Capsule](https://capsule.clastix.io). Why does Kamaji matter?
-
-A. A multi-tenancy solution, like Capsule shares the Kubernetes control plane among all tenants keeping tenant namespaces isolated by policies. While the solution is the right choice by balancing between features and ease of usage, there are cases where a tenant user requires access to the control plane, for example, when a tenant requires to manage CRDs on his own. With Kamaji, you can provide cluster admin permissions to the tenant.
-
-Q. Well you convinced me, how to get a try?
-
-A. It is possible to get started with Kamaji on a laptop with [KinD](getting-started.md) installed.

docs/content/reference/benchmark.md

@@ -0,0 +1,239 @@
+# Benchmark
+
+Kamaji has been designed to operate a large scale of Kubernetes Tenant Control Plane resources.
+
+In the Operator jargon, a manager is created to start several controllers, each one with their own responsibility.
+When a manager is started, all the underlying controllers are started, along with other "runnable" resources, like the webhook server.
+
+Kamaji operates several reconciliation operations, both in the admin and tenant clusters.
+With that said, a main manager is responsible to reconcile the admin resources (Deployment, Secret, ConfigMap, etc.), for each Tenant Control Plane a new manager will be spin-up as a main manager controller.
+These Tenant Control Plane managers, named in the code base as soot managers, in turn, start and run controllers to ensure the desired state of the underlying add-ons, and required resources such as kubeadm ones.
+
+With that said, monitoring the Kamaji stack is essential to understand any anomaly in memory consumption, or CPU usage.
+The provided Helm Chart is offering a [`ServiceMonitor`](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/user-guides/getting-started.md) that can be used to extract all the required metrics of the Kamaji operator.
+
+# Running 100 Tenant Control Planes using a single DataStore
+
+- _Cloud platform:_ AWS
+- _Amount of reconciled TCPs:_ 100
+- _Number of DataStore resources:_ 1
+- _DataStore driver_: `etcd`
+- _Reconciliation time requested:_ ~7m30s
+
+The following benchmark wants to shed a light on the Kamaji resource consumption such as CPU and memory to orchestrate 100 TCPs using a single shared datastore.
+Your mileage may vary and just want to share with the community how it has been elaborated.
+
+## Infrastructure
+
+The benchmark has been issued on a Kubernetes cluster backed by Elastic Kubernetes Service used as an Admin cluster.
+
+Two node pools have been created to avoid the noisy neighbour effect, and to increase the performances:
+
+- `infra`: hosting Kamaji, the monitoring stack, and the `DataStore` resources, made of 2 `t3.medium` instances.
+- `workload`: hosting the deployed Tenant Control Plane resources, made of 25 `r3.xlarge` instances.
+
+### Monitoring stack
+
+The following monitoring stack was required to perform the benchmark.
+
+#### Deploy the Prometheus operator
+
+```
+helm repo add bitnami https://charts.bitnami.com/bitnami
+helm repo update
+helm upgrade --install kube-prometheus bitnami/kube-prometheus --namespace monitoring-system --create-namespace \
+    --set operator.nodeSelector."eks\.amazonaws\.com/nodegroup"=infra \
+    --set "operator.tolerations[0].key=pool" \
+    --set "operator.tolerations[0].operator=Equal" \
+    --set "operator.tolerations[0].value=infra" \
+    --set "operator.tolerations[0].effect=NoExecute" \
+    --set "operator.tolerations[1].key=pool" \
+    --set "operator.tolerations[1].operator=Equal" \
+    --set "operator.tolerations[1].value=infra" \
+    --set "operator.tolerations[1].effect=NoSchedule" \
+    --set prometheus.nodeSelector."eks\.amazonaws\.com/nodegroup"=infra \
+    --set "prometheus.tolerations[0].key=pool" \
+    --set "prometheus.tolerations[0].operator=Equal" \
+    --set "prometheus.tolerations[0].value=infra" \
+    --set "prometheus.tolerations[0].effect=NoExecute" \
+    --set "prometheus.tolerations[1].key=pool" \
+    --set "prometheus.tolerations[1].operator=Equal" \
+    --set "prometheus.tolerations[1].value=infra" \
+    --set "prometheus.tolerations[1].effect=NoSchedule" \
+    --set alertmanager.nodeSelector."eks\.amazonaws\.com/nodegroup"=infra \
+    --set "alertmanager.tolerations[0].key=pool" \
+    --set "alertmanager.tolerations[0].operator=Equal" \
+    --set "alertmanager.tolerations[0].value=infra" \
+    --set "alertmanager.tolerations[0].effect=NoExecute" \
+    --set "alertmanager.tolerations[1].key=pool" \
+    --set "alertmanager.tolerations[1].operator=Equal" \
+    --set "alertmanager.tolerations[1].value=infra" \
+    --set "alertmanager.tolerations[1].effect=NoSchedule" \
+    --set kube-state-metrics.nodeSelector."eks\.amazonaws\.com/nodegroup"=infra \
+    --set "kube-state-metrics.tolerations[0].key=pool" \
+    --set "kube-state-metrics.tolerations[0].operator=Equal" \
+    --set "kube-state-metrics.tolerations[0].value=infra" \
+    --set "kube-state-metrics.tolerations[0].effect=NoExecute" \
+    --set "kube-state-metrics.tolerations[1].key=pool" \
+    --set "kube-state-metrics.tolerations[1].operator=Equal" \
+    --set "kube-state-metrics.tolerations[1].value=infra" \
+    --set "kube-state-metrics.tolerations[1].effect=NoSchedule" \
+    --set blackboxExporter.nodeSelector."eks\.amazonaws\.com/nodegroup"=infra \
+    --set "blackboxExporter.tolerations[0].key=pool" \
+    --set "blackboxExporter.tolerations[0].operator=Equal" \
+    --set "blackboxExporter.tolerations[0].value=infra" \
+    --set "blackboxExporter.tolerations[0].effect=NoExecute" \
+    --set "blackboxExporter.tolerations[1].key=pool" \
+    --set "blackboxExporter.tolerations[1].operator=Equal" \
+    --set "blackboxExporter.tolerations[1].value=infra" \
+    --set "blackboxExporter.tolerations[1].effect=NoSchedule"
+```
+
+### Deploy a Grafana dashboard
+
+```
+helm repo add bitnami https://charts.bitnami.com/bitnami
+helm repo update
+helm upgrade --install grafana bitnami/grafana --namespace monitoring-system --create-namespace \
+    --set grafana.nodeSelector."eks\.amazonaws\.com/nodegroup"=infra \
+    --set "grafana.tolerations[0].key=pool" \
+    --set "grafana.tolerations[0].operator=Equal" \
+    --set "grafana.tolerations[0].value=infra" \
+    --set "grafana.tolerations[0].effect=NoSchedule" \
+    --set "grafana.tolerations[1].key=pool" \
+    --set "grafana.tolerations[1].operator=Equal" \
+    --set "grafana.tolerations[1].value=infra" \
+    --set "grafana.tolerations[1].effect=NoExecute"
+
+```
+
+> The dashboard can be used to have a visual representation of the global cluster resources usage, and getting information about the single Pods resources consumption.
+>
+> Besides that, Grafana has been used to track the etcd cluster status and performances, although it's not the subject of the benchmark.
+
+### Install cert-manager
+
+```
+helm repo add bitnami https://charts.bitnami.com/bitnami
+helm repo update
+helm upgrade --install cert-manager bitnami/cert-manager --namespace certmanager-system --create-namespace \
+    --set cainjector.nodeSelector."eks\.amazonaws\.com/nodegroup"=infra \
+    --set "cainjector.tolerations[0].key=pool" \
+    --set "cainjector.tolerations[0].operator=Equal" \
+    --set "cainjector.tolerations[0].value=infra" \
+    --set "cainjector.tolerations[0].effect=NoSchedule" \
+    --set "cainjector.tolerations[1].key=pool" \
+    --set "cainjector.tolerations[1].operator=Equal" \
+    --set "cainjector.tolerations[1].value=infra" \
+    --set "cainjector.tolerations[1].effect=NoExecute" \
+    --set controller.nodeSelector."eks\.amazonaws\.com/nodegroup"=infra \
+    --set "controller.tolerations[0].key=pool" \
+    --set "controller.tolerations[0].operator=Equal" \
+    --set "controller.tolerations[0].value=infra" \
+    --set "controller.tolerations[0].effect=NoExecute" \
+    --set "controller.tolerations[1].key=pool" \
+    --set "controller.tolerations[1].operator=Equal" \
+    --set "controller.tolerations[1].value=infra" \
+    --set "controller.tolerations[1].effect=NoSchedule" \
+    --set webhook.nodeSelector."eks\.amazonaws\.com/nodegroup"=infra \
+    --set "webhook.tolerations[0].key=pool" \
+    --set "webhook.tolerations[0].operator=Equal" \
+    --set "webhook.tolerations[0].value=infra" \
+    --set "webhook.tolerations[0].effect=NoSchedule" \
+    --set "webhook.tolerations[1].key=pool" \
+    --set "webhook.tolerations[1].operator=Equal" \
+    --set "webhook.tolerations[1].value=infra" \
+    --set "webhook.tolerations[1].effect=NoExecute" \
+    --set "installCRDs=true"
+```
+
+### Install Kamaji
+
+```
+helm upgrade --install kamaji clastix/kamaji --namespace kamaji-system --create-namespace \
+    --set nodeSelector."eks\.amazonaws\.com/nodegroup"=infra \
+    --set "tolerations[0].key=pool" \
+    --set "tolerations[0].operator=Equal" \
+    --set "tolerations[0].value=infra" \
+    --set "tolerations[0].effect=NoExecute" \
+    --set "tolerations[1].key=pool" \
+    --set "tolerations[1].operator=Equal" \
+    --set "tolerations[1].value=infra" \
+    --set "tolerations[1].effect=NoSchedule" \
+    --set "resources=null"
+```
+
+> For the benchmark, Kamaji is running without any resource constraint to benefit of all the available resources.
+
+### Install etcd as a DataStore
+
+```
+helm upgrade --install etcd-01 clastix/kamaji-etcd --namespace kamaji-etcd --create-namespace --set "serviceMonitor.enabled=true" --set "datastore.enabled=true"
+```
+
+## Creating 100 Tenant Control Planes
+
+Once all the required components have been deployed, a simple bash for loop can be used to deploy the TCP resources.
+
+```
+kubectl create ns benchmark01
+
+for I in {001..100}; do
+    DS=etcd-01 NS=benchmark01 I=$I envsubst < kamaji_v1alpha1_tenantcontrolplane.yaml | kubectl apply -f -
+done
+```
+
+Content of the benchmark file:
+
+```yaml
+apiVersion: kamaji.clastix.io/v1alpha1
+kind: TenantControlPlane
+metadata:
+  name: benchmark$I
+  namespace: $NS
+spec:
+  dataStore: $DS
+  controlPlane:
+    deployment:
+      replicas: 2
+      resources:
+        apiServer:
+          requests:
+            memory: 320Mi
+            cpu: 100m
+    service:
+      serviceType: ClusterIP
+  kubernetes:
+    version: "v1.25.4"
+    kubelet:
+      cgroupfs: cgroupfs
+    admissionControllers:
+      - ResourceQuota
+      - LimitRanger
+  networkProfile:
+    port: 6443
+  addons:
+    coreDNS: {}
+    kubeProxy: {}
+```
+
+> For the benchmark we're not creating a LoadBalancer service, or an Ingress.
+> The goal of the benchmark is to monitor resource consumption, and the average time required to create the requested resources.
+
+## Conclusion
+
+Our latest benchmark showed the ability to fully reconcile 100 Tenant Control Plane resources in ~7m30s.
+All the Tenant Control Planes were in `Ready` state and able to handle any request.
+
+The CPU consumption of Kamaji was fluctuating between 100 and 1200 mCPU during the peaks due to certificate generations.
+
+The memory consumption of Kamaji hit ~600 MiB, although this data is not entirely representative since we didn't put any memory limit.
+
+In conclusion, Kamaji was able to reconcile a Tenant Control Plane every 4.5 seconds, requiring 12 mCPU, and 6 MiB of memory.
+
+The following values may vary according to the nodes, resource limits, and other constraints.
+If you're encountering different results, please, engage with the community to share them. 
+
+# Running a thousand of Tenant Control Planes using multiple DataStores
+
+The next benchmark must address the use case where a Kamaji admin cluster manages up to a thousand Tenant Control Plane instances.

docs/content/reference/configuration.md

@@ -4,17 +4,22 @@ Currently, **Kamaji** allows customization using CLI flags for the `manager` sub
 
 Available flags are the following:
 
-```
-      --datastore string                   The default DataStore that should be used by Kamaji to setup the required storage (default "etcd")
-      --health-probe-bind-address string   The address the probe endpoint binds to. (default ":8081")
-  -h, --help                               help for manager
-      --kine-image string                  Container image along with tag to use for the Kine sidecar container (used only if etcd-storage-type is set to one of kine strategies) (default "rancher/kine:v0.9.2-amd64")
-      --leader-elect                       Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager. (default true)
-      --metrics-bind-address string        The address the metric endpoint binds to. (default ":8080")
-      --tmp-directory string               Directory which will be used to work with temporary files. (default "/tmp/kamaji")
-      --zap-devel                          Development Mode defaults(encoder=consoleEncoder,logLevel=Debug,stackTraceLevel=Warn). Production Mode defaults(encoder=jsonEncoder,logLevel=Info,stackTraceLevel=Error) (default true)
-      --zap-encoder encoder                Zap log encoding (one of 'json' or 'console')
-      --zap-log-level level                Zap Level to configure the verbosity of logging. Can be one of 'debug', 'info', 'error', or any integer value > 0 which corresponds to custom debug levels of increasing verbosity
-      --zap-stacktrace-level level         Zap Level at and above which stacktraces are captured (one of 'info', 'error', 'panic').
-      --zap-time-encoding time-encoding    Zap time encoding (one of 'epoch', 'millis', 'nano', 'iso8601', 'rfc3339' or 'rfc3339nano'). Defaults to 'epoch'.
-```
+| Flag | Usage | Default |
+| ---- | ------ | --- |
+| `--metrics-bind-address` | The address the metric endpoint binds to. | `:8080` |
+| `--health-probe-bind-address` | The address the probe endpoint binds to. | `:8081` |
+| `--leader-elect` | Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager. | `true` |
+| `--tmp-directory` | Directory which will be used to work with temporary files. | `/tmp/kamaji` |
+| `--kine-image` | Container image along with tag to use for the Kine sidecar container (used only if etcd-storage-type is set to one of kine strategies). | `rancher/kine:v0.9.2-amd64` |
+| `--datastore` | The default DataStore that should be used by Kamaji to setup the required storage. | `etcd` |
+| `--migrate-image` | Specify the container image to launch when a TenantControlPlane is migrated to a new datastore. | `migrate-image` |
+| `--max-concurrent-tcp-reconciles` | Specify the number of workers for the Tenant Control Plane controller (beware of CPU consumption). | `1` |
+| `--pod-namespace` | The Kubernetes Namespace on which the Operator is running in, required for the TenantControlPlane migration jobs. | `os.Getenv("POD_NAMESPACE")` |
+| `--webhook-service-name` | The Kamaji webhook server Service name which is used to get validation webhooks, required for the TenantControlPlane migration jobs. | `kamaji-webhook-service` |
+| `--serviceaccount-name` | The Kubernetes ServiceAccount used by the Operator, required for the TenantControlPlane migration jobs. | `os.Getenv("SERVICE_ACCOUNT")` |
+| `--webhook-ca-path` | Path to the Manager webhook server CA, required for the TenantControlPlane migration jobs. | `/tmp/k8s-webhook-server/serving-certs/ca.crt` |
+| `--zap-devel`  | Development Mode (encoder=consoleEncoder,logLevel=Debug,stackTraceLevel=Warn). Production Mode (encoder=jsonEncoder,logLevel=Info,stackTraceLevel=Error).  |  `true`  |
+| `--zap-encoder`  | Zap log encoding, one of 'json' or 'console'  |  `console`  |
+| `--zap-log-level`  |  Zap Level to configure the verbosity of logging. Can be one of 'debug', 'info', 'error', or any integer value > 0 which corresponds to custom debug levels of increasing verbosity |  `info`  |
+| `--zap-stacktrace-level`  | Zap Level at and above which stacktraces are captured (one of 'info', 'error', 'panic').  |  `info` |
+| `--zap-time-encoding`  |  Zap time encoding (one of 'epoch', 'millis', 'nano', 'iso8601', 'rfc3339' or 'rfc3339nano') |  `epoch`  |

docs/content/reference/versioning.md

@@ -2,9 +2,10 @@
 
 In Kamaji, there are different components that might require independent versioning and support level:
 
-|Kamaji|Admin Cluster|Tenant Cluster (min)|Tenant Cluster (max)|Konnectivity|Tenant etcd |
-|------|-------------|--------------------|--------------------|------------|------------|
-|0.0.1 |1.22.0+      |1.21.0              |1.23.5              |0.0.31      |3.5.4       |
-|0.0.2 |1.22.0+      |1.21.0              |1.25.0              |0.0.32      |3.5.4       | 
-
-Other combinations might work but they have not been yet tested.
+| Kamaji | Admin Cluster | Tenant Cluster       |
+|--------|---------------|----------------------|
+| v0.0   | v1.22+        | [v1.21.0 .. v1.23.5] |
+| v0.1   | v1.22+        | [v1.21.0 .. v1.25.0] |
+| v0.2   | v1.22+        | [v1.21.0 .. v1.27.0] |
+| v0.3.0 | v1.22+        | [v1.21.0 .. v1.27.0] |
+| v0.3.1 | v1.22+        | [v1.21.0 .. v1.27.3] |

docs/mkdocs.yml

@@ -1,39 +1,56 @@
-site_name: Kamaji
 repo_name: clastix/kamaji
 repo_url: https://github.com/clastix/kamaji
+
+site_name: Kamaji
 site_url: https://kamaji.clastix.io/
 docs_dir: content
 site_dir: site
+site_author: bsctl
+site_description: >-
+  Kamaji deploys and operates Kubernetes Control Plane at scale with a fraction of the operational burden.
+
+copyright: Copyright © 2020 - 2023 Clastix Labs
 
 theme:
   name: material
   features:
     - navigation.tabs
-    - navigation.tabs.sticky
+    - navigation.indexes
     - navigation.instant
     - navigation.sections
+    - navigation.path
+    - navigation.footer
+    - content.code.copy
   include_sidebar: true
   palette:
-
     # Palette toggle for automatic mode
     - media: "(prefers-color-scheme)"
+      primary: white
       toggle:
         icon: material/brightness-auto
         name: Switch to light mode
-
     # Palette toggle for light mode
     - media: "(prefers-color-scheme: light)"
-      scheme: default 
+      scheme: default
+      primary: white 
       toggle:
         icon: material/lightbulb
         name: Switch to dark mode
-
     # Palette toggle for dark mode
     - media: "(prefers-color-scheme: dark)"
       scheme: slate
+      primary: white
       toggle:
         icon: material/lightbulb-outline
         name: Switch to system preference
+  favicon: images/favicon.png
+  logo: images/logo.png
+
+markdown_extensions:
+  - admonition
+  - attr_list
+  - def_list
+  - md_in_html
 
 # Generate navigation bar
 nav:
@@ -42,20 +59,18 @@ nav:
 - 'Concepts': concepts.md
 - 'Guides':
   - guides/index.md
-  - guides/kamaji-deployment-guide.md
-  - guides/kamaji-azure-deployment-guide.md
-  - guides/postgresql-datastore.md
-  - guides/mysql-datastore.md
+  - guides/kamaji-azure-deployment.md
+  - guides/alternative-datastore.md
+  - guides/kamaji-gitops-flux.md
   - guides/upgrade.md
+  - guides/datastore-migration.md
+  - guides/backup-and-restore.md
 - 'Use Cases': use-cases.md
 - 'Reference':
   - reference/index.md
+  - reference/benchmark.md
   - reference/configuration.md
   - reference/conformance.md
   - reference/versioning.md
   - reference/api.md
-- 'Contribute':
-  - contribute/index.md
-  - contribute/guidelines.md
-  - contribute/governance.md
-
+- 'Contribute': contribute.md

e2e/tcp_additional_resources_blocked_test.go

@@ -0,0 +1,179 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package e2e
+
+import (
+	"context"
+	"time"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+)
+
+var _ = Describe("Deploy a TenantControlPlane resource with additional resources", func() {
+	// TenantControlPlane object with additional resources
+	tcp := &kamajiv1alpha1.TenantControlPlane{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "validated-additional-resources",
+			Namespace: "default",
+		},
+		Spec: kamajiv1alpha1.TenantControlPlaneSpec{
+			ControlPlane: kamajiv1alpha1.ControlPlane{
+				Deployment: kamajiv1alpha1.DeploymentSpec{
+					Replicas: 1,
+					AdditionalInitContainers: []corev1.Container{{
+						Name:  initContainerName,
+						Image: initContainerImage,
+						Command: []string{
+							"/bin/sh",
+							"-c",
+							"echo hello world",
+						},
+					}},
+					AdditionalContainers: []corev1.Container{{
+						Name:  additionalContainerName,
+						Image: additionalContainerImage,
+					}},
+					AdditionalVolumes: []corev1.Volume{
+						{
+							Name: apiServerVolumeName,
+							VolumeSource: corev1.VolumeSource{
+								ConfigMap: &corev1.ConfigMapVolumeSource{
+									LocalObjectReference: corev1.LocalObjectReference{
+										Name: "api-server",
+									},
+								},
+							},
+						},
+						{
+							Name: controllerManagerVolumeName,
+							VolumeSource: corev1.VolumeSource{
+								ConfigMap: &corev1.ConfigMapVolumeSource{
+									LocalObjectReference: corev1.LocalObjectReference{
+										Name: "controller-manager",
+									},
+								},
+							},
+						},
+						{
+							Name: schedulerVolumeName,
+							VolumeSource: corev1.VolumeSource{
+								ConfigMap: &corev1.ConfigMapVolumeSource{
+									LocalObjectReference: corev1.LocalObjectReference{
+										Name: "scheduler",
+									},
+								},
+							},
+						},
+					},
+					AdditionalVolumeMounts: &kamajiv1alpha1.AdditionalVolumeMounts{
+						APIServer: []corev1.VolumeMount{
+							{
+								Name:      apiServerVolumeName,
+								MountPath: "/etc/api-server",
+							},
+						},
+						ControllerManager: []corev1.VolumeMount{
+							{
+								Name:      controllerManagerVolumeName,
+								MountPath: "/etc/controller-manager",
+							},
+						},
+						Scheduler: []corev1.VolumeMount{
+							{
+								Name:      schedulerVolumeName,
+								MountPath: "/etc/scheduler",
+							},
+						},
+					},
+				},
+				Service: kamajiv1alpha1.ServiceSpec{
+					ServiceType: "ClusterIP",
+				},
+			},
+			Kubernetes: kamajiv1alpha1.KubernetesSpec{
+				Version: "v1.23.6",
+			},
+		},
+	}
+
+	apiServerConfigMap := &corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "api-server",
+			Namespace: tcp.Namespace,
+		},
+		Data: map[string]string{
+			"api-server": "true",
+		},
+	}
+
+	controllerManagerConfigMap := &corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "controller-manager",
+			Namespace: tcp.Namespace,
+		},
+		Data: map[string]string{
+			"controller-manager": "true",
+		},
+	}
+
+	schedulerConfigMap := &corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "scheduler",
+			Namespace: tcp.Namespace,
+		},
+		Data: map[string]string{
+			"scheduler": "true",
+		},
+	}
+	// Create a TenantControlPlane resource into the cluster
+	JustBeforeEach(func() {
+		Expect(k8sClient.Create(context.Background(), apiServerConfigMap)).NotTo(HaveOccurred())
+		Expect(k8sClient.Create(context.Background(), controllerManagerConfigMap)).NotTo(HaveOccurred())
+		Expect(k8sClient.Create(context.Background(), schedulerConfigMap)).NotTo(HaveOccurred())
+		Expect(k8sClient.Create(context.Background(), tcp)).NotTo(HaveOccurred())
+	})
+	// Delete the TenantControlPlane resource after test is finished
+	JustAfterEach(func() {
+		Expect(k8sClient.Delete(context.Background(), tcp)).Should(Succeed())
+		Expect(k8sClient.Delete(context.Background(), apiServerConfigMap)).NotTo(HaveOccurred())
+		Expect(k8sClient.Delete(context.Background(), controllerManagerConfigMap)).NotTo(HaveOccurred())
+		Expect(k8sClient.Delete(context.Background(), schedulerConfigMap)).NotTo(HaveOccurred())
+	})
+	It("should block wrong Deployment configuration", func() {
+		// Should be ready
+		StatusMustEqualTo(tcp, kamajiv1alpha1.VersionReady)
+
+		By("duplicating mount path", func() {
+			Consistently(func() error {
+				Expect(k8sClient.Get(context.Background(), types.NamespacedName{Name: tcp.Name, Namespace: tcp.Namespace}, tcp)).NotTo(HaveOccurred())
+
+				lastVolumeMountIndex := len(tcp.Spec.ControlPlane.Deployment.AdditionalVolumeMounts.APIServer) - 1
+				additionalVolumeMount := tcp.Spec.ControlPlane.Deployment.AdditionalVolumeMounts.APIServer[lastVolumeMountIndex]
+				additionalVolumeMount.Name = "duplicated"
+				tcp.Spec.ControlPlane.Deployment.AdditionalVolumeMounts.APIServer = append(tcp.Spec.ControlPlane.Deployment.AdditionalVolumeMounts.APIServer, additionalVolumeMount)
+
+				return k8sClient.Update(context.Background(), tcp)
+			}, 10*time.Second, time.Second).ShouldNot(Succeed())
+		})
+
+		By("duplicating container", func() {
+			Consistently(func() error {
+				Expect(k8sClient.Get(context.Background(), types.NamespacedName{Name: tcp.Name, Namespace: tcp.Namespace}, tcp)).NotTo(HaveOccurred())
+
+				tcp.Spec.ControlPlane.Deployment.AdditionalContainers = append(tcp.Spec.ControlPlane.Deployment.AdditionalContainers, corev1.Container{
+					Name:  "kube-apiserver",
+					Image: "mocked",
+				})
+
+				return k8sClient.Update(context.Background(), tcp)
+			}, 10*time.Second, time.Second).ShouldNot(Succeed())
+		})
+	})
+})

e2e/tcp_additional_resources_ready_test.go

@@ -0,0 +1,323 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package e2e
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/sets"
+
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+	"github.com/clastix/kamaji/internal/utilities"
+)
+
+const (
+	namespace                   = "default"
+	tcpName                     = "tcp-additional"
+	initContainerName           = "init"
+	initContainerImage          = "registry.k8s.io/e2e-test-images/busybox:1.29-4"
+	additionalContainerName     = "nginx"
+	additionalContainerImage    = "registry.k8s.io/e2e-test-images/nginx:1.15-4"
+	apiServerVolumeName         = "api-server-volume"
+	controllerManagerVolumeName = "controller-manager-volume"
+	schedulerVolumeName         = "scheduler-volume"
+)
+
+var _ = Describe("Deploy a TenantControlPlane resource with additional options", func() {
+	// TenantControlPlane object with additional resources
+	tcp := &kamajiv1alpha1.TenantControlPlane{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      tcpName,
+			Namespace: namespace,
+		},
+		Spec: kamajiv1alpha1.TenantControlPlaneSpec{
+			ControlPlane: kamajiv1alpha1.ControlPlane{
+				Deployment: kamajiv1alpha1.DeploymentSpec{
+					Replicas: 1,
+					AdditionalInitContainers: []corev1.Container{{
+						Name:  initContainerName,
+						Image: initContainerImage,
+						Command: []string{
+							"/bin/sh",
+							"-c",
+							"echo hello world",
+						},
+					}},
+					AdditionalContainers: []corev1.Container{{
+						Name:  additionalContainerName,
+						Image: additionalContainerImage,
+					}},
+					AdditionalVolumes: []corev1.Volume{
+						{
+							Name: apiServerVolumeName,
+							VolumeSource: corev1.VolumeSource{
+								ConfigMap: &corev1.ConfigMapVolumeSource{
+									LocalObjectReference: corev1.LocalObjectReference{
+										Name: "api-server",
+									},
+								},
+							},
+						},
+						{
+							Name: controllerManagerVolumeName,
+							VolumeSource: corev1.VolumeSource{
+								ConfigMap: &corev1.ConfigMapVolumeSource{
+									LocalObjectReference: corev1.LocalObjectReference{
+										Name: "controller-manager",
+									},
+								},
+							},
+						},
+						{
+							Name: schedulerVolumeName,
+							VolumeSource: corev1.VolumeSource{
+								ConfigMap: &corev1.ConfigMapVolumeSource{
+									LocalObjectReference: corev1.LocalObjectReference{
+										Name: "scheduler",
+									},
+								},
+							},
+						},
+					},
+					AdditionalVolumeMounts: &kamajiv1alpha1.AdditionalVolumeMounts{
+						APIServer: []corev1.VolumeMount{
+							{
+								Name:      apiServerVolumeName,
+								MountPath: "/etc/api-server",
+							},
+						},
+						ControllerManager: []corev1.VolumeMount{
+							{
+								Name:      controllerManagerVolumeName,
+								MountPath: "/etc/controller-manager",
+							},
+						},
+						Scheduler: []corev1.VolumeMount{
+							{
+								Name:      schedulerVolumeName,
+								MountPath: "/etc/scheduler",
+							},
+						},
+					},
+				},
+				Service: kamajiv1alpha1.ServiceSpec{
+					ServiceType: "ClusterIP",
+				},
+			},
+			NetworkProfile: kamajiv1alpha1.NetworkProfileSpec{
+				Address: "172.18.0.2",
+			},
+			Kubernetes: kamajiv1alpha1.KubernetesSpec{
+				Version: "v1.23.6",
+				Kubelet: kamajiv1alpha1.KubeletSpec{
+					CGroupFS: "cgroupfs",
+				},
+				AdmissionControllers: kamajiv1alpha1.AdmissionControllers{
+					"LimitRanger",
+					"ResourceQuota",
+				},
+			},
+			Addons: kamajiv1alpha1.AddonsSpec{},
+		},
+	}
+	apiServerConfigMap := &corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "api-server",
+			Namespace: namespace,
+		},
+		Data: map[string]string{
+			"api-server": "true",
+		},
+	}
+	controllerManagerConfigMap := &corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "controller-manager",
+			Namespace: namespace,
+		},
+		Data: map[string]string{
+			"controller-manager": "true",
+		},
+	}
+	schedulerConfigMap := &corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "scheduler",
+			Namespace: namespace,
+		},
+		Data: map[string]string{
+			"scheduler": "true",
+		},
+	}
+
+	// Create a TenantControlPlane resource into the cluster
+	JustBeforeEach(func() {
+		Expect(k8sClient.Create(context.Background(), apiServerConfigMap)).NotTo(HaveOccurred())
+		Expect(k8sClient.Create(context.Background(), controllerManagerConfigMap)).NotTo(HaveOccurred())
+		Expect(k8sClient.Create(context.Background(), schedulerConfigMap)).NotTo(HaveOccurred())
+		Expect(k8sClient.Create(context.Background(), tcp)).NotTo(HaveOccurred())
+	})
+
+	// Delete the TenantControlPlane resource after test is finished
+	JustAfterEach(func() {
+		Expect(k8sClient.Delete(context.Background(), tcp)).Should(Succeed())
+		Expect(k8sClient.Delete(context.Background(), apiServerConfigMap)).NotTo(HaveOccurred())
+		Expect(k8sClient.Delete(context.Background(), controllerManagerConfigMap)).NotTo(HaveOccurred())
+		Expect(k8sClient.Delete(context.Background(), schedulerConfigMap)).NotTo(HaveOccurred())
+	})
+
+	It("should have the additional resources", func() {
+		// Should be ready
+		StatusMustEqualTo(tcp, kamajiv1alpha1.VersionReady)
+		// Should have a TCP deployment
+		deploy := appsv1.Deployment{}
+		Expect(k8sClient.Get(context.Background(), types.NamespacedName{
+			Name:      tcpName,
+			Namespace: namespace,
+		}, &deploy)).NotTo(HaveOccurred())
+
+		By("checking additional init containers", func() {
+			found, _ := utilities.HasNamedContainer(deploy.Spec.Template.Spec.InitContainers, initContainerName)
+			Expect(found).To(BeTrue(), "Should have the configured AdditionalInitContainers")
+		})
+
+		By("checking additional containers", func() {
+			found, _ := utilities.HasNamedContainer(deploy.Spec.Template.Spec.Containers, additionalContainerName)
+			Expect(found).To(BeTrue(), "Should have the configured AdditionalContainers")
+		})
+
+		By("checking kube-apiserver volumes", func() {
+			found, _ := utilities.HasNamedVolume(deploy.Spec.Template.Spec.Volumes, apiServerVolumeName)
+			Expect(found).To(BeTrue())
+
+			found, containerIndex := utilities.HasNamedContainer(deploy.Spec.Template.Spec.Containers, "kube-apiserver")
+			Expect(found).To(BeTrue())
+
+			found, _ = utilities.HasNamedVolumeMount(deploy.Spec.Template.Spec.Containers[containerIndex].VolumeMounts, apiServerVolumeName)
+			Expect(found).To(BeTrue())
+		})
+
+		By("checking kube-scheduler volumes", func() {
+			found, _ := utilities.HasNamedVolume(deploy.Spec.Template.Spec.Volumes, schedulerVolumeName)
+			Expect(found).To(BeTrue())
+
+			found, containerIndex := utilities.HasNamedContainer(deploy.Spec.Template.Spec.Containers, "kube-scheduler")
+			Expect(found).To(BeTrue())
+
+			found, _ = utilities.HasNamedVolumeMount(deploy.Spec.Template.Spec.Containers[containerIndex].VolumeMounts, schedulerVolumeName)
+			Expect(found).To(BeTrue())
+		})
+
+		By("checking kube-controller-manager volumes", func() {
+			found, _ := utilities.HasNamedVolume(deploy.Spec.Template.Spec.Volumes, controllerManagerVolumeName)
+			Expect(found).To(BeTrue())
+
+			found, containerIndex := utilities.HasNamedContainer(deploy.Spec.Template.Spec.Containers, "kube-controller-manager")
+			Expect(found).To(BeTrue())
+
+			found, _ = utilities.HasNamedVolumeMount(deploy.Spec.Template.Spec.Containers[containerIndex].VolumeMounts, controllerManagerVolumeName)
+			Expect(found).To(BeTrue())
+		})
+
+		By("removing the additional resources", func() {
+			var containerName string
+			volumeNames, volumeMounts := sets.New[string](), tcp.Spec.ControlPlane.Deployment.AdditionalVolumeMounts
+
+			Eventually(func() error {
+				Expect(k8sClient.Get(context.Background(), types.NamespacedName{Name: tcp.Name, Namespace: tcp.Namespace}, tcp)).NotTo(HaveOccurred())
+				containerName = tcp.Spec.ControlPlane.Deployment.AdditionalContainers[0].Name
+
+				for _, volume := range tcp.Spec.ControlPlane.Deployment.AdditionalVolumes {
+					volumeNames.Insert(volume.Name)
+				}
+
+				tcp.Spec.ControlPlane.Deployment.AdditionalInitContainers = nil
+				tcp.Spec.ControlPlane.Deployment.AdditionalContainers = nil
+				tcp.Spec.ControlPlane.Deployment.AdditionalVolumeMounts = nil
+				tcp.Spec.ControlPlane.Deployment.AdditionalVolumes = nil
+
+				return k8sClient.Update(context.Background(), tcp)
+			}, 10*time.Second, time.Second).ShouldNot(HaveOccurred())
+
+			Eventually(func() []corev1.Container {
+				deploy := appsv1.Deployment{}
+
+				Expect(k8sClient.Get(context.Background(), types.NamespacedName{
+					Name:      tcpName,
+					Namespace: namespace,
+				}, &deploy)).NotTo(HaveOccurred())
+
+				return deploy.Spec.Template.Spec.InitContainers
+			}, 10*time.Second, time.Second).Should(HaveLen(0), "Deployment should not contain anymore the init container")
+
+			Eventually(func() bool {
+				deploy := appsv1.Deployment{}
+
+				Expect(k8sClient.Get(context.Background(), types.NamespacedName{
+					Name:      tcpName,
+					Namespace: namespace,
+				}, &deploy)).NotTo(HaveOccurred())
+
+				found, _ := utilities.HasNamedContainer(deploy.Spec.Template.Spec.Containers, containerName)
+
+				return found
+			}, 10*time.Second, time.Second).Should(BeFalse(), "Deployment should not contain anymore the additional container")
+
+			Eventually(func() error {
+				deploy := appsv1.Deployment{}
+
+				Expect(k8sClient.Get(context.Background(), types.NamespacedName{
+					Name:      tcpName,
+					Namespace: namespace,
+				}, &deploy)).NotTo(HaveOccurred())
+
+				for _, volume := range deploy.Spec.Template.Spec.Volumes {
+					if volumeNames.Has(volume.Name) {
+						return fmt.Errorf("extra volume with name %s is still present", volume.Name)
+					}
+				}
+
+				type testCase struct {
+					containerName string
+					volumeMounts  []corev1.VolumeMount
+				}
+
+				for _, tc := range []testCase{
+					{
+						containerName: "kube-scheduler",
+						volumeMounts:  volumeMounts.Scheduler,
+					},
+					{
+						containerName: "kube-apiserver",
+						volumeMounts:  volumeMounts.APIServer,
+					},
+					{
+						containerName: "kube-scheduler",
+						volumeMounts:  volumeMounts.Scheduler,
+					},
+				} {
+					for _, volumeMount := range tc.volumeMounts {
+						found, containerIndex := utilities.HasNamedContainer(deploy.Spec.Template.Spec.Containers, tc.containerName)
+						if !found {
+							return fmt.Errorf("expected %s, container not found", tc.containerName)
+						}
+
+						found, _ = utilities.HasNamedVolumeMount(deploy.Spec.Template.Spec.Containers[containerIndex].VolumeMounts, volumeMount.Name)
+						if found {
+							return fmt.Errorf("extra volume mount with name %s is still present", volumeMount.Name)
+						}
+					}
+				}
+
+				return nil
+			}, 10*time.Second, time.Second).Should(BeNil(), "Deployment should not contain anymore the extra volumes")
+		})
+	})
+})

e2e/tcp_migration_test.go

@@ -62,6 +62,8 @@ var _ = Describe("When migrating a Tenant Control Plane to another datastore", f
 	})
 	// Check if TenantControlPlane resource has been created
 	It("Should contain all the migrated data", func() {
+		time.Sleep(10 * time.Second)
+
 		By("getting TCP rest.Config")
 		config, err := utilities.GetTenantKubeconfig(context.Background(), k8sClient, tcp)
 		Expect(err).ToNot(HaveOccurred())

e2e/tcp_validation_preferredkubeletaddresstypes_test.go

@@ -0,0 +1,89 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package e2e
+
+import (
+	"context"
+	"time"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+)
+
+var _ = Describe("Deploy a TenantControlPlane with wrong preferred kubelet address type entries", func() {
+	It("should fail when using duplicates", func() {
+		Consistently(func() error {
+			tcp := &kamajiv1alpha1.TenantControlPlane{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "duplicated-kubelet-preferred-address-type",
+					Namespace: "default",
+				},
+				Spec: kamajiv1alpha1.TenantControlPlaneSpec{
+					DataStore: "default",
+					ControlPlane: kamajiv1alpha1.ControlPlane{
+						Deployment: kamajiv1alpha1.DeploymentSpec{
+							Replicas: 1,
+						},
+						Service: kamajiv1alpha1.ServiceSpec{
+							ServiceType: "ClusterIP",
+						},
+					},
+					Kubernetes: kamajiv1alpha1.KubernetesSpec{
+						Version: "v1.23.6",
+						Kubelet: kamajiv1alpha1.KubeletSpec{
+							PreferredAddressTypes: []kamajiv1alpha1.KubeletPreferredAddressType{
+								kamajiv1alpha1.NodeHostName,
+								kamajiv1alpha1.NodeInternalIP,
+								kamajiv1alpha1.NodeExternalIP,
+								kamajiv1alpha1.NodeHostName,
+							},
+							CGroupFS: "cgroupfs",
+						},
+					},
+				},
+			}
+
+			return k8sClient.Create(context.Background(), tcp)
+		}, 10*time.Second, time.Second).ShouldNot(Succeed())
+	})
+
+	It("should fail when using non valid entries", func() {
+		Consistently(func() error {
+			tcp := &kamajiv1alpha1.TenantControlPlane{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "duplicated-kubelet-preferred-address-type",
+					Namespace: "default",
+				},
+				Spec: kamajiv1alpha1.TenantControlPlaneSpec{
+					DataStore: "default",
+					ControlPlane: kamajiv1alpha1.ControlPlane{
+						Deployment: kamajiv1alpha1.DeploymentSpec{
+							Replicas: 1,
+						},
+						Service: kamajiv1alpha1.ServiceSpec{
+							ServiceType: "ClusterIP",
+						},
+					},
+					Kubernetes: kamajiv1alpha1.KubernetesSpec{
+						Version: "v1.23.6",
+						Kubelet: kamajiv1alpha1.KubeletSpec{
+							PreferredAddressTypes: []kamajiv1alpha1.KubeletPreferredAddressType{
+								kamajiv1alpha1.NodeHostName,
+								kamajiv1alpha1.NodeInternalIP,
+								kamajiv1alpha1.NodeExternalIP,
+								"Foo",
+							},
+							CGroupFS: "cgroupfs",
+						},
+					},
+				},
+			}
+
+			return k8sClient.Create(context.Background(), tcp)
+		}, 10*time.Second, time.Second).ShouldNot(Succeed())
+	})
+})

go.mod

@@ -8,8 +8,10 @@ require (
 	github.com/go-logr/logr v1.2.3
 	github.com/go-pg/pg/v10 v10.10.6
 	github.com/go-sql-driver/mysql v1.6.0
+	github.com/google/go-cmp v0.5.9
 	github.com/google/uuid v1.3.0
 	github.com/json-iterator/go v1.1.12
+	github.com/juju/mutex/v2 v2.0.0
 	github.com/onsi/ginkgo/v2 v2.6.0
 	github.com/onsi/gomega v1.24.1
 	github.com/pkg/errors v0.9.1
@@ -19,14 +21,16 @@ require (
 	github.com/testcontainers/testcontainers-go v0.13.0
 	go.etcd.io/etcd/api/v3 v3.5.6
 	go.etcd.io/etcd/client/v3 v3.5.6
-	k8s.io/api v0.26.0
-	k8s.io/apimachinery v0.26.0
-	k8s.io/apiserver v0.26.0
-	k8s.io/client-go v0.26.0
+	go.uber.org/automaxprocs v1.5.1
+	gomodules.xyz/jsonpatch/v2 v2.2.0
+	k8s.io/api v0.26.1
+	k8s.io/apimachinery v0.26.1
+	k8s.io/apiserver v0.26.1
+	k8s.io/client-go v0.26.1
 	k8s.io/cluster-bootstrap v0.0.0
 	k8s.io/klog/v2 v2.80.1
 	k8s.io/kubelet v0.0.0
-	k8s.io/kubernetes v1.26.0
+	k8s.io/kubernetes v1.26.1
 	k8s.io/utils v0.0.0-20221128185143-99ec85e7a448
 	sigs.k8s.io/controller-runtime v0.14.0
 )
@@ -73,7 +77,6 @@ require (
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/google/btree v1.0.1 // indirect
 	github.com/google/gnostic v0.5.7-v3refs // indirect
-	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/google/gofuzz v1.1.0 // indirect
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
 	github.com/googleapis/gax-go/v2 v2.1.1 // indirect
@@ -84,6 +87,7 @@ require (
 	github.com/inconshreveable/mousetrap v1.0.1 // indirect
 	github.com/jinzhu/inflection v1.0.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
+	github.com/juju/errors v0.0.0-20220203013757-bd733f3c86b9 // indirect
 	github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
 	github.com/lithammer/dedent v1.1.0 // indirect
 	github.com/magiconair/properties v1.8.5 // indirect
@@ -133,7 +137,6 @@ require (
 	golang.org/x/text v0.5.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
-	gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect
 	google.golang.org/api v0.63.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect
@@ -144,9 +147,9 @@ require (
 	gopkg.in/square/go-jose.v2 v2.5.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	k8s.io/apiextensions-apiserver v0.26.0 // indirect
-	k8s.io/cli-runtime v0.26.0 // indirect
-	k8s.io/component-base v0.26.0 // indirect
+	k8s.io/apiextensions-apiserver v0.26.1 // indirect
+	k8s.io/cli-runtime v0.26.1 // indirect
+	k8s.io/component-base v0.26.1 // indirect
 	k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 // indirect
 	k8s.io/kube-proxy v0.0.0 // indirect
 	k8s.io/system-validators v1.8.0 // indirect
@@ -159,32 +162,32 @@ require (
 )
 
 replace (
-	k8s.io/api => k8s.io/api v0.26.0
-	k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.26.0
-	k8s.io/apimachinery => k8s.io/apimachinery v0.26.0
-	k8s.io/apiserver => k8s.io/apiserver v0.26.0
-	k8s.io/cli-runtime => k8s.io/cli-runtime v0.26.0
-	k8s.io/client-go => k8s.io/client-go v0.26.0
-	k8s.io/cloud-provider => k8s.io/cloud-provider v0.26.0
-	k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.26.0
-	k8s.io/code-generator => k8s.io/code-generator v0.26.0
-	k8s.io/component-base => k8s.io/component-base v0.26.0
-	k8s.io/component-helpers => k8s.io/component-helpers v0.26.0
-	k8s.io/controller-manager => k8s.io/controller-manager v0.26.0
-	k8s.io/cri-api => k8s.io/cri-api v0.26.0
-	k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.26.0
-	k8s.io/dynamic-resource-allocation => k8s.io/dynamic-resource-allocation v0.26.0
-	k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.26.0
-	k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.26.0
-	k8s.io/kube-proxy => k8s.io/kube-proxy v0.26.0
-	k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.26.0
-	k8s.io/kubectl => k8s.io/kubectl v0.26.0
-	k8s.io/kubelet => k8s.io/kubelet v0.26.0
-	k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.26.0
-	k8s.io/metrics => k8s.io/metrics v0.26.0
-	k8s.io/mount-utils => k8s.io/mount-utils v0.26.0
-	k8s.io/pod-security-admission => k8s.io/pod-security-admission v0.26.0
-	k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.26.0
+	k8s.io/api => k8s.io/api v0.26.1
+	k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.26.1
+	k8s.io/apimachinery => k8s.io/apimachinery v0.26.1
+	k8s.io/apiserver => k8s.io/apiserver v0.26.1
+	k8s.io/cli-runtime => k8s.io/cli-runtime v0.26.1
+	k8s.io/client-go => k8s.io/client-go v0.26.1
+	k8s.io/cloud-provider => k8s.io/cloud-provider v0.26.1
+	k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.26.1
+	k8s.io/code-generator => k8s.io/code-generator v0.26.1
+	k8s.io/component-base => k8s.io/component-base v0.26.1
+	k8s.io/component-helpers => k8s.io/component-helpers v0.26.1
+	k8s.io/controller-manager => k8s.io/controller-manager v0.26.1
+	k8s.io/cri-api => k8s.io/cri-api v0.26.1
+	k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.26.1
+	k8s.io/dynamic-resource-allocation => k8s.io/dynamic-resource-allocation v0.26.1
+	k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.26.1
+	k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.26.1
+	k8s.io/kube-proxy => k8s.io/kube-proxy v0.26.1
+	k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.26.1
+	k8s.io/kubectl => k8s.io/kubectl v0.26.1
+	k8s.io/kubelet => k8s.io/kubelet v0.26.1
+	k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.26.1
+	k8s.io/metrics => k8s.io/metrics v0.26.1
+	k8s.io/mount-utils => k8s.io/mount-utils v0.26.1
+	k8s.io/pod-security-admission => k8s.io/pod-security-admission v0.26.1
+	k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.26.1
 )
 
 replace github.com/JamesStewy/go-mysqldump => github.com/vtoma/go-mysqldump v1.0.0