feat: making kine container image configurable via kamaji flag

A new CLI flag (`--kine-container`) has been introduced, with the
default value of `rancher/kine:v0.9.2-amd64`. It can be overridden also
using the kamaji configuration file (`kamaji.yaml`) using the key
`kine-image`.

.github/workflows/ci.yaml

@@ -12,6 +12,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
+      - uses: actions/setup-go@v2
+        with:
+          go-version: '1.18'
       - name: Run golangci-lint
         uses: golangci/golangci-lint-action@v2.3.0
         with:
@@ -27,7 +30,7 @@ jobs:
           fetch-depth: 0
       - uses: actions/setup-go@v2
         with:
-          go-version: '1.17'
+          go-version: '1.18'
       - run: make yaml-installation-file
       - name: Checking if YAML installer file is not aligned
         run: if [[ $(git diff | wc -l) -gt 0 ]]; then echo ">>> Untracked generated files have not been committed" && git --no-pager diff && exit 1; fi

.github/workflows/docker-ci.yml

@@ -0,0 +1,74 @@
+name: docker-ci
+
+on:
+  push:
+    tags:        
+      - "v*"
+
+jobs:
+  docker-ci:
+    runs-on: ubuntu-20.04
+    steps:
+
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v3
+        with:
+          images: |
+             quay.io/${{ github.repository }}
+             docker.io/${{ github.repository }}
+          tags: |
+            type=semver,pattern={{raw}}
+          flavor: |
+            latest=false
+
+      - name: Set up QEMU
+        id: qemu
+        uses: docker/setup-qemu-action@v1
+        with:
+          platforms: arm64,arm
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v1
+        with:
+          install: true
+
+      - name: Inspect builder
+        run: |
+          echo "Name:      ${{ steps.buildx.outputs.name }}"
+          echo "Endpoint:  ${{ steps.buildx.outputs.endpoint }}"
+          echo "Status:    ${{ steps.buildx.outputs.status }}"
+          echo "Flags:     ${{ steps.buildx.outputs.flags }}"
+          echo "Platforms: ${{ steps.buildx.outputs.platforms }}"
+
+      - name: Login to quay.io Container Registry
+        uses: docker/login-action@v1 
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_IO_USERNAME }}
+          password: ${{ secrets.QUAY_IO_TOKEN }}
+
+      - name: Login to docker.io Container Registry
+        uses: docker/login-action@v1 
+        with:
+          registry: docker.io
+          username: ${{ secrets.DOCKER_IO_USERNAME }}
+          password: ${{ secrets.DOCKER_IO_TOKEN }}
+
+      - name: Build and push
+        id: build-release
+        uses: docker/build-push-action@v2
+        with:
+          file: Dockerfile
+          context: .
+          platforms: linux/amd64,linux/arm64,linux/arm
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          build-args:
+
+      - name: Image digest
+        run: echo ${{ steps.build-release.outputs.digest }}

.github/workflows/e2e.yaml

@@ -0,0 +1,44 @@
+name: e2e
+
+on:
+  push:
+    branches: [ "*" ]
+    paths:
+      - '.github/workflows/e2e.yml'
+      - 'api/**'
+      - 'controllers/**'
+      - 'e2e/*'
+      - 'Dockerfile'
+      - 'go.*'
+      - 'main.go'
+      - 'Makefile'
+      - 'internal/**'
+  pull_request:
+    branches: [ "*" ]
+    paths:
+      - '.github/workflows/e2e.yml'
+      - 'api/**'
+      - 'controllers/**'
+      - 'e2e/*'
+      - 'Dockerfile'
+      - 'go.*'
+      - 'main.go'
+      - 'Makefile'
+      - 'internal/**'
+
+jobs:
+  kind:
+    name: Kubernetes
+    runs-on: ubuntu-18.04
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-go@v2
+        with:
+          go-version: '1.18'
+      - run: |
+          sudo apt-get update
+          sudo apt-get install -y golang-cfssl
+      - name: e2e testing
+        run: make e2e

.github/workflows/helm.yaml

@@ -0,0 +1,36 @@
+name: Helm Chart
+
+on:
+  push:
+    branches: [ "*" ]
+    tags: [ "helm-v" ]
+  pull_request:
+    branches: [ "*" ]
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: azure/setup-helm@v1
+        with:
+          version: 3.3.4
+      - name: Linting Chart
+        run: helm lint ./helm/kamaji
+  release:
+    if: startsWith(github.ref, 'refs/tags/helm-v')
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Publish Helm chart
+        uses: stefanprodan/helm-gh-pages@master
+        with:
+          token: ${{ secrets.BOT_GITHUB_TOKEN }}
+          charts_dir: helm
+          charts_url: https://clastix.github.io/charts
+          owner: clastix
+          repository: charts
+          branch: gh-pages
+          target_dir: .
+          commit_username: prometherion
+          commit_email: dario@tranchitella.eu

.golangci.yml

@@ -4,6 +4,10 @@ linters-settings:
       - standard
       - default
       - prefix(github.com/clastix/kamaji)
+  goheader:
+    template: |-
+      Copyright 2022 Clastix Labs
+      SPDX-License-Identifier: Apache-2.0
 
 linters:
   disable:

Dockerfile

@@ -1,5 +1,13 @@
 # Build the manager binary
-FROM golang:1.17 as builder
+FROM golang:1.18 as builder
+
+ARG TARGETARCH
+ARG GIT_HEAD_COMMIT
+ARG GIT_TAG_COMMIT
+ARG GIT_LAST_TAG
+ARG GIT_MODIFIED
+ARG GIT_REPO
+ARG BUILD_DATE
 
 WORKDIR /workspace
 # Copy the Go Modules manifests
@@ -16,7 +24,9 @@ COPY controllers/ controllers/
 COPY internal/ internal/
 
 # Build
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o manager main.go
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=$TARGETARCH go build \
+    -ldflags "-X github.com/clastix/kamaji/internal.GitRepo=$GIT_REPO -X github.com/clastix/kamaji/internal.GitTag=$GIT_LAST_TAG -X github.com/clastix/kamaji/internal.GitCommit=$GIT_HEAD_COMMIT -X github.com/clastix/kamaji/internal.GitDirty=$GIT_MODIFIED -X github.com/clastix/kamaji/internal.BuildTime=$BUILD_DATE" \
+    -a -o manager main.go
 
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details

Makefile

@@ -36,11 +36,9 @@ IMAGE_TAG_BASE ?= clastix.io/operator
 BUNDLE_IMG ?= $(IMAGE_TAG_BASE)-bundle:v$(VERSION)
 
 # Image URL to use all building/pushing image targets
-IMG ?= controller:latest
+IMG ?= clastix/kamaji:latest
 # Produce CRDs that work back to Kubernetes 1.11 (no version conversion)
 CRD_OPTIONS ?= "crd:trivialVersions=true,preserveUnknownFields=false"
-# ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary.
-ENVTEST_K8S_VERSION = 1.21
 
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
 ifeq (,$(shell go env GOBIN))
@@ -50,7 +48,6 @@ GOBIN=$(shell go env GOBIN)
 endif
 
 # Setting SHELL to bash allows bash commands to be executed by recipes.
-# This is a requirement for 'setup-envtest.sh' in the test target.
 # Options are set to exit when a recipe line exits non-zero or a piped command fails.
 SHELL = /usr/bin/env bash -o pipefail
 .SHELLFLAGS = -ec
@@ -73,33 +70,65 @@ all: build
 help: ## Display this help.
 	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf "  \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
 
+##@ Binary
+
+.PHONY: helm
+HELM = $(shell pwd)/bin/helm
+helm: ## Download helm locally if necessary.
+	$(call go-install-tool,$(HELM),helm.sh/helm/v3/cmd/helm@v3.9.0)
+
+GINKGO = $(shell pwd)/bin/ginkgo
+ginkgo: ## Download ginkgo locally if necessary.
+	$(call go-install-tool,$(GINKGO),github.com/onsi/ginkgo/ginkgo@v1.16.5)
+
+KIND = $(shell pwd)/bin/kind
+kind: ## Download kind locally if necessary.
+	$(call go-install-tool,$(KIND),sigs.k8s.io/kind/cmd/kind@v0.14.0)
+
+CONTROLLER_GEN = $(shell pwd)/bin/controller-gen
+controller-gen: ## Download controller-gen locally if necessary.
+	$(call go-install-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.6.1)
+
+KUSTOMIZE = $(shell pwd)/bin/kustomize
+kustomize: ## Download kustomize locally if necessary.
+	$(call install-kustomize,$(KUSTOMIZE),3.8.7)
+
 ##@ Development
 
 manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects.
 	$(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=manager-role webhook paths="./..." output:crd:artifacts:config=config/crd/bases
+	cp config/crd/bases/kamaji.clastix.io_tenantcontrolplanes.yaml helm/kamaji/crds/tenantcontrolplane.yaml
 
 generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations.
 	$(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..."
 
-fmt: ## Run go fmt against code.
-	go fmt ./...
-
-vet: ## Run go vet against code.
-	go vet ./...
-
-test: manifests generate fmt vet envtest ## Run tests.
-	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" go test ./... -coverprofile cover.out
+test:
+	go test ./... -coverprofile cover.out
 
 ##@ Build
 
+# Get information about git current status
+GIT_HEAD_COMMIT ?= $$(git rev-parse --short HEAD)
+GIT_TAG_COMMIT  ?= $$(git rev-parse --short $(VERSION))
+GIT_MODIFIED_1  ?= $$(git diff $(GIT_HEAD_COMMIT) $(GIT_TAG_COMMIT) --quiet && echo "" || echo ".dev")
+GIT_MODIFIED_2  ?= $$(git diff --quiet && echo "" || echo ".dirty")
+GIT_MODIFIED    ?= $$(echo "$(GIT_MODIFIED_1)$(GIT_MODIFIED_2)")
+GIT_REPO        ?= $$(git config --get remote.origin.url)
+BUILD_DATE      ?= $$(git log -1 --format="%at" | xargs -I{} date -d @{} +%Y-%m-%dT%H:%M:%S)
+
 build: generate fmt vet ## Build manager binary.
 	go build -o bin/manager main.go
 
 run: manifests generate fmt vet ## Run a controller from your host.
 	go run ./main.go
 
-docker-build: test ## Build docker image with the manager.
-	docker build -t ${IMG} .
+docker-build: ## Build docker image with the manager.
+	docker build -t ${IMG} . --build-arg GIT_HEAD_COMMIT=$(GIT_HEAD_COMMIT) \
+		--build-arg GIT_TAG_COMMIT=$(GIT_TAG_COMMIT) \
+		--build-arg GIT_MODIFIED=$(GIT_MODIFIED) \
+		--build-arg GIT_REPO=$(GIT_REPO) \
+		--build-arg GIT_LAST_TAG=$(VERSION) \
+		--build-arg BUILD_DATE=$(BUILD_DATE)
 
 docker-push: ## Push docker image with the manager.
 	docker push ${IMG}
@@ -109,8 +138,8 @@ docker-push: ## Push docker image with the manager.
 dev: generate manifests uninstall install rbac ## Full installation for development purposes
 	go fmt ./...
 
-load: dev docker-build
-	kind load docker-image --name kamaji ${IMG}
+load: docker-build kind
+	$(KIND) load docker-image --name kamaji ${IMG}
 
 rbac: manifests kustomize ## Install RBAC into the K8s cluster specified in ~/.kube/config.
 	$(KUSTOMIZE) build config/rbac | kubectl apply -f -
@@ -131,33 +160,6 @@ undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/confi
 yaml-installation-file: manifests kustomize ## Create yaml installation file
 	$(KUSTOMIZE) build config/default > config/install.yaml
 
-
-CONTROLLER_GEN = $(shell pwd)/bin/controller-gen
-controller-gen: ## Download controller-gen locally if necessary.
-	$(call go-get-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.6.1)
-
-KUSTOMIZE = $(shell pwd)/bin/kustomize
-kustomize: ## Download kustomize locally if necessary.
-	$(call go-get-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/v3@v3.8.7)
-
-ENVTEST = $(shell pwd)/bin/setup-envtest
-envtest: ## Download envtest-setup locally if necessary.
-	$(call go-get-tool,$(ENVTEST),sigs.k8s.io/controller-runtime/tools/setup-envtest@latest)
-
-# go-get-tool will 'go get' any package $2 and install it to $1.
-PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST))))
-define go-get-tool
-@[ -f $(1) ] || { \
-set -e ;\
-TMP_DIR=$$(mktemp -d) ;\
-cd $$TMP_DIR ;\
-go mod init tmp ;\
-echo "Downloading $(2)" ;\
-GOBIN=$(PROJECT_DIR)/bin go get $(2) ;\
-rm -rf $$TMP_DIR ;\
-}
-endef
-
 .PHONY: bundle
 bundle: manifests kustomize ## Generate bundle manifests and metadata, then validate generated files.
 	operator-sdk generate kustomize manifests -q
@@ -213,3 +215,34 @@ catalog-build: opm ## Build a catalog image.
 .PHONY: catalog-push
 catalog-push: ## Push a catalog image.
 	$(MAKE) docker-push IMG=$(CATALOG_IMG)
+
+define install-kustomize
+@[ -f $(1) ] || { \
+set -e ;\
+echo "Installing v$(2)" ;\
+cd bin ;\
+wget "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" ;\
+bash ./install_kustomize.sh $(2) ;\
+}
+endef
+
+# go-install-tool will 'go install' any package $2 and install it to $1.
+PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST))))
+define go-install-tool
+@[ -f $(1) ] || { \
+set -e ;\
+echo "Installing $(2)" ;\
+GOBIN=$(PROJECT_DIR)/bin go install $(2) ;\
+}
+endef
+
+.PHONY: env
+env:
+	@make -C deploy/kind kind ingress-nginx
+
+##@ e2e
+
+.PHONY: e2e
+e2e: env load helm ginkgo ## Create a KinD cluster, install Kamaji on it and run the test suite.
+	$(HELM) upgrade --debug --install kamaji ./helm/kamaji --create-namespace --namespace kamaji-system --set "image.pullPolicy=Never"
+	$(GINKGO) -v ./e2e

README.md

@@ -42,20 +42,30 @@ A dedicated `etcd` cluster for each tenant cluster doesn’t scale well for a ma
 
 With this solution, the resiliency is guaranteed by the usual `etcd` mechanism, and the pods' count remains under control, so it solves the main goal of resiliency and costs optimization. The trade-off here is that we have to operate an external `etcd` cluster and manage the access to be sure that each tenant cluster uses only its data. Also, there are limits in size in `etcd`, defaulted to 2GB and configurable to a maximum of 8GB. We’re solving this issue by pooling multiple `etcd` and sharding the tenant control planes.
 
+## Getting started
+
+Please refer to the [Getting Started guide](./docs/getting-started-with-kamaji.md) to deploy a minimal setup of Kamaji on KinD.
+
 ## Use cases
-Kamaji project has been initially started as a solution for actual and common problems such as minimizing the Total Cost of Ownership while running Kubernetes at scale. However, it can open a wider range of use cases. Here are a few:
+Kamaji project has been initially started as a solution for actual and common problems such as minimizing the Total Cost of Ownership while running Kubernetes at large scale. However, it can open a wider range of use cases. Here are a few:
 
 ### Managed Kubernetes
 Enabling companies to provide Cloud Native Infrastructure with ease by introducing a strong separation of concerns between management and workloads. Centralize clusters management, monitoring, and observability by leaving developers to focus on the applications, increase productivity and reduce operational costs. 
 
+### Kubernetes as a Service
+Provide Kubernetes clusters in a self-service fashion by running management and workloads on different infrastructures and cost centers with the option of Bring Your Own Device - BYOD.
+
 ### Control Plane as a Service
-Provide Kubernetes control plane in a self-service fashion by running management and workloads on different infrastructures and cost centers with the option of Bring Your Own Device - BYOD.
+Provide multiple Kubernetes control planes running on top of a single Kubernetes cluster. Tenants who use namespaces based isolation often still need access to cluster wide resources like Cluster Roles, Admission Webhooks, or Custom Resource Definitions.
 
 ### Edge Computing
 Distribute Kubernetes workloads across edge computing locations without having to manage multiple clusters across various providers. Centralize management of hundreds of control planes while leaving workloads to run isolated on their own dedicated infrastructure.
 
-### Cluster Simulations
-Test a new Kubernetes API or experimental flag or a new tool without impacting production operations. Kamaji will let you simulate such things in a safe and controlled environment.
+### Cluster Simulation
+Check new Kubernetes API or experimental flag or a new tool without impacting production operations. Kamaji will let you simulate such things in a safe and controlled environment.
+
+### Workloads Testing
+Check the behaviour of your workloads on different and multiple versions of Kubernetes with ease by deploying multiple Control Planes in a single cluster.
 
 ## Features
 
@@ -88,7 +98,6 @@ Tenant clusters are fully CNCF compliant built with upstream Kubernetes binaries
 - [ ] `kine` integration, i.e. use MySQL, SQLite, PostgreSQL as datastore
 - [ ] Deeper `kubeadm` integration
 - [ ] `etcd` pooling
-- [ ] Tenant Control Planes sharding
 
 ## Documentation
 Please, check the project's [documentation](./docs/) for getting started with Kamaji.
@@ -96,6 +105,9 @@ Please, check the project's [documentation](./docs/) for getting started with Ka
 ## Contributions
 Kamaji is Open Source with Apache 2 license and any contribution is welcome.
 
+## Community
+Join the [Kubernetes Slack Workspace](https://slack.k8s.io/) and the [`#kamaji`](https://kubernetes.slack.com/archives/C03GLTTMWNN) channel to meet end-users and contributors.
+
 ## FAQ
 Q. What does Kamaji means?
 
@@ -119,4 +131,4 @@ A. Lighter Multi-Tenancy solutions, like Capsule shares the Kubernetes control p
 
 Q. So I need a costly cloud infrastructure to try Kamaji?
 
-A. No, it is possible to try Kamaji on your laptop with [KinD](./deploy/kind/README.md).
+A. No, it is possible to getting started Kamaji on your laptop with [KinD](./docs/getting-started-with-kamaji.md).

api/v1alpha1/tenantcontrolplane_addons_funcs.go

@@ -0,0 +1,17 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package v1alpha1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func (in AddonStatus) GetChecksum() string {
+	return in.Checksum
+}
+
+func (in *AddonStatus) SetChecksum(checksum string) {
+	in.LastUpdate = metav1.Now()
+	in.Checksum = checksum
+}

api/v1alpha1/tenantcontrolplane_funcs.go

@@ -6,14 +6,41 @@ package v1alpha1
 import (
 	"context"
 	"fmt"
+	"net"
+	"strconv"
 
 	"github.com/pkg/errors"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	kamajierrors "github.com/clastix/kamaji/internal/errors"
 )
 
-func (in *TenantControlPlane) GetAddress(ctx context.Context, client client.Client) (string, error) {
+// AssignedControlPlaneAddress returns the announced address and port of a Tenant Control Plane.
+// In case of non-well formed values, or missing announcement, an error is returned.
+func (in *TenantControlPlane) AssignedControlPlaneAddress() (string, int32, error) {
+	if len(in.Status.ControlPlaneEndpoint) == 0 {
+		return "", 0, fmt.Errorf("the Tenant Control Plane is not yet exposed")
+	}
+
+	address, portString, err := net.SplitHostPort(in.Status.ControlPlaneEndpoint)
+	if err != nil {
+		return "", 0, errors.Wrap(err, "cannot split host port from Tenant Control Plane endpoint")
+	}
+
+	port, err := strconv.Atoi(portString)
+	if err != nil {
+		return "", 0, errors.Wrap(err, "cannot convert Tenant Control Plane port from endpoint")
+	}
+
+	return address, int32(port), nil
+}
+
+// DeclaredControlPlaneAddress returns the desired Tenant Control Plane address.
+// In case of dynamic allocation, e.g. using a Load Balancer, it queries the API Server looking for the allocated IP.
+// When an IP has not been yet assigned, or it is expected, an error is returned.
+func (in *TenantControlPlane) DeclaredControlPlaneAddress(ctx context.Context, client client.Client) (string, error) {
 	var loadBalancerStatus corev1.LoadBalancerStatus
 
 	svc := &corev1.Service{}
@@ -26,10 +53,12 @@ func (in *TenantControlPlane) GetAddress(ctx context.Context, client client.Clie
 	case len(in.Spec.NetworkProfile.Address) > 0:
 		// Returning the hard-coded value in the specification in case of non LoadBalanced resources
 		return in.Spec.NetworkProfile.Address, nil
+	case svc.Spec.Type == corev1.ServiceTypeClusterIP:
+		return svc.Spec.ClusterIP, nil
 	case svc.Spec.Type == corev1.ServiceTypeLoadBalancer:
 		loadBalancerStatus = svc.Status.LoadBalancer
 		if len(loadBalancerStatus.Ingress) == 0 {
-			return "", fmt.Errorf("cannot retrieve the TenantControlPlane address, Service resource is not yet exposed as LoadBalancer")
+			return "", kamajierrors.NonExposedLoadBalancerError{}
 		}
 
 		for _, lb := range loadBalancerStatus.Ingress {
@@ -39,5 +68,5 @@ func (in *TenantControlPlane) GetAddress(ctx context.Context, client client.Clie
 		}
 	}
 
-	return "", fmt.Errorf("the actual resource doesn't have yet a valid IP address")
+	return "", kamajierrors.MissingValidIPError{}
 }

api/v1alpha1/tenantcontrolplane_interfaces.go

@@ -0,0 +1,12 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package v1alpha1
+
+// KubeadmConfigChecksumDependant is the interface used to retrieve the checksum of the kubeadm phases and addons
+// configuration, required to validate the changes and, upon from that, perform the required reconciliation.
+// +kubebuilder:object:generate=false
+type KubeadmConfigChecksumDependant interface {
+	GetChecksum() string
+	SetChecksum(string)
+}

api/v1alpha1/tenantcontrolplane_kubeadmphase_funcs.go

@@ -0,0 +1,17 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package v1alpha1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func (in KubeadmPhaseStatus) GetChecksum() string {
+	return in.Checksum
+}
+
+func (in *KubeadmPhaseStatus) SetChecksum(checksum string) {
+	in.LastUpdate = metav1.Now()
+	in.Checksum = checksum
+}

api/v1alpha1/tenantcontrolplane_status.go

@@ -0,0 +1,242 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package v1alpha1
+
+import (
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	networkingv1 "k8s.io/api/networking/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/clastix/kamaji/internal/etcd"
+)
+
+// APIServerCertificatesStatus defines the observed state of ETCD Certificate for API server.
+type APIServerCertificatesStatus struct {
+	SecretName string      `json:"secretName,omitempty"`
+	LastUpdate metav1.Time `json:"lastUpdate,omitempty"`
+}
+
+// ETCDCertificateStatus defines the observed state of ETCD Certificate for API server.
+type ETCDCertificateStatus struct {
+	SecretName string      `json:"secretName,omitempty"`
+	LastUpdate metav1.Time `json:"lastUpdate,omitempty"`
+}
+
+// ETCDCertificatesStatus defines the observed state of ETCD Certificate for API server.
+type ETCDCertificatesStatus struct {
+	APIServer ETCDCertificateStatus `json:"apiServer,omitempty"`
+	CA        ETCDCertificateStatus `json:"ca,omitempty"`
+}
+
+// CertificatePrivateKeyPairStatus defines the status.
+type CertificatePrivateKeyPairStatus struct {
+	SecretName      string      `json:"secretName,omitempty"`
+	LastUpdate      metav1.Time `json:"lastUpdate,omitempty"`
+	ResourceVersion string      `json:"resourceVersion,omitempty"`
+}
+
+// PublicKeyPrivateKeyPairStatus defines the status.
+type PublicKeyPrivateKeyPairStatus struct {
+	SecretName string      `json:"secretName,omitempty"`
+	LastUpdate metav1.Time `json:"lastUpdate,omitempty"`
+}
+
+// CertificatesStatus defines the observed state of ETCD Certificates.
+type CertificatesStatus struct {
+	CA                     CertificatePrivateKeyPairStatus `json:"ca,omitempty"`
+	APIServer              CertificatePrivateKeyPairStatus `json:"apiServer,omitempty"`
+	APIServerKubeletClient CertificatePrivateKeyPairStatus `json:"apiServerKubeletClient,omitempty"`
+	FrontProxyCA           CertificatePrivateKeyPairStatus `json:"frontProxyCA,omitempty"`
+	FrontProxyClient       CertificatePrivateKeyPairStatus `json:"frontProxyClient,omitempty"`
+	SA                     PublicKeyPrivateKeyPairStatus   `json:"sa,omitempty"`
+	ETCD                   *ETCDCertificatesStatus         `json:"etcd,omitempty"`
+}
+
+// ETCDStatus defines the observed state of ETCDStatus.
+type ETCDStatus struct {
+	Role etcd.Role `json:"role,omitempty"`
+	User etcd.User `json:"user,omitempty"`
+}
+
+type SQLCertificateStatus struct {
+	SecretName      string      `json:"secretName,omitempty"`
+	ResourceVersion string      `json:"resourceVersion,omitempty"`
+	LastUpdate      metav1.Time `json:"lastUpdate,omitempty"`
+}
+
+type SQLConfigStatus struct {
+	SecretName      string `json:"secretName,omitempty"`
+	ResourceVersion string `json:"resourceVersion,omitempty"`
+}
+
+type SQLSetupStatus struct {
+	Schema                   string      `json:"schema,omitempty"`
+	User                     string      `json:"user,omitempty"`
+	LastUpdate               metav1.Time `json:"lastUpdate,omitempty"`
+	SQLConfigResourceVersion string      `json:"sqlConfigResourceVersion,omitempty"`
+}
+
+type KineMySQLStatus struct {
+	Config      SQLConfigStatus      `json:"config,omitempty"`
+	Setup       SQLSetupStatus       `json:"setup,omitempty"`
+	Certificate SQLCertificateStatus `json:"certificate,omitempty"`
+}
+
+// StorageStatus defines the observed state of StorageStatus.
+type StorageStatus struct {
+	ETCD      *ETCDStatus      `json:"etcd,omitempty"`
+	KineMySQL *KineMySQLStatus `json:"kineMySQL,omitempty"`
+}
+
+// KubeconfigStatus contains information about the generated kubeconfig.
+type KubeconfigStatus struct {
+	SecretName string      `json:"secretName,omitempty"`
+	LastUpdate metav1.Time `json:"lastUpdate,omitempty"`
+	Checksum   string      `json:"checksum,omitempty"`
+}
+
+// KubeconfigsStatus stores information about all the generated kubeconfig resources.
+type KubeconfigsStatus struct {
+	Admin             KubeconfigStatus `json:"admin,omitempty"`
+	ControllerManager KubeconfigStatus `json:"controllerManager,omitempty"`
+	Scheduler         KubeconfigStatus `json:"scheduler,omitempty"`
+}
+
+// KubeadmConfigStatus contains the status of the configuration required by kubeadm.
+type KubeadmConfigStatus struct {
+	ConfigmapName string      `json:"configmapName,omitempty"`
+	LastUpdate    metav1.Time `json:"lastUpdate,omitempty"`
+	// Checksum of the kubeadm configuration to detect changes
+	Checksum string `json:"checksum,omitempty"`
+}
+
+// KubeadmPhaseStatus contains the status of a kubeadm phase action.
+type KubeadmPhaseStatus struct {
+	Checksum   string      `json:"checksum,omitempty"`
+	LastUpdate metav1.Time `json:"lastUpdate,omitempty"`
+}
+
+// KubeadmPhasesStatus contains the status of the different kubeadm phases action.
+type KubeadmPhasesStatus struct {
+	UploadConfigKubeadm KubeadmPhaseStatus `json:"uploadConfigKubeadm"`
+	UploadConfigKubelet KubeadmPhaseStatus `json:"uploadConfigKubelet"`
+	BootstrapToken      KubeadmPhaseStatus `json:"bootstrapToken"`
+}
+
+type ExternalKubernetesObjectStatus struct {
+	Name      string `json:"name,omitempty"`
+	Namespace string `json:"namespace,omitempty"`
+	// Resource version of k8s object
+	RV string `json:"resourceVersion,omitempty"`
+	// Last time when k8s object was updated
+	LastUpdate metav1.Time `json:"lastUpdate,omitempty"`
+}
+
+// KonnectivityStatus defines the status of Konnectivity as Addon.
+type KonnectivityStatus struct {
+	Enabled                     bool                            `json:"enabled"`
+	EgressSelectorConfiguration string                          `json:"egressSelectorConfiguration,omitempty"`
+	Certificate                 CertificatePrivateKeyPairStatus `json:"certificate,omitempty"`
+	Kubeconfig                  KubeconfigStatus                `json:"kubeconfig,omitempty"`
+	ServiceAccount              ExternalKubernetesObjectStatus  `json:"sa,omitempty"`
+	ClusterRoleBinding          ExternalKubernetesObjectStatus  `json:"clusterrolebinding,omitempty"`
+	Agent                       ExternalKubernetesObjectStatus  `json:"agent,omitempty"`
+	Service                     KubernetesServiceStatus         `json:"service,omitempty"`
+}
+
+// AddonStatus defines the observed state of an Addon.
+type AddonStatus struct {
+	Enabled    bool        `json:"enabled"`
+	Checksum   string      `json:"checksum,omitempty"`
+	LastUpdate metav1.Time `json:"lastUpdate,omitempty"`
+}
+
+// AddonsStatus defines the observed state of the different Addons.
+type AddonsStatus struct {
+	CoreDNS   AddonStatus `json:"coreDNS,omitempty"`
+	KubeProxy AddonStatus `json:"kubeProxy,omitempty"`
+
+	Konnectivity KonnectivityStatus `json:"konnectivity,omitempty"`
+}
+
+// TenantControlPlaneStatus defines the observed state of TenantControlPlane.
+type TenantControlPlaneStatus struct {
+	// Storage Status contains information about Kubernetes storage system
+	Storage StorageStatus `json:"storage,omitempty"`
+	// Certificates contains information about the different certificates
+	// that are necessary to run a kubernetes control plane
+	Certificates CertificatesStatus `json:"certificates,omitempty"`
+	// KubeConfig contains information about the kubenconfigs that control plane pieces need
+	KubeConfig KubeconfigsStatus `json:"kubeconfig,omitempty"`
+	// Kubernetes contains information about the reconciliation of the required Kubernetes resources deployed in the admin cluster
+	Kubernetes KubernetesStatus `json:"kubernetesResources,omitempty"`
+	// KubeadmConfig contains the status of the configuration required by kubeadm
+	KubeadmConfig KubeadmConfigStatus `json:"kubeadmconfig,omitempty"`
+	// KubeadmPhase contains the status of the kubeadm phases action
+	KubeadmPhase KubeadmPhasesStatus `json:"kubeadmPhase,omitempty"`
+	// ControlPlaneEndpoint contains the status of the kubernetes control plane
+	ControlPlaneEndpoint string `json:"controlPlaneEndpoint,omitempty"`
+	// Addons contains the status of the different Addons
+	Addons AddonsStatus `json:"addons,omitempty"`
+}
+
+// KubernetesStatus defines the status of the resources deployed in the management cluster,
+// such as Deployment and Service.
+type KubernetesStatus struct {
+	// KubernetesVersion contains the information regarding the running Kubernetes version, and its upgrade status.
+	Version    KubernetesVersion          `json:"version,omitempty"`
+	Deployment KubernetesDeploymentStatus `json:"deployment,omitempty"`
+	Service    KubernetesServiceStatus    `json:"service,omitempty"`
+	Ingress    KubernetesIngressStatus    `json:"ingress,omitempty"`
+}
+
+// +kubebuilder:validation:Enum=Provisioning;Upgrading;Ready;NotReady
+type KubernetesVersionStatus string
+
+var (
+	VersionProvisioning KubernetesVersionStatus = "Provisioning"
+	VersionUpgrading    KubernetesVersionStatus = "Upgrading"
+	VersionReady        KubernetesVersionStatus = "Ready"
+	VersionNotReady     KubernetesVersionStatus = "NotReady"
+)
+
+type KubernetesVersion struct {
+	// Version is the running Kubernetes version of the Tenant Control Plane.
+	Version string `json:"version,omitempty"`
+	// +kubebuilder:default=Provisioning
+	// Status returns the current status of the Kubernetes version, such as its provisioning state, or completed upgrade.
+	Status *KubernetesVersionStatus `json:"status"`
+}
+
+// KubernetesDeploymentStatus defines the status for the Tenant Control Plane Deployment in the management cluster.
+type KubernetesDeploymentStatus struct {
+	appsv1.DeploymentStatus `json:",inline"`
+	// The name of the Deployment for the given cluster.
+	Name string `json:"name"`
+	// The namespace which the Deployment for the given cluster is deployed.
+	Namespace string `json:"namespace"`
+	// Last time when deployment was updated
+	LastUpdate metav1.Time `json:"lastUpdate,omitempty"`
+}
+
+// KubernetesServiceStatus defines the status for the Tenant Control Plane Service in the management cluster.
+type KubernetesServiceStatus struct {
+	corev1.ServiceStatus `json:",inline"`
+	// The name of the Service for the given cluster.
+	Name string `json:"name"`
+	// The namespace which the Service for the given cluster is deployed.
+	Namespace string `json:"namespace"`
+	// The port where the service is running
+	Port int32 `json:"port"`
+}
+
+// KubernetesIngressStatus defines the status for the Tenant Control Plane Ingress in the management cluster.
+type KubernetesIngressStatus struct {
+	networkingv1.IngressStatus `json:",inline"`
+	// The name of the Ingress for the given cluster.
+	Name string `json:"name"`
+	// The namespace which the Ingress for the given cluster is deployed.
+	Namespace string `json:"namespace"`
+}

api/v1alpha1/tenantcontrolplane_types.go

@@ -4,12 +4,8 @@
 package v1alpha1
 
 import (
-	appv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
-	networkingv1 "k8s.io/api/networking/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-
-	"github.com/clastix/kamaji/internal/etcd"
 )
 
 // NetworkProfileSpec defines the desired state of NetworkProfile.
@@ -22,15 +18,18 @@ type NetworkProfileSpec struct {
 	AllowAddressAsExternalIP bool `json:"allowAddressAsExternalIP,omitempty"`
 	// Port where API server of will be exposed
 	// +kubebuilder:default=6443
-	Port int32 `json:"port"`
-
-	// Domain of the tenant control plane
-	Domain string `json:"domain"`
+	Port int32 `json:"port,omitempty"`
+	// CertSANs sets extra Subject Alternative Names (SANs) for the API Server signing certificate.
+	// Use this field to add additional hostnames when exposing the Tenant Control Plane with third solutions.
+	CertSANs []string `json:"certSANs,omitempty"`
 	// Kubernetes Service
-	ServiceCIDR string `json:"serviceCidr"`
+	// +kubebuilder:default="10.96.0.0/16"
+	ServiceCIDR string `json:"serviceCidr,omitempty"`
 	// CIDR for Kubernetes Pods
-	PodCIDR       string   `json:"podCidr"`
-	DNSServiceIPs []string `json:"dnsServiceIPs"`
+	// +kubebuilder:default="10.244.0.0/16"
+	PodCIDR string `json:"podCidr,omitempty"`
+	// +kubebuilder:default={"10.96.0.10"}
+	DNSServiceIPs []string `json:"dnsServiceIPs,omitempty"`
 }
 
 type KubeletSpec struct {
@@ -78,10 +77,31 @@ type IngressSpec struct {
 	Hostname string `json:"hostname,omitempty"`
 }
 
+type ControlPlaneComponentsResources struct {
+	APIServer         *corev1.ResourceRequirements `json:"apiServer,omitempty"`
+	ControllerManager *corev1.ResourceRequirements `json:"controllerManager,omitempty"`
+	Scheduler         *corev1.ResourceRequirements `json:"scheduler,omitempty"`
+}
+
 type DeploymentSpec struct {
 	// +kubebuilder:default=2
-	Replicas           int32              `json:"replicas,omitempty"`
-	AdditionalMetadata AdditionalMetadata `json:"additionalMetadata,omitempty"`
+	Replicas int32 `json:"replicas,omitempty"`
+	// Resources defines the amount of memory and CPU to allocate to each component of the Control Plane
+	// (kube-apiserver, controller-manager, and scheduler).
+	Resources *ControlPlaneComponentsResources `json:"resources,omitempty"`
+	// ExtraArgs allows adding additional arguments to the Control Plane components,
+	// such as kube-apiserver, controller-manager, and scheduler.
+	ExtraArgs          *ControlPlaneExtraArgs `json:"extraArgs,omitempty"`
+	AdditionalMetadata AdditionalMetadata     `json:"additionalMetadata,omitempty"`
+}
+
+// ControlPlaneExtraArgs allows specifying additional arguments to the Control Plane components.
+type ControlPlaneExtraArgs struct {
+	APIServer         []string `json:"apiServer,omitempty"`
+	ControllerManager []string `json:"controllerManager,omitempty"`
+	Scheduler         []string `json:"scheduler,omitempty"`
+	// Available only if Kamaji is running using Kine as backing storage.
+	Kine []string `json:"kine,omitempty"`
 }
 
 type ServiceSpec struct {
@@ -90,178 +110,42 @@ type ServiceSpec struct {
 	ServiceType ServiceType `json:"serviceType"`
 }
 
+// AddonSpec defines the spec for every addon.
+type AddonSpec struct{}
+
+// KonnectivitySpec defines the spec for Konnectivity.
+type KonnectivitySpec struct {
+	// Port of Konnectivity proxy server.
+	ProxyPort int32 `json:"proxyPort"`
+	// Version for Konnectivity server and agent.
+	// +kubebuilder:default=v0.0.31
+	Version string `json:"version,omitempty"`
+	// ServerImage defines the container image for Konnectivity's server.
+	// +kubebuilder:default=us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-server
+	ServerImage string `json:"serverImage,omitempty"`
+	// AgentImage defines the container image for Konnectivity's agent.
+	// +kubebuilder:default=us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-agent
+	AgentImage string `json:"agentImage,omitempty"`
+	// Resources define the amount of CPU and memory to allocate to the Konnectivity server.
+	Resources *corev1.ResourceRequirements `json:"resources,omitempty"`
+}
+
+// AddonsSpec defines the enabled addons and their features.
+type AddonsSpec struct {
+	CoreDNS      *AddonSpec        `json:"coreDNS,omitempty"`
+	Konnectivity *KonnectivitySpec `json:"konnectivity,omitempty"`
+	KubeProxy    *AddonSpec        `json:"kubeProxy,omitempty"`
+}
+
 // TenantControlPlaneSpec defines the desired state of TenantControlPlane.
 type TenantControlPlaneSpec struct {
 	ControlPlane ControlPlane `json:"controlPlane"`
-
 	// Kubernetes specification for tenant control plane
 	Kubernetes KubernetesSpec `json:"kubernetes"`
-
 	// NetworkProfile specifies how the network is
 	NetworkProfile NetworkProfileSpec `json:"networkProfile,omitempty"`
-}
-
-// ETCDAPIServerCertificate defines the observed state of ETCD Certificate for API server.
-type APIServerCertificatesStatus struct {
-	SecretName string      `json:"secretName,omitempty"`
-	LastUpdate metav1.Time `json:"lastUpdate,omitempty"`
-}
-
-// ETCDAPIServerCertificate defines the observed state of ETCD Certificate for API server.
-type ETCDCertificateStatus struct {
-	SecretName string      `json:"secretName,omitempty"`
-	LastUpdate metav1.Time `json:"lastUpdate,omitempty"`
-}
-
-// ETCDAPIServerCertificate defines the observed state of ETCD Certificate for API server.
-type ETCDCertificatesStatus struct {
-	APIServer ETCDCertificateStatus `json:"apiServer,omitempty"`
-	CA        ETCDCertificateStatus `json:"ca,omitempty"`
-}
-
-// CertificatePrivateKeyPair defines the status.
-type CertificatePrivateKeyPairStatus struct {
-	SecretName string      `json:"secretName,omitempty"`
-	LastUpdate metav1.Time `json:"lastUpdate,omitempty"`
-}
-
-// CertificatePrivateKeyPair defines the status.
-type PublicKeyPrivateKeyPairStatus struct {
-	SecretName string      `json:"secretName,omitempty"`
-	LastUpdate metav1.Time `json:"lastUpdate,omitempty"`
-}
-
-// ETCDCertificates defines the observed state of ETCD Certificates.
-type CertificatesStatus struct {
-	CA                     CertificatePrivateKeyPairStatus `json:"ca,omitempty"`
-	APIServer              CertificatePrivateKeyPairStatus `json:"apiServer,omitempty"`
-	APIServerKubeletClient CertificatePrivateKeyPairStatus `json:"apiServerKubeletClient,omitempty"`
-	FrontProxyCA           CertificatePrivateKeyPairStatus `json:"frontProxyCA,omitempty"`
-	FrontProxyClient       CertificatePrivateKeyPairStatus `json:"frontProxyClient,omitempty"`
-	SA                     PublicKeyPrivateKeyPairStatus   `json:"sa,omitempty"`
-	ETCD                   *ETCDCertificatesStatus         `json:"etcd,omitempty"`
-}
-
-// ETCDStatus defines the observed state of ETCDStatus.
-type ETCDStatus struct {
-	Role etcd.Role `json:"role,omitempty"`
-	User etcd.User `json:"user,omitempty"`
-}
-
-// StorageStatus defines the observed state of StorageStatus.
-type StorageStatus struct {
-	ETCD *ETCDStatus `json:"etcd,omitempty"`
-}
-
-// TenantControlPlaneKubeconfigsStatus contains information about a the generated kubeconfig.
-type KubeconfigStatus struct {
-	SecretName string      `json:"secretName,omitempty"`
-	LastUpdate metav1.Time `json:"lastUpdate,omitempty"`
-}
-
-// TenantControlPlaneKubeconfigsStatus stores information about all the generated kubeconfigs.
-type KubeconfigsStatus struct {
-	Admin             KubeconfigStatus `json:"admin,omitempty"`
-	ControllerManager KubeconfigStatus `json:"controlerManager,omitempty"`
-	Scheduler         KubeconfigStatus `json:"scheduler,omitempty"`
-}
-
-// KubeadmConfigStatus contains the status of the configuration required by kubeadm.
-type KubeadmConfigStatus struct {
-	ConfigmapName   string      `json:"configmapName,omitempty"`
-	LastUpdate      metav1.Time `json:"lastUpdate,omitempty"`
-	ResourceVersion string      `json:"resourceVersion"`
-}
-
-// KubeadmPhasesStatus contains the status of of a kubeadm phase action.
-type KubeadmPhaseStatus struct {
-	KubeadmConfigResourceVersion string      `json:"kubeadmConfigResourceVersion,omitempty"`
-	LastUpdate                   metav1.Time `json:"lastUpdate,omitempty"`
-}
-
-// KubeadmPhasesStatus contains the status of the different kubeadm phases action.
-type KubeadmPhasesStatus struct {
-	UploadConfigKubeadm KubeadmPhaseStatus `json:"uploadConfigKubeadm"`
-	UploadConfigKubelet KubeadmPhaseStatus `json:"uploadConfigKubelet"`
-	AddonCoreDNS        KubeadmPhaseStatus `json:"addonCoreDNS"`
-	AddonKubeProxy      KubeadmPhaseStatus `json:"addonKubeProxy"`
-	BootstrapToken      KubeadmPhaseStatus `json:"bootstrapToken"`
-}
-
-// TenantControlPlaneStatus defines the observed state of TenantControlPlane.
-type TenantControlPlaneStatus struct {
-	// Storage Status contains information about Kubernetes storage system
-	Storage StorageStatus `json:"storage,omitempty"`
-	// Certificates contains information about the different certificates
-	// that are necessary to run a kubernetes control plane
-	Certificates CertificatesStatus `json:"certificates,omitempty"`
-	// KubeConfig contains information about the kubenconfigs that control plane pieces need
-	KubeConfig KubeconfigsStatus `json:"kubeconfig,omitempty"`
-	// Kubernetes contains information about the reconciliation of the required Kubernetes resources deployed in the admin cluster
-	Kubernetes KubernetesStatus `json:"kubernetesResources,omitempty"`
-	// KubeadmConfig contains the status of the configuration required by kubeadm
-	KubeadmConfig KubeadmConfigStatus `json:"kubeadmconfig,omitempty"`
-	// KubeadmPhase contains the status of the kubeadm phases action
-	KubeadmPhase KubeadmPhasesStatus `json:"kubeadmPhase,omitempty"`
-	// ControlPlaneEndpoint contains the status of the kubernetes control plane
-	ControlPlaneEndpoint string `json:"controlPlaneEndpoint,omitempty"`
-}
-
-// KubernetesStatus defines the status of the resources deployed in the management cluster,
-// such as Deployment and Service.
-type KubernetesStatus struct {
-	// KubernetesVersion contains the information regarding the running Kubernetes version, and its upgrade status.
-	Version    KubernetesVersion          `json:"version,omitempty"`
-	Deployment KubernetesDeploymentStatus `json:"deployment,omitempty"`
-	Service    KubernetesServiceStatus    `json:"service,omitempty"`
-	Ingress    KubernetesIngressStatus    `json:"ingress,omitempty"`
-}
-
-// +kubebuilder:validation:Enum=Provisioning;Upgrading;Ready;NotReady
-type KubernetesVersionStatus string
-
-var (
-	VersionProvisioning KubernetesVersionStatus = "Provisioning"
-	VersionUpgrading    KubernetesVersionStatus = "Upgrading"
-	VersionReady        KubernetesVersionStatus = "Ready"
-	VersionNotReady     KubernetesVersionStatus = "NotReady"
-)
-
-type KubernetesVersion struct {
-	// Version is the running Kubernetes version of the Tenant Control Plane.
-	Version string `json:"version,omitempty"`
-	// +kubebuilder:default=Provisioning
-	// Status returns the current status of the Kubernetes version, such as its provisioning state, or completed upgrade.
-	Status *KubernetesVersionStatus `json:"status"`
-}
-
-// KubernetesDeploymentStatus defines the status for the Tenant Control Plane Deployment in the management cluster.
-type KubernetesDeploymentStatus struct {
-	appv1.DeploymentStatus `json:",inline"`
-	// The name of the Deployment for the given cluster.
-	Name string `json:"name"`
-	// The namespace which the Deployment for the given cluster is deployed.
-	Namespace string `json:"namespace"`
-}
-
-// KubernetesServiceStatus defines the status for the Tenant Control Plane Service in the management cluster.
-type KubernetesServiceStatus struct {
-	corev1.ServiceStatus `json:",inline"`
-	// The name of the Service for the given cluster.
-	Name string `json:"name"`
-	// The namespace which the Service for the given cluster is deployed.
-	Namespace string `json:"namespace"`
-	// The port where the service is running
-	Port int32 `json:"port"`
-}
-
-// KubernetesIngressStatus defines the status for the Tenant Control Plane Ingress in the management cluster.
-type KubernetesIngressStatus struct {
-	networkingv1.IngressStatus `json:",inline"`
-	// The name of the Ingress for the given cluster.
-	Name string `json:"name"`
-	// The namespace which the Ingress for the given cluster is deployed.
-	Namespace string `json:"namespace"`
+	// Addons contain which addons are enabled
+	Addons AddonsSpec `json:"addons,omitempty"`
 }
 
 // +kubebuilder:object:root=true

api/v1alpha1/zz_generated.deepcopy.go

@@ -9,6 +9,7 @@
 package v1alpha1
 
 import (
+	"k8s.io/api/core/v1"
 	runtime "k8s.io/apimachinery/pkg/runtime"
 )
 
@@ -57,6 +58,85 @@ func (in *AdditionalMetadata) DeepCopy() *AdditionalMetadata {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *AddonSpec) DeepCopyInto(out *AddonSpec) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AddonSpec.
+func (in *AddonSpec) DeepCopy() *AddonSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(AddonSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *AddonStatus) DeepCopyInto(out *AddonStatus) {
+	*out = *in
+	in.LastUpdate.DeepCopyInto(&out.LastUpdate)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AddonStatus.
+func (in *AddonStatus) DeepCopy() *AddonStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(AddonStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *AddonsSpec) DeepCopyInto(out *AddonsSpec) {
+	*out = *in
+	if in.CoreDNS != nil {
+		in, out := &in.CoreDNS, &out.CoreDNS
+		*out = new(AddonSpec)
+		**out = **in
+	}
+	if in.Konnectivity != nil {
+		in, out := &in.Konnectivity, &out.Konnectivity
+		*out = new(KonnectivitySpec)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.KubeProxy != nil {
+		in, out := &in.KubeProxy, &out.KubeProxy
+		*out = new(AddonSpec)
+		**out = **in
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AddonsSpec.
+func (in *AddonsSpec) DeepCopy() *AddonsSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(AddonsSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *AddonsStatus) DeepCopyInto(out *AddonsStatus) {
+	*out = *in
+	in.CoreDNS.DeepCopyInto(&out.CoreDNS)
+	in.KubeProxy.DeepCopyInto(&out.KubeProxy)
+	in.Konnectivity.DeepCopyInto(&out.Konnectivity)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AddonsStatus.
+func (in *AddonsStatus) DeepCopy() *AddonsStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(AddonsStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in AdmissionControllers) DeepCopyInto(out *AdmissionControllers) {
 	{
@@ -136,9 +216,84 @@ func (in *ControlPlane) DeepCopy() *ControlPlane {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ControlPlaneComponentsResources) DeepCopyInto(out *ControlPlaneComponentsResources) {
+	*out = *in
+	if in.APIServer != nil {
+		in, out := &in.APIServer, &out.APIServer
+		*out = new(v1.ResourceRequirements)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.ControllerManager != nil {
+		in, out := &in.ControllerManager, &out.ControllerManager
+		*out = new(v1.ResourceRequirements)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.Scheduler != nil {
+		in, out := &in.Scheduler, &out.Scheduler
+		*out = new(v1.ResourceRequirements)
+		(*in).DeepCopyInto(*out)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ControlPlaneComponentsResources.
+func (in *ControlPlaneComponentsResources) DeepCopy() *ControlPlaneComponentsResources {
+	if in == nil {
+		return nil
+	}
+	out := new(ControlPlaneComponentsResources)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ControlPlaneExtraArgs) DeepCopyInto(out *ControlPlaneExtraArgs) {
+	*out = *in
+	if in.APIServer != nil {
+		in, out := &in.APIServer, &out.APIServer
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
+	if in.ControllerManager != nil {
+		in, out := &in.ControllerManager, &out.ControllerManager
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
+	if in.Scheduler != nil {
+		in, out := &in.Scheduler, &out.Scheduler
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
+	if in.Kine != nil {
+		in, out := &in.Kine, &out.Kine
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ControlPlaneExtraArgs.
+func (in *ControlPlaneExtraArgs) DeepCopy() *ControlPlaneExtraArgs {
+	if in == nil {
+		return nil
+	}
+	out := new(ControlPlaneExtraArgs)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *DeploymentSpec) DeepCopyInto(out *DeploymentSpec) {
 	*out = *in
+	if in.Resources != nil {
+		in, out := &in.Resources, &out.Resources
+		*out = new(ControlPlaneComponentsResources)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.ExtraArgs != nil {
+		in, out := &in.ExtraArgs, &out.ExtraArgs
+		*out = new(ControlPlaneExtraArgs)
+		(*in).DeepCopyInto(*out)
+	}
 	in.AdditionalMetadata.DeepCopyInto(&out.AdditionalMetadata)
 }
 
@@ -202,6 +357,22 @@ func (in *ETCDStatus) DeepCopy() *ETCDStatus {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ExternalKubernetesObjectStatus) DeepCopyInto(out *ExternalKubernetesObjectStatus) {
+	*out = *in
+	in.LastUpdate.DeepCopyInto(&out.LastUpdate)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExternalKubernetesObjectStatus.
+func (in *ExternalKubernetesObjectStatus) DeepCopy() *ExternalKubernetesObjectStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(ExternalKubernetesObjectStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *IngressSpec) DeepCopyInto(out *IngressSpec) {
 	*out = *in
@@ -218,6 +389,65 @@ func (in *IngressSpec) DeepCopy() *IngressSpec {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *KineMySQLStatus) DeepCopyInto(out *KineMySQLStatus) {
+	*out = *in
+	out.Config = in.Config
+	in.Setup.DeepCopyInto(&out.Setup)
+	in.Certificate.DeepCopyInto(&out.Certificate)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KineMySQLStatus.
+func (in *KineMySQLStatus) DeepCopy() *KineMySQLStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(KineMySQLStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *KonnectivitySpec) DeepCopyInto(out *KonnectivitySpec) {
+	*out = *in
+	if in.Resources != nil {
+		in, out := &in.Resources, &out.Resources
+		*out = new(v1.ResourceRequirements)
+		(*in).DeepCopyInto(*out)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KonnectivitySpec.
+func (in *KonnectivitySpec) DeepCopy() *KonnectivitySpec {
+	if in == nil {
+		return nil
+	}
+	out := new(KonnectivitySpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *KonnectivityStatus) DeepCopyInto(out *KonnectivityStatus) {
+	*out = *in
+	in.Certificate.DeepCopyInto(&out.Certificate)
+	in.Kubeconfig.DeepCopyInto(&out.Kubeconfig)
+	in.ServiceAccount.DeepCopyInto(&out.ServiceAccount)
+	in.ClusterRoleBinding.DeepCopyInto(&out.ClusterRoleBinding)
+	in.Agent.DeepCopyInto(&out.Agent)
+	in.Service.DeepCopyInto(&out.Service)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KonnectivityStatus.
+func (in *KonnectivityStatus) DeepCopy() *KonnectivityStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(KonnectivityStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *KubeadmConfigStatus) DeepCopyInto(out *KubeadmConfigStatus) {
 	*out = *in
@@ -255,8 +485,6 @@ func (in *KubeadmPhasesStatus) DeepCopyInto(out *KubeadmPhasesStatus) {
 	*out = *in
 	in.UploadConfigKubeadm.DeepCopyInto(&out.UploadConfigKubeadm)
 	in.UploadConfigKubelet.DeepCopyInto(&out.UploadConfigKubelet)
-	in.AddonCoreDNS.DeepCopyInto(&out.AddonCoreDNS)
-	in.AddonKubeProxy.DeepCopyInto(&out.AddonKubeProxy)
 	in.BootstrapToken.DeepCopyInto(&out.BootstrapToken)
 }
 
@@ -323,6 +551,7 @@ func (in *KubeletSpec) DeepCopy() *KubeletSpec {
 func (in *KubernetesDeploymentStatus) DeepCopyInto(out *KubernetesDeploymentStatus) {
 	*out = *in
 	in.DeploymentStatus.DeepCopyInto(&out.DeploymentStatus)
+	in.LastUpdate.DeepCopyInto(&out.LastUpdate)
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubernetesDeploymentStatus.
@@ -430,6 +659,11 @@ func (in *KubernetesVersion) DeepCopy() *KubernetesVersion {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *NetworkProfileSpec) DeepCopyInto(out *NetworkProfileSpec) {
 	*out = *in
+	if in.CertSANs != nil {
+		in, out := &in.CertSANs, &out.CertSANs
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
 	if in.DNSServiceIPs != nil {
 		in, out := &in.DNSServiceIPs, &out.DNSServiceIPs
 		*out = make([]string, len(*in))
@@ -463,6 +697,53 @@ func (in *PublicKeyPrivateKeyPairStatus) DeepCopy() *PublicKeyPrivateKeyPairStat
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *SQLCertificateStatus) DeepCopyInto(out *SQLCertificateStatus) {
+	*out = *in
+	in.LastUpdate.DeepCopyInto(&out.LastUpdate)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SQLCertificateStatus.
+func (in *SQLCertificateStatus) DeepCopy() *SQLCertificateStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(SQLCertificateStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *SQLConfigStatus) DeepCopyInto(out *SQLConfigStatus) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SQLConfigStatus.
+func (in *SQLConfigStatus) DeepCopy() *SQLConfigStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(SQLConfigStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *SQLSetupStatus) DeepCopyInto(out *SQLSetupStatus) {
+	*out = *in
+	in.LastUpdate.DeepCopyInto(&out.LastUpdate)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SQLSetupStatus.
+func (in *SQLSetupStatus) DeepCopy() *SQLSetupStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(SQLSetupStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ServiceSpec) DeepCopyInto(out *ServiceSpec) {
 	*out = *in
@@ -487,6 +768,11 @@ func (in *StorageStatus) DeepCopyInto(out *StorageStatus) {
 		*out = new(ETCDStatus)
 		(*in).DeepCopyInto(*out)
 	}
+	if in.KineMySQL != nil {
+		in, out := &in.KineMySQL, &out.KineMySQL
+		*out = new(KineMySQLStatus)
+		(*in).DeepCopyInto(*out)
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StorageStatus.
@@ -564,6 +850,7 @@ func (in *TenantControlPlaneSpec) DeepCopyInto(out *TenantControlPlaneSpec) {
 	in.ControlPlane.DeepCopyInto(&out.ControlPlane)
 	in.Kubernetes.DeepCopyInto(&out.Kubernetes)
 	in.NetworkProfile.DeepCopyInto(&out.NetworkProfile)
+	in.Addons.DeepCopyInto(&out.Addons)
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TenantControlPlaneSpec.
@@ -585,6 +872,7 @@ func (in *TenantControlPlaneStatus) DeepCopyInto(out *TenantControlPlaneStatus)
 	in.Kubernetes.DeepCopyInto(&out.Kubernetes)
 	in.KubeadmConfig.DeepCopyInto(&out.KubeadmConfig)
 	in.KubeadmPhase.DeepCopyInto(&out.KubeadmPhase)
+	in.Addons.DeepCopyInto(&out.Addons)
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TenantControlPlaneStatus.

config/crd/bases/kamaji.clastix.io_tenantcontrolplanes.yaml

@@ -60,6 +60,68 @@ spec:
           spec:
             description: TenantControlPlaneSpec defines the desired state of TenantControlPlane.
             properties:
+              addons:
+                description: Addons contain which addons are enabled
+                properties:
+                  coreDNS:
+                    description: AddonSpec defines the spec for every addon.
+                    type: object
+                  konnectivity:
+                    description: KonnectivitySpec defines the spec for Konnectivity.
+                    properties:
+                      agentImage:
+                        default: us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-agent
+                        description: AgentImage defines the container image for Konnectivity's
+                          agent.
+                        type: string
+                      proxyPort:
+                        description: Port of Konnectivity proxy server.
+                        format: int32
+                        type: integer
+                      resources:
+                        description: Resources define the amount of CPU and memory
+                          to allocate to the Konnectivity server.
+                        properties:
+                          limits:
+                            additionalProperties:
+                              anyOf:
+                              - type: integer
+                              - type: string
+                              pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                              x-kubernetes-int-or-string: true
+                            description: 'Limits describes the maximum amount of compute
+                              resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                            type: object
+                          requests:
+                            additionalProperties:
+                              anyOf:
+                              - type: integer
+                              - type: string
+                              pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                              x-kubernetes-int-or-string: true
+                            description: 'Requests describes the minimum amount of
+                              compute resources required. If Requests is omitted for
+                              a container, it defaults to Limits if that is explicitly
+                              specified, otherwise to an implementation-defined value.
+                              More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                            type: object
+                        type: object
+                      serverImage:
+                        default: us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-server
+                        description: ServerImage defines the container image for Konnectivity's
+                          server.
+                        type: string
+                      version:
+                        default: v0.0.31
+                        description: Version for Konnectivity server and agent.
+                        type: string
+                    required:
+                    - proxyPort
+                    type: object
+                  kubeProxy:
+                    description: AddonSpec defines the spec for every addon.
+                    type: object
+                type: object
               controlPlane:
                 description: ControlPlane defines how the Tenant Control Plane Kubernetes
                   resources must be created in the Admin Cluster, such as the number
@@ -83,10 +145,124 @@ spec:
                               type: string
                             type: object
                         type: object
+                      extraArgs:
+                        description: ExtraArgs allows adding additional arguments
+                          to the Control Plane components, such as kube-apiserver,
+                          controller-manager, and scheduler.
+                        properties:
+                          apiServer:
+                            items:
+                              type: string
+                            type: array
+                          controllerManager:
+                            items:
+                              type: string
+                            type: array
+                          kine:
+                            description: Available only if Kamaji is running using
+                              Kine as backing storage.
+                            items:
+                              type: string
+                            type: array
+                          scheduler:
+                            items:
+                              type: string
+                            type: array
+                        type: object
                       replicas:
                         default: 2
                         format: int32
                         type: integer
+                      resources:
+                        description: Resources defines the amount of memory and CPU
+                          to allocate to each component of the Control Plane (kube-apiserver,
+                          controller-manager, and scheduler).
+                        properties:
+                          apiServer:
+                            description: ResourceRequirements describes the compute
+                              resource requirements.
+                            properties:
+                              limits:
+                                additionalProperties:
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
+                                description: 'Limits describes the maximum amount
+                                  of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                                type: object
+                              requests:
+                                additionalProperties:
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
+                                description: 'Requests describes the minimum amount
+                                  of compute resources required. If Requests is omitted
+                                  for a container, it defaults to Limits if that is
+                                  explicitly specified, otherwise to an implementation-defined
+                                  value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                                type: object
+                            type: object
+                          controllerManager:
+                            description: ResourceRequirements describes the compute
+                              resource requirements.
+                            properties:
+                              limits:
+                                additionalProperties:
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
+                                description: 'Limits describes the maximum amount
+                                  of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                                type: object
+                              requests:
+                                additionalProperties:
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
+                                description: 'Requests describes the minimum amount
+                                  of compute resources required. If Requests is omitted
+                                  for a container, it defaults to Limits if that is
+                                  explicitly specified, otherwise to an implementation-defined
+                                  value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                                type: object
+                            type: object
+                          scheduler:
+                            description: ResourceRequirements describes the compute
+                              resource requirements.
+                            properties:
+                              limits:
+                                additionalProperties:
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
+                                description: 'Limits describes the maximum amount
+                                  of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                                type: object
+                              requests:
+                                additionalProperties:
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
+                                description: 'Requests describes the minimum amount
+                                  of compute resources required. If Requests is omitted
+                                  for a container, it defaults to Limits if that is
+                                  explicitly specified, otherwise to an implementation-defined
+                                  value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                                type: object
+                            type: object
+                        type: object
                     type: object
                   ingress:
                     description: Defining the options for an Optional Ingress which
@@ -246,14 +422,22 @@ spec:
                       in the section of ExternalIPs of the Kubernetes Service (only
                       ClusterIP or NodePort)
                     type: boolean
-                  dnsServiceIPs:
+                  certSANs:
+                    description: CertSANs sets extra Subject Alternative Names (SANs)
+                      for the API Server signing certificate. Use this field to add
+                      additional hostnames when exposing the Tenant Control Plane
+                      with third solutions.
+                    items:
+                      type: string
+                    type: array
+                  dnsServiceIPs:
+                    default:
+                    - 10.96.0.10
                     items:
                       type: string
                     type: array
-                  domain:
-                    description: Domain of the tenant control plane
-                    type: string
                   podCidr:
+                    default: 10.244.0.0/16
                     description: CIDR for Kubernetes Pods
                     type: string
                   port:
@@ -262,14 +446,9 @@ spec:
                     format: int32
                     type: integer
                   serviceCidr:
+                    default: 10.96.0.0/16
                     description: Kubernetes Service
                     type: string
-                required:
-                - dnsServiceIPs
-                - domain
-                - podCidr
-                - port
-                - serviceCidr
                 type: object
             required:
             - controlPlane
@@ -278,44 +457,324 @@ spec:
           status:
             description: TenantControlPlaneStatus defines the observed state of TenantControlPlane.
             properties:
+              addons:
+                description: Addons contains the status of the different Addons
+                properties:
+                  coreDNS:
+                    description: AddonStatus defines the observed state of an Addon.
+                    properties:
+                      checksum:
+                        type: string
+                      enabled:
+                        type: boolean
+                      lastUpdate:
+                        format: date-time
+                        type: string
+                    required:
+                    - enabled
+                    type: object
+                  konnectivity:
+                    description: KonnectivityStatus defines the status of Konnectivity
+                      as Addon.
+                    properties:
+                      agent:
+                        properties:
+                          lastUpdate:
+                            description: Last time when k8s object was updated
+                            format: date-time
+                            type: string
+                          name:
+                            type: string
+                          namespace:
+                            type: string
+                          resourceVersion:
+                            description: Resource version of k8s object
+                            type: string
+                        type: object
+                      certificate:
+                        description: CertificatePrivateKeyPairStatus defines the status.
+                        properties:
+                          lastUpdate:
+                            format: date-time
+                            type: string
+                          resourceVersion:
+                            type: string
+                          secretName:
+                            type: string
+                        type: object
+                      clusterrolebinding:
+                        properties:
+                          lastUpdate:
+                            description: Last time when k8s object was updated
+                            format: date-time
+                            type: string
+                          name:
+                            type: string
+                          namespace:
+                            type: string
+                          resourceVersion:
+                            description: Resource version of k8s object
+                            type: string
+                        type: object
+                      egressSelectorConfiguration:
+                        type: string
+                      enabled:
+                        type: boolean
+                      kubeconfig:
+                        description: KubeconfigStatus contains information about the
+                          generated kubeconfig.
+                        properties:
+                          checksum:
+                            type: string
+                          lastUpdate:
+                            format: date-time
+                            type: string
+                          secretName:
+                            type: string
+                        type: object
+                      sa:
+                        properties:
+                          lastUpdate:
+                            description: Last time when k8s object was updated
+                            format: date-time
+                            type: string
+                          name:
+                            type: string
+                          namespace:
+                            type: string
+                          resourceVersion:
+                            description: Resource version of k8s object
+                            type: string
+                        type: object
+                      service:
+                        description: KubernetesServiceStatus defines the status for
+                          the Tenant Control Plane Service in the management cluster.
+                        properties:
+                          conditions:
+                            description: Current service state
+                            items:
+                              description: "Condition contains details for one aspect
+                                of the current state of this API Resource. --- This
+                                struct is intended for direct use as an array at the
+                                field path .status.conditions.  For example, type
+                                FooStatus struct{     // Represents the observations
+                                of a foo's current state.     // Known .status.conditions.type
+                                are: \"Available\", \"Progressing\", and \"Degraded\"
+                                \    // +patchMergeKey=type     // +patchStrategy=merge
+                                \    // +listType=map     // +listMapKey=type     Conditions
+                                []metav1.Condition `json:\"conditions,omitempty\"
+                                patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"`
+                                \n     // other fields }"
+                              properties:
+                                lastTransitionTime:
+                                  description: lastTransitionTime is the last time
+                                    the condition transitioned from one status to
+                                    another. This should be when the underlying condition
+                                    changed.  If that is not known, then using the
+                                    time when the API field changed is acceptable.
+                                  format: date-time
+                                  type: string
+                                message:
+                                  description: message is a human readable message
+                                    indicating details about the transition. This
+                                    may be an empty string.
+                                  maxLength: 32768
+                                  type: string
+                                observedGeneration:
+                                  description: observedGeneration represents the .metadata.generation
+                                    that the condition was set based upon. For instance,
+                                    if .metadata.generation is currently 12, but the
+                                    .status.conditions[x].observedGeneration is 9,
+                                    the condition is out of date with respect to the
+                                    current state of the instance.
+                                  format: int64
+                                  minimum: 0
+                                  type: integer
+                                reason:
+                                  description: reason contains a programmatic identifier
+                                    indicating the reason for the condition's last
+                                    transition. Producers of specific condition types
+                                    may define expected values and meanings for this
+                                    field, and whether the values are considered a
+                                    guaranteed API. The value should be a CamelCase
+                                    string. This field may not be empty.
+                                  maxLength: 1024
+                                  minLength: 1
+                                  pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
+                                  type: string
+                                status:
+                                  description: status of the condition, one of True,
+                                    False, Unknown.
+                                  enum:
+                                  - "True"
+                                  - "False"
+                                  - Unknown
+                                  type: string
+                                type:
+                                  description: type of condition in CamelCase or in
+                                    foo.example.com/CamelCase. --- Many .condition.type
+                                    values are consistent across resources like Available,
+                                    but because arbitrary conditions can be useful
+                                    (see .node.status.conditions), the ability to
+                                    deconflict is important. The regex it matches
+                                    is (dns1123SubdomainFmt/)?(qualifiedNameFmt)
+                                  maxLength: 316
+                                  pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
+                                  type: string
+                              required:
+                              - lastTransitionTime
+                              - message
+                              - reason
+                              - status
+                              - type
+                              type: object
+                            type: array
+                            x-kubernetes-list-map-keys:
+                            - type
+                            x-kubernetes-list-type: map
+                          loadBalancer:
+                            description: LoadBalancer contains the current status
+                              of the load-balancer, if one is present.
+                            properties:
+                              ingress:
+                                description: Ingress is a list containing ingress
+                                  points for the load-balancer. Traffic intended for
+                                  the service should be sent to these ingress points.
+                                items:
+                                  description: 'LoadBalancerIngress represents the
+                                    status of a load-balancer ingress point: traffic
+                                    intended for the service should be sent to an
+                                    ingress point.'
+                                  properties:
+                                    hostname:
+                                      description: Hostname is set for load-balancer
+                                        ingress points that are DNS based (typically
+                                        AWS load-balancers)
+                                      type: string
+                                    ip:
+                                      description: IP is set for load-balancer ingress
+                                        points that are IP based (typically GCE or
+                                        OpenStack load-balancers)
+                                      type: string
+                                    ports:
+                                      description: Ports is a list of records of service
+                                        ports If used, every port defined in the service
+                                        should have an entry in it
+                                      items:
+                                        properties:
+                                          error:
+                                            description: 'Error is to record the problem
+                                              with the service port The format of
+                                              the error shall comply with the following
+                                              rules: - built-in error values shall
+                                              be specified in this file and those
+                                              shall use   CamelCase names - cloud
+                                              provider specific error values must
+                                              have names that comply with the   format
+                                              foo.example.com/CamelCase. --- The regex
+                                              it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)'
+                                            maxLength: 316
+                                            pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
+                                            type: string
+                                          port:
+                                            description: Port is the port number of
+                                              the service port of which status is
+                                              recorded here
+                                            format: int32
+                                            type: integer
+                                          protocol:
+                                            default: TCP
+                                            description: 'Protocol is the protocol
+                                              of the service port of which status
+                                              is recorded here The supported values
+                                              are: "TCP", "UDP", "SCTP"'
+                                            type: string
+                                        required:
+                                        - port
+                                        - protocol
+                                        type: object
+                                      type: array
+                                      x-kubernetes-list-type: atomic
+                                  type: object
+                                type: array
+                            type: object
+                          name:
+                            description: The name of the Service for the given cluster.
+                            type: string
+                          namespace:
+                            description: The namespace which the Service for the given
+                              cluster is deployed.
+                            type: string
+                          port:
+                            description: The port where the service is running
+                            format: int32
+                            type: integer
+                        required:
+                        - name
+                        - namespace
+                        - port
+                        type: object
+                    required:
+                    - enabled
+                    type: object
+                  kubeProxy:
+                    description: AddonStatus defines the observed state of an Addon.
+                    properties:
+                      checksum:
+                        type: string
+                      enabled:
+                        type: boolean
+                      lastUpdate:
+                        format: date-time
+                        type: string
+                    required:
+                    - enabled
+                    type: object
+                type: object
               certificates:
                 description: Certificates contains information about the different
                   certificates that are necessary to run a kubernetes control plane
                 properties:
                   apiServer:
-                    description: CertificatePrivateKeyPair defines the status.
+                    description: CertificatePrivateKeyPairStatus defines the status.
                     properties:
                       lastUpdate:
                         format: date-time
                         type: string
+                      resourceVersion:
+                        type: string
                       secretName:
                         type: string
                     type: object
                   apiServerKubeletClient:
-                    description: CertificatePrivateKeyPair defines the status.
+                    description: CertificatePrivateKeyPairStatus defines the status.
                     properties:
                       lastUpdate:
                         format: date-time
                         type: string
+                      resourceVersion:
+                        type: string
                       secretName:
                         type: string
                     type: object
                   ca:
-                    description: CertificatePrivateKeyPair defines the status.
+                    description: CertificatePrivateKeyPairStatus defines the status.
                     properties:
                       lastUpdate:
                         format: date-time
                         type: string
+                      resourceVersion:
+                        type: string
                       secretName:
                         type: string
                     type: object
                   etcd:
-                    description: ETCDAPIServerCertificate defines the observed state
+                    description: ETCDCertificatesStatus defines the observed state
                       of ETCD Certificate for API server.
                     properties:
                       apiServer:
-                        description: ETCDAPIServerCertificate defines the observed
-                          state of ETCD Certificate for API server.
+                        description: ETCDCertificateStatus defines the observed state
+                          of ETCD Certificate for API server.
                         properties:
                           lastUpdate:
                             format: date-time
@@ -324,8 +783,8 @@ spec:
                             type: string
                         type: object
                       ca:
-                        description: ETCDAPIServerCertificate defines the observed
-                          state of ETCD Certificate for API server.
+                        description: ETCDCertificateStatus defines the observed state
+                          of ETCD Certificate for API server.
                         properties:
                           lastUpdate:
                             format: date-time
@@ -335,25 +794,29 @@ spec:
                         type: object
                     type: object
                   frontProxyCA:
-                    description: CertificatePrivateKeyPair defines the status.
+                    description: CertificatePrivateKeyPairStatus defines the status.
                     properties:
                       lastUpdate:
                         format: date-time
                         type: string
+                      resourceVersion:
+                        type: string
                       secretName:
                         type: string
                     type: object
                   frontProxyClient:
-                    description: CertificatePrivateKeyPair defines the status.
+                    description: CertificatePrivateKeyPairStatus defines the status.
                     properties:
                       lastUpdate:
                         format: date-time
                         type: string
+                      resourceVersion:
+                        type: string
                       secretName:
                         type: string
                     type: object
                   sa:
-                    description: CertificatePrivateKeyPair defines the status.
+                    description: PublicKeyPrivateKeyPairStatus defines the status.
                     properties:
                       lastUpdate:
                         format: date-time
@@ -370,59 +833,37 @@ spec:
                 description: KubeadmPhase contains the status of the kubeadm phases
                   action
                 properties:
-                  addonCoreDNS:
-                    description: KubeadmPhasesStatus contains the status of of a kubeadm
-                      phase action.
-                    properties:
-                      kubeadmConfigResourceVersion:
-                        type: string
-                      lastUpdate:
-                        format: date-time
-                        type: string
-                    type: object
-                  addonKubeProxy:
-                    description: KubeadmPhasesStatus contains the status of of a kubeadm
-                      phase action.
-                    properties:
-                      kubeadmConfigResourceVersion:
-                        type: string
-                      lastUpdate:
-                        format: date-time
-                        type: string
-                    type: object
                   bootstrapToken:
-                    description: KubeadmPhasesStatus contains the status of of a kubeadm
+                    description: KubeadmPhaseStatus contains the status of a kubeadm
                       phase action.
                     properties:
-                      kubeadmConfigResourceVersion:
+                      checksum:
                         type: string
                       lastUpdate:
                         format: date-time
                         type: string
                     type: object
                   uploadConfigKubeadm:
-                    description: KubeadmPhasesStatus contains the status of of a kubeadm
+                    description: KubeadmPhaseStatus contains the status of a kubeadm
                       phase action.
                     properties:
-                      kubeadmConfigResourceVersion:
+                      checksum:
                         type: string
                       lastUpdate:
                         format: date-time
                         type: string
                     type: object
                   uploadConfigKubelet:
-                    description: KubeadmPhasesStatus contains the status of of a kubeadm
+                    description: KubeadmPhaseStatus contains the status of a kubeadm
                       phase action.
                     properties:
-                      kubeadmConfigResourceVersion:
+                      checksum:
                         type: string
                       lastUpdate:
                         format: date-time
                         type: string
                     type: object
                 required:
-                - addonCoreDNS
-                - addonKubeProxy
                 - bootstrapToken
                 - uploadConfigKubeadm
                 - uploadConfigKubelet
@@ -431,34 +872,37 @@ spec:
                 description: KubeadmConfig contains the status of the configuration
                   required by kubeadm
                 properties:
+                  checksum:
+                    description: Checksum of the kubeadm configuration to detect changes
+                    type: string
                   configmapName:
                     type: string
                   lastUpdate:
                     format: date-time
                     type: string
-                  resourceVersion:
-                    type: string
-                required:
-                - resourceVersion
                 type: object
               kubeconfig:
                 description: KubeConfig contains information about the kubenconfigs
                   that control plane pieces need
                 properties:
                   admin:
-                    description: TenantControlPlaneKubeconfigsStatus contains information
-                      about a the generated kubeconfig.
+                    description: KubeconfigStatus contains information about the generated
+                      kubeconfig.
                     properties:
+                      checksum:
+                        type: string
                       lastUpdate:
                         format: date-time
                         type: string
                       secretName:
                         type: string
                     type: object
-                  controlerManager:
-                    description: TenantControlPlaneKubeconfigsStatus contains information
-                      about a the generated kubeconfig.
+                  controllerManager:
+                    description: KubeconfigStatus contains information about the generated
+                      kubeconfig.
                     properties:
+                      checksum:
+                        type: string
                       lastUpdate:
                         format: date-time
                         type: string
@@ -466,9 +910,11 @@ spec:
                         type: string
                     type: object
                   scheduler:
-                    description: TenantControlPlaneKubeconfigsStatus contains information
-                      about a the generated kubeconfig.
+                    description: KubeconfigStatus contains information about the generated
+                      kubeconfig.
                     properties:
+                      checksum:
+                        type: string
                       lastUpdate:
                         format: date-time
                         type: string
@@ -531,6 +977,10 @@ spec:
                           - type
                           type: object
                         type: array
+                      lastUpdate:
+                        description: Last time when deployment was updated
+                        format: date-time
+                        type: string
                       name:
                         description: The name of the Deployment for the given cluster.
                         type: string
@@ -874,6 +1324,38 @@ spec:
                         - name
                         type: object
                     type: object
+                  kineMySQL:
+                    properties:
+                      certificate:
+                        properties:
+                          lastUpdate:
+                            format: date-time
+                            type: string
+                          resourceVersion:
+                            type: string
+                          secretName:
+                            type: string
+                        type: object
+                      config:
+                        properties:
+                          resourceVersion:
+                            type: string
+                          secretName:
+                            type: string
+                        type: object
+                      setup:
+                        properties:
+                          lastUpdate:
+                            format: date-time
+                            type: string
+                          schema:
+                            type: string
+                          sqlConfigResourceVersion:
+                            type: string
+                          user:
+                            type: string
+                        type: object
+                    type: object
                 type: object
             type: object
         type: object

config/install.yaml

@@ -60,6 +60,60 @@ spec:
           spec:
             description: TenantControlPlaneSpec defines the desired state of TenantControlPlane.
             properties:
+              addons:
+                description: Addons contain which addons are enabled
+                properties:
+                  coreDNS:
+                    description: AddonSpec defines the spec for every addon.
+                    type: object
+                  konnectivity:
+                    description: KonnectivitySpec defines the spec for Konnectivity.
+                    properties:
+                      agentImage:
+                        default: us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-agent
+                        description: AgentImage defines the container image for Konnectivity's agent.
+                        type: string
+                      proxyPort:
+                        description: Port of Konnectivity proxy server.
+                        format: int32
+                        type: integer
+                      resources:
+                        description: Resources define the amount of CPU and memory to allocate to the Konnectivity server.
+                        properties:
+                          limits:
+                            additionalProperties:
+                              anyOf:
+                              - type: integer
+                              - type: string
+                              pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                              x-kubernetes-int-or-string: true
+                            description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                            type: object
+                          requests:
+                            additionalProperties:
+                              anyOf:
+                              - type: integer
+                              - type: string
+                              pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                              x-kubernetes-int-or-string: true
+                            description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                            type: object
+                        type: object
+                      serverImage:
+                        default: us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-server
+                        description: ServerImage defines the container image for Konnectivity's server.
+                        type: string
+                      version:
+                        default: v0.0.31
+                        description: Version for Konnectivity server and agent.
+                        type: string
+                    required:
+                    - proxyPort
+                    type: object
+                  kubeProxy:
+                    description: AddonSpec defines the spec for every addon.
+                    type: object
+                type: object
               controlPlane:
                 description: ControlPlane defines how the Tenant Control Plane Kubernetes resources must be created in the Admin Cluster, such as the number of Pod replicas, the Service resource, or the Ingress.
                 properties:
@@ -78,10 +132,101 @@ spec:
                               type: string
                             type: object
                         type: object
+                      extraArgs:
+                        description: ExtraArgs allows adding additional arguments to the Control Plane components, such as kube-apiserver, controller-manager, and scheduler.
+                        properties:
+                          apiServer:
+                            items:
+                              type: string
+                            type: array
+                          controllerManager:
+                            items:
+                              type: string
+                            type: array
+                          kine:
+                            description: Available only if Kamaji is running using Kine as backing storage.
+                            items:
+                              type: string
+                            type: array
+                          scheduler:
+                            items:
+                              type: string
+                            type: array
+                        type: object
                       replicas:
                         default: 2
                         format: int32
                         type: integer
+                      resources:
+                        description: Resources defines the amount of memory and CPU to allocate to each component of the Control Plane (kube-apiserver, controller-manager, and scheduler).
+                        properties:
+                          apiServer:
+                            description: ResourceRequirements describes the compute resource requirements.
+                            properties:
+                              limits:
+                                additionalProperties:
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
+                                description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                                type: object
+                              requests:
+                                additionalProperties:
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
+                                description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                                type: object
+                            type: object
+                          controllerManager:
+                            description: ResourceRequirements describes the compute resource requirements.
+                            properties:
+                              limits:
+                                additionalProperties:
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
+                                description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                                type: object
+                              requests:
+                                additionalProperties:
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
+                                description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                                type: object
+                            type: object
+                          scheduler:
+                            description: ResourceRequirements describes the compute resource requirements.
+                            properties:
+                              limits:
+                                additionalProperties:
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
+                                description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                                type: object
+                              requests:
+                                additionalProperties:
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
+                                description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                                type: object
+                            type: object
+                        type: object
                     type: object
                   ingress:
                     description: Defining the options for an Optional Ingress which will expose API Server of the Tenant Control Plane
@@ -225,14 +370,19 @@ spec:
                   allowAddressAsExternalIP:
                     description: AllowAddressAsExternalIP will include tenantControlPlane.Spec.NetworkProfile.Address in the section of ExternalIPs of the Kubernetes Service (only ClusterIP or NodePort)
                     type: boolean
-                  dnsServiceIPs:
+                  certSANs:
+                    description: CertSANs sets extra Subject Alternative Names (SANs) for the API Server signing certificate. Use this field to add additional hostnames when exposing the Tenant Control Plane with third solutions.
+                    items:
+                      type: string
+                    type: array
+                  dnsServiceIPs:
+                    default:
+                    - 10.96.0.10
                     items:
                       type: string
                     type: array
-                  domain:
-                    description: Domain of the tenant control plane
-                    type: string
                   podCidr:
+                    default: 10.244.0.0/16
                     description: CIDR for Kubernetes Pods
                     type: string
                   port:
@@ -241,14 +391,9 @@ spec:
                     format: int32
                     type: integer
                   serviceCidr:
+                    default: 10.96.0.0/16
                     description: Kubernetes Service
                     type: string
-                required:
-                - dnsServiceIPs
-                - domain
-                - podCidr
-                - port
-                - serviceCidr
                 type: object
             required:
             - controlPlane
@@ -257,41 +402,256 @@ spec:
           status:
             description: TenantControlPlaneStatus defines the observed state of TenantControlPlane.
             properties:
+              addons:
+                description: Addons contains the status of the different Addons
+                properties:
+                  coreDNS:
+                    description: AddonStatus defines the observed state of an Addon.
+                    properties:
+                      checksum:
+                        type: string
+                      enabled:
+                        type: boolean
+                      lastUpdate:
+                        format: date-time
+                        type: string
+                    required:
+                    - enabled
+                    type: object
+                  konnectivity:
+                    description: KonnectivityStatus defines the status of Konnectivity as Addon.
+                    properties:
+                      agent:
+                        properties:
+                          lastUpdate:
+                            description: Last time when k8s object was updated
+                            format: date-time
+                            type: string
+                          name:
+                            type: string
+                          namespace:
+                            type: string
+                          resourceVersion:
+                            description: Resource version of k8s object
+                            type: string
+                        type: object
+                      certificate:
+                        description: CertificatePrivateKeyPairStatus defines the status.
+                        properties:
+                          lastUpdate:
+                            format: date-time
+                            type: string
+                          resourceVersion:
+                            type: string
+                          secretName:
+                            type: string
+                        type: object
+                      clusterrolebinding:
+                        properties:
+                          lastUpdate:
+                            description: Last time when k8s object was updated
+                            format: date-time
+                            type: string
+                          name:
+                            type: string
+                          namespace:
+                            type: string
+                          resourceVersion:
+                            description: Resource version of k8s object
+                            type: string
+                        type: object
+                      egressSelectorConfiguration:
+                        type: string
+                      enabled:
+                        type: boolean
+                      kubeconfig:
+                        description: KubeconfigStatus contains information about the generated kubeconfig.
+                        properties:
+                          checksum:
+                            type: string
+                          lastUpdate:
+                            format: date-time
+                            type: string
+                          secretName:
+                            type: string
+                        type: object
+                      sa:
+                        properties:
+                          lastUpdate:
+                            description: Last time when k8s object was updated
+                            format: date-time
+                            type: string
+                          name:
+                            type: string
+                          namespace:
+                            type: string
+                          resourceVersion:
+                            description: Resource version of k8s object
+                            type: string
+                        type: object
+                      service:
+                        description: KubernetesServiceStatus defines the status for the Tenant Control Plane Service in the management cluster.
+                        properties:
+                          conditions:
+                            description: Current service state
+                            items:
+                              description: "Condition contains details for one aspect of the current state of this API Resource. --- This struct is intended for direct use as an array at the field path .status.conditions.  For example, type FooStatus struct{     // Represents the observations of a foo's current state.     // Known .status.conditions.type are: \"Available\", \"Progressing\", and \"Degraded\"     // +patchMergeKey=type     // +patchStrategy=merge     // +listType=map     // +listMapKey=type     Conditions []metav1.Condition `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"` \n     // other fields }"
+                              properties:
+                                lastTransitionTime:
+                                  description: lastTransitionTime is the last time the condition transitioned from one status to another. This should be when the underlying condition changed.  If that is not known, then using the time when the API field changed is acceptable.
+                                  format: date-time
+                                  type: string
+                                message:
+                                  description: message is a human readable message indicating details about the transition. This may be an empty string.
+                                  maxLength: 32768
+                                  type: string
+                                observedGeneration:
+                                  description: observedGeneration represents the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date with respect to the current state of the instance.
+                                  format: int64
+                                  minimum: 0
+                                  type: integer
+                                reason:
+                                  description: reason contains a programmatic identifier indicating the reason for the condition's last transition. Producers of specific condition types may define expected values and meanings for this field, and whether the values are considered a guaranteed API. The value should be a CamelCase string. This field may not be empty.
+                                  maxLength: 1024
+                                  minLength: 1
+                                  pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
+                                  type: string
+                                status:
+                                  description: status of the condition, one of True, False, Unknown.
+                                  enum:
+                                  - "True"
+                                  - "False"
+                                  - Unknown
+                                  type: string
+                                type:
+                                  description: type of condition in CamelCase or in foo.example.com/CamelCase. --- Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be useful (see .node.status.conditions), the ability to deconflict is important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)
+                                  maxLength: 316
+                                  pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
+                                  type: string
+                              required:
+                              - lastTransitionTime
+                              - message
+                              - reason
+                              - status
+                              - type
+                              type: object
+                            type: array
+                            x-kubernetes-list-map-keys:
+                            - type
+                            x-kubernetes-list-type: map
+                          loadBalancer:
+                            description: LoadBalancer contains the current status of the load-balancer, if one is present.
+                            properties:
+                              ingress:
+                                description: Ingress is a list containing ingress points for the load-balancer. Traffic intended for the service should be sent to these ingress points.
+                                items:
+                                  description: 'LoadBalancerIngress represents the status of a load-balancer ingress point: traffic intended for the service should be sent to an ingress point.'
+                                  properties:
+                                    hostname:
+                                      description: Hostname is set for load-balancer ingress points that are DNS based (typically AWS load-balancers)
+                                      type: string
+                                    ip:
+                                      description: IP is set for load-balancer ingress points that are IP based (typically GCE or OpenStack load-balancers)
+                                      type: string
+                                    ports:
+                                      description: Ports is a list of records of service ports If used, every port defined in the service should have an entry in it
+                                      items:
+                                        properties:
+                                          error:
+                                            description: 'Error is to record the problem with the service port The format of the error shall comply with the following rules: - built-in error values shall be specified in this file and those shall use   CamelCase names - cloud provider specific error values must have names that comply with the   format foo.example.com/CamelCase. --- The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)'
+                                            maxLength: 316
+                                            pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
+                                            type: string
+                                          port:
+                                            description: Port is the port number of the service port of which status is recorded here
+                                            format: int32
+                                            type: integer
+                                          protocol:
+                                            default: TCP
+                                            description: 'Protocol is the protocol of the service port of which status is recorded here The supported values are: "TCP", "UDP", "SCTP"'
+                                            type: string
+                                        required:
+                                        - port
+                                        - protocol
+                                        type: object
+                                      type: array
+                                      x-kubernetes-list-type: atomic
+                                  type: object
+                                type: array
+                            type: object
+                          name:
+                            description: The name of the Service for the given cluster.
+                            type: string
+                          namespace:
+                            description: The namespace which the Service for the given cluster is deployed.
+                            type: string
+                          port:
+                            description: The port where the service is running
+                            format: int32
+                            type: integer
+                        required:
+                        - name
+                        - namespace
+                        - port
+                        type: object
+                    required:
+                    - enabled
+                    type: object
+                  kubeProxy:
+                    description: AddonStatus defines the observed state of an Addon.
+                    properties:
+                      checksum:
+                        type: string
+                      enabled:
+                        type: boolean
+                      lastUpdate:
+                        format: date-time
+                        type: string
+                    required:
+                    - enabled
+                    type: object
+                type: object
               certificates:
                 description: Certificates contains information about the different certificates that are necessary to run a kubernetes control plane
                 properties:
                   apiServer:
-                    description: CertificatePrivateKeyPair defines the status.
+                    description: CertificatePrivateKeyPairStatus defines the status.
                     properties:
                       lastUpdate:
                         format: date-time
                         type: string
+                      resourceVersion:
+                        type: string
                       secretName:
                         type: string
                     type: object
                   apiServerKubeletClient:
-                    description: CertificatePrivateKeyPair defines the status.
+                    description: CertificatePrivateKeyPairStatus defines the status.
                     properties:
                       lastUpdate:
                         format: date-time
                         type: string
+                      resourceVersion:
+                        type: string
                       secretName:
                         type: string
                     type: object
                   ca:
-                    description: CertificatePrivateKeyPair defines the status.
+                    description: CertificatePrivateKeyPairStatus defines the status.
                     properties:
                       lastUpdate:
                         format: date-time
                         type: string
+                      resourceVersion:
+                        type: string
                       secretName:
                         type: string
                     type: object
                   etcd:
-                    description: ETCDAPIServerCertificate defines the observed state of ETCD Certificate for API server.
+                    description: ETCDCertificatesStatus defines the observed state of ETCD Certificate for API server.
                     properties:
                       apiServer:
-                        description: ETCDAPIServerCertificate defines the observed state of ETCD Certificate for API server.
+                        description: ETCDCertificateStatus defines the observed state of ETCD Certificate for API server.
                         properties:
                           lastUpdate:
                             format: date-time
@@ -300,7 +660,7 @@ spec:
                             type: string
                         type: object
                       ca:
-                        description: ETCDAPIServerCertificate defines the observed state of ETCD Certificate for API server.
+                        description: ETCDCertificateStatus defines the observed state of ETCD Certificate for API server.
                         properties:
                           lastUpdate:
                             format: date-time
@@ -310,25 +670,29 @@ spec:
                         type: object
                     type: object
                   frontProxyCA:
-                    description: CertificatePrivateKeyPair defines the status.
+                    description: CertificatePrivateKeyPairStatus defines the status.
                     properties:
                       lastUpdate:
                         format: date-time
                         type: string
+                      resourceVersion:
+                        type: string
                       secretName:
                         type: string
                     type: object
                   frontProxyClient:
-                    description: CertificatePrivateKeyPair defines the status.
+                    description: CertificatePrivateKeyPairStatus defines the status.
                     properties:
                       lastUpdate:
                         format: date-time
                         type: string
+                      resourceVersion:
+                        type: string
                       secretName:
                         type: string
                     type: object
                   sa:
-                    description: CertificatePrivateKeyPair defines the status.
+                    description: PublicKeyPrivateKeyPairStatus defines the status.
                     properties:
                       lastUpdate:
                         format: date-time
@@ -343,54 +707,34 @@ spec:
               kubeadmPhase:
                 description: KubeadmPhase contains the status of the kubeadm phases action
                 properties:
-                  addonCoreDNS:
-                    description: KubeadmPhasesStatus contains the status of of a kubeadm phase action.
-                    properties:
-                      kubeadmConfigResourceVersion:
-                        type: string
-                      lastUpdate:
-                        format: date-time
-                        type: string
-                    type: object
-                  addonKubeProxy:
-                    description: KubeadmPhasesStatus contains the status of of a kubeadm phase action.
-                    properties:
-                      kubeadmConfigResourceVersion:
-                        type: string
-                      lastUpdate:
-                        format: date-time
-                        type: string
-                    type: object
                   bootstrapToken:
-                    description: KubeadmPhasesStatus contains the status of of a kubeadm phase action.
+                    description: KubeadmPhaseStatus contains the status of a kubeadm phase action.
                     properties:
-                      kubeadmConfigResourceVersion:
+                      checksum:
                         type: string
                       lastUpdate:
                         format: date-time
                         type: string
                     type: object
                   uploadConfigKubeadm:
-                    description: KubeadmPhasesStatus contains the status of of a kubeadm phase action.
+                    description: KubeadmPhaseStatus contains the status of a kubeadm phase action.
                     properties:
-                      kubeadmConfigResourceVersion:
+                      checksum:
                         type: string
                       lastUpdate:
                         format: date-time
                         type: string
                     type: object
                   uploadConfigKubelet:
-                    description: KubeadmPhasesStatus contains the status of of a kubeadm phase action.
+                    description: KubeadmPhaseStatus contains the status of a kubeadm phase action.
                     properties:
-                      kubeadmConfigResourceVersion:
+                      checksum:
                         type: string
                       lastUpdate:
                         format: date-time
                         type: string
                     type: object
                 required:
-                - addonCoreDNS
-                - addonKubeProxy
                 - bootstrapToken
                 - uploadConfigKubeadm
                 - uploadConfigKubelet
@@ -398,31 +742,34 @@ spec:
               kubeadmconfig:
                 description: KubeadmConfig contains the status of the configuration required by kubeadm
                 properties:
+                  checksum:
+                    description: Checksum of the kubeadm configuration to detect changes
+                    type: string
                   configmapName:
                     type: string
                   lastUpdate:
                     format: date-time
                     type: string
-                  resourceVersion:
-                    type: string
-                required:
-                - resourceVersion
                 type: object
               kubeconfig:
                 description: KubeConfig contains information about the kubenconfigs that control plane pieces need
                 properties:
                   admin:
-                    description: TenantControlPlaneKubeconfigsStatus contains information about a the generated kubeconfig.
+                    description: KubeconfigStatus contains information about the generated kubeconfig.
                     properties:
+                      checksum:
+                        type: string
                       lastUpdate:
                         format: date-time
                         type: string
                       secretName:
                         type: string
                     type: object
-                  controlerManager:
-                    description: TenantControlPlaneKubeconfigsStatus contains information about a the generated kubeconfig.
+                  controllerManager:
+                    description: KubeconfigStatus contains information about the generated kubeconfig.
                     properties:
+                      checksum:
+                        type: string
                       lastUpdate:
                         format: date-time
                         type: string
@@ -430,8 +777,10 @@ spec:
                         type: string
                     type: object
                   scheduler:
-                    description: TenantControlPlaneKubeconfigsStatus contains information about a the generated kubeconfig.
+                    description: KubeconfigStatus contains information about the generated kubeconfig.
                     properties:
+                      checksum:
+                        type: string
                       lastUpdate:
                         format: date-time
                         type: string
@@ -483,6 +832,10 @@ spec:
                           - type
                           type: object
                         type: array
+                      lastUpdate:
+                        description: Last time when deployment was updated
+                        format: date-time
+                        type: string
                       name:
                         description: The name of the Deployment for the given cluster.
                         type: string
@@ -732,6 +1085,38 @@ spec:
                         - name
                         type: object
                     type: object
+                  kineMySQL:
+                    properties:
+                      certificate:
+                        properties:
+                          lastUpdate:
+                            format: date-time
+                            type: string
+                          resourceVersion:
+                            type: string
+                          secretName:
+                            type: string
+                        type: object
+                      config:
+                        properties:
+                          resourceVersion:
+                            type: string
+                          secretName:
+                            type: string
+                        type: object
+                      setup:
+                        properties:
+                          lastUpdate:
+                            format: date-time
+                            type: string
+                          schema:
+                            type: string
+                          sqlConfigResourceVersion:
+                            type: string
+                          user:
+                            type: string
+                        type: object
+                    type: object
                 type: object
             type: object
         type: object
@@ -1021,8 +1406,8 @@ spec:
         - --leader-elect
         command:
         - /manager
-        image: quay.io/clastix/kamaji:latest
-        imagePullPolicy: IfNotPresent
+        image: clastix/kamaji:latest
+        imagePullPolicy: Always
         livenessProbe:
           httpGet:
             path: /healthz

config/manager/kustomization.yaml

@@ -12,5 +12,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 images:
 - name: controller
-  newName: quay.io/clastix/kamaji
+  newName: clastix/kamaji
   newTag: latest

config/manager/manager.yaml

@@ -30,7 +30,7 @@ spec:
         args:
         - --leader-elect
         image: controller:latest
-        imagePullPolicy: IfNotPresent
+        imagePullPolicy: Always
         name: manager
         securityContext:
           allowPrivilegeEscalation: false

config/samples/kamaji_v1alpha1_tenantcontrolplane.yaml

@@ -21,7 +21,7 @@ spec:
         labels:
           tenant.clastix.io: test
           kind.clastix.io: service
-      serviceType: ClusterIP
+      serviceType: LoadBalancer
     ingress:
       enabled: true
       hostname: kamaji.local
@@ -30,7 +30,7 @@ spec:
         annotations:
           kubernetes.io/ingress.allow-http: "false"
           nginx.ingress.kubernetes.io/secure-backends: "true"
-          nginx.ingress.kubernetes.io/ssl-passthrough: "true"      
+          nginx.ingress.kubernetes.io/ssl-passthrough: "true"
   kubernetes:
     version: "v1.23.1"
     kubelet:
@@ -41,8 +41,12 @@ spec:
   networkProfile:
     address: "127.0.0.1"
     port: 6443
-    domain: "clastix.labs"
+    certSANs:
+      - "test.clastix.labs"
     serviceCidr: "10.96.0.0/16"
     podCidr: "10.244.0.0/16"
     dnsServiceIPs:
       - "10.96.0.10"
+  addons:
+    coreDNS: {}
+    kubeProxy: {}

controllers/resources.go

@@ -0,0 +1,360 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+package controllers
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/go-logr/logr"
+	"github.com/google/uuid"
+	k8stypes "k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+	"github.com/clastix/kamaji/internal/resources"
+	"github.com/clastix/kamaji/internal/resources/konnectivity"
+	"github.com/clastix/kamaji/internal/sql"
+	"github.com/clastix/kamaji/internal/types"
+)
+
+const (
+	separator = ","
+)
+
+type GroupResourceBuilderConfiguration struct {
+	client              client.Client
+	log                 logr.Logger
+	tcpReconcilerConfig TenantControlPlaneReconcilerConfig
+	tenantControlPlane  kamajiv1alpha1.TenantControlPlane
+	DBConnection        sql.DBConnection
+}
+
+type GroupDeleteableResourceBuilderConfiguration struct {
+	client              client.Client
+	log                 logr.Logger
+	tcpReconcilerConfig TenantControlPlaneReconcilerConfig
+	tenantControlPlane  kamajiv1alpha1.TenantControlPlane
+	DBConnection        sql.DBConnection
+}
+
+// GetResources returns a list of resources that will be used to provide tenant control planes
+// Currently there is only a default approach
+// TODO: the idea of this function is to become a factory to return the group of resources according to the given configuration.
+func GetResources(config GroupResourceBuilderConfiguration) []resources.Resource {
+	return getDefaultResources(config)
+}
+
+// GetDeletableResources returns a list of resources that have to be deleted when tenant control planes are deleted
+// Currently there is only a default approach
+// TODO: the idea of this function is to become a factory to return the group of deleteable resources according to the given configuration.
+func GetDeletableResources(config GroupDeleteableResourceBuilderConfiguration) []resources.DeleteableResource {
+	return getDefaultDeleteableResources(config)
+}
+
+func getDefaultResources(config GroupResourceBuilderConfiguration) []resources.Resource {
+	resources := append(getUpgradeResources(config.client, config.tenantControlPlane), getKubernetesServiceResources(config.client, config.tenantControlPlane)...)
+	resources = append(resources, getKubeadmConfigResources(config.client, config.tcpReconcilerConfig, config.tenantControlPlane)...)
+	resources = append(resources, getKubernetesCertificatesResources(config.client, config.log, config.tcpReconcilerConfig, config.tenantControlPlane)...)
+	resources = append(resources, getKubeconfigResources(config.client, config.log, config.tcpReconcilerConfig, config.tenantControlPlane)...)
+	resources = append(resources, getKubernetesStorageResources(config.client, config.log, config.tcpReconcilerConfig, config.DBConnection, config.tenantControlPlane)...)
+	resources = append(resources, getInternalKonnectivityResources(config.client, config.log, config.tcpReconcilerConfig, config.tenantControlPlane)...)
+	resources = append(resources, getKubernetesDeploymentResources(config.client, config.tcpReconcilerConfig, config.tenantControlPlane)...)
+	resources = append(resources, getKubernetesIngressResources(config.client, config.tenantControlPlane)...)
+	resources = append(resources, getKubeadmPhaseResources(config.client, config.log, config.tenantControlPlane)...)
+	resources = append(resources, getKubeadmAddonResources(config.client, config.log, config.tenantControlPlane)...)
+	resources = append(resources, getExternalKonnectivityResources(config.client, config.log, config.tcpReconcilerConfig, config.tenantControlPlane)...)
+
+	return resources
+}
+
+func getDefaultDeleteableResources(config GroupDeleteableResourceBuilderConfiguration) []resources.DeleteableResource {
+	switch config.tcpReconcilerConfig.ETCDStorageType {
+	case types.ETCD:
+		return []resources.DeleteableResource{
+			&resources.ETCDSetupResource{
+				Name:                  "etcd-setup",
+				Client:                config.client,
+				Log:                   config.log,
+				ETCDClientCertsSecret: getNamespacedName(config.tcpReconcilerConfig.ETCDClientSecretNamespace, config.tcpReconcilerConfig.ETCDClientSecretName),
+				ETCDCACertsSecret:     getNamespacedName(config.tcpReconcilerConfig.ETCDCASecretNamespace, config.tcpReconcilerConfig.ETCDCASecretName),
+				Endpoints:             getArrayFromString(config.tcpReconcilerConfig.ETCDEndpoints),
+			},
+		}
+	case types.KineMySQL:
+		return []resources.DeleteableResource{
+			&resources.SQLSetup{
+				Client:       config.client,
+				Name:         "sql-setup",
+				DBConnection: config.DBConnection,
+			},
+		}
+	default:
+		return []resources.DeleteableResource{}
+	}
+}
+
+func getUpgradeResources(c client.Client, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource {
+	return []resources.Resource{
+		&resources.KubernetesUpgrade{
+			Name:   "upgrade",
+			Client: c,
+		},
+	}
+}
+
+func getKubernetesServiceResources(c client.Client, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource {
+	return []resources.Resource{
+		&resources.KubernetesServiceResource{
+			Client: c,
+		},
+	}
+}
+
+func getKubeadmConfigResources(c client.Client, tcpReconcilerConfig TenantControlPlaneReconcilerConfig, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource {
+	return []resources.Resource{
+		&resources.KubeadmConfigResource{
+			Name:                   "kubeadmconfig",
+			ETCDs:                  getArrayFromString(tcpReconcilerConfig.ETCDEndpoints),
+			ETCDCompactionInterval: tcpReconcilerConfig.ETCDCompactionInterval,
+			Client:                 c,
+			TmpDirectory:           getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
+		},
+	}
+}
+
+func getKubernetesCertificatesResources(c client.Client, log logr.Logger, tcpReconcilerConfig TenantControlPlaneReconcilerConfig, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource {
+	return []resources.Resource{
+		&resources.CACertificate{
+			Name:         "ca",
+			Client:       c,
+			Log:          log,
+			TmpDirectory: getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
+		},
+		&resources.FrontProxyCACertificate{
+			Name:         "front-proxy-ca-certificate",
+			Client:       c,
+			Log:          log,
+			TmpDirectory: getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
+		},
+		&resources.SACertificate{
+			Name:         "sa-certificate",
+			Client:       c,
+			Log:          log,
+			TmpDirectory: getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
+		},
+		&resources.APIServerCertificate{
+			Name:         "api-server-certificate",
+			Client:       c,
+			Log:          log,
+			TmpDirectory: getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
+		},
+		&resources.APIServerKubeletClientCertificate{
+			Name:         "api-server-kubelet-client-certificate",
+			Client:       c,
+			Log:          log,
+			TmpDirectory: getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
+		},
+		&resources.FrontProxyClientCertificate{
+			Name:         "front-proxy-client-certificate",
+			Client:       c,
+			Log:          log,
+			TmpDirectory: getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
+		},
+	}
+}
+
+func getKubeconfigResources(c client.Client, log logr.Logger, tcpReconcilerConfig TenantControlPlaneReconcilerConfig, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource {
+	return []resources.Resource{
+		&resources.KubeconfigResource{
+			Name:               "admin-kubeconfig",
+			Client:             c,
+			Log:                log,
+			KubeConfigFileName: resources.AdminKubeConfigFileName,
+			TmpDirectory:       getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
+		},
+		&resources.KubeconfigResource{
+			Name:               "controller-manager-kubeconfig",
+			Client:             c,
+			Log:                log,
+			KubeConfigFileName: resources.ControllerManagerKubeConfigFileName,
+			TmpDirectory:       getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
+		},
+		&resources.KubeconfigResource{
+			Name:               "scheduler-kubeconfig",
+			Client:             c,
+			Log:                log,
+			KubeConfigFileName: resources.SchedulerKubeConfigFileName,
+			TmpDirectory:       getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
+		},
+	}
+}
+
+func getKubernetesStorageResources(c client.Client, log logr.Logger, tcpReconcilerConfig TenantControlPlaneReconcilerConfig, dbConnection sql.DBConnection, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource {
+	switch tcpReconcilerConfig.ETCDStorageType {
+	case types.ETCD:
+		return []resources.Resource{
+			&resources.ETCDCACertificatesResource{
+				Name:                  "etcd-ca-certificates",
+				Client:                c,
+				Log:                   log,
+				ETCDCASecretName:      tcpReconcilerConfig.ETCDCASecretName,
+				ETCDCASecretNamespace: tcpReconcilerConfig.ETCDCASecretNamespace,
+			},
+			&resources.ETCDCertificatesResource{
+				Name:   "etcd-certificates",
+				Client: c,
+				Log:    log,
+			},
+			&resources.ETCDSetupResource{
+				Name:                  "etcd-setup",
+				Client:                c,
+				Log:                   log,
+				ETCDClientCertsSecret: getNamespacedName(tcpReconcilerConfig.ETCDClientSecretNamespace, tcpReconcilerConfig.ETCDClientSecretName),
+				ETCDCACertsSecret:     getNamespacedName(tcpReconcilerConfig.ETCDCASecretNamespace, tcpReconcilerConfig.ETCDCASecretName),
+				Endpoints:             getArrayFromString(tcpReconcilerConfig.ETCDEndpoints),
+			},
+		}
+	case types.KineMySQL:
+		return []resources.Resource{
+			&resources.SQLStorageConfig{
+				Client: c,
+				Name:   "sql-config",
+				Host:   dbConnection.GetHost(),
+				Port:   dbConnection.GetPort(),
+			},
+			&resources.SQLSetup{
+				Client:       c,
+				Name:         "sql-setup",
+				DBConnection: dbConnection,
+			},
+			&resources.SQLCertificate{
+				Client:                   c,
+				Name:                     "sql-certificate",
+				StorageType:              tcpReconcilerConfig.ETCDStorageType,
+				SQLConfigSecretName:      tcpReconcilerConfig.KineMySQLSecretName,
+				SQLConfigSecretNamespace: tcpReconcilerConfig.KineMySQLSecretNamespace,
+			},
+		}
+	default:
+		return []resources.Resource{}
+	}
+}
+
+func getKubernetesDeploymentResources(c client.Client, tcpReconcilerConfig TenantControlPlaneReconcilerConfig, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource {
+	return []resources.Resource{
+		&resources.KubernetesDeploymentResource{
+			Client:                 c,
+			ETCDEndpoints:          getArrayFromString(tcpReconcilerConfig.ETCDEndpoints),
+			ETCDCompactionInterval: tcpReconcilerConfig.ETCDCompactionInterval,
+			ETCDStorageType:        tcpReconcilerConfig.ETCDStorageType,
+			KineContainerImage:     tcpReconcilerConfig.KineContainerImage,
+		},
+	}
+}
+
+func getKubernetesIngressResources(c client.Client, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource {
+	return []resources.Resource{
+		&resources.KubernetesIngressResource{
+			Client: c,
+		},
+	}
+}
+
+func getKubeadmPhaseResources(c client.Client, log logr.Logger, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource {
+	return []resources.Resource{
+		&resources.KubeadmPhase{
+			Name:   "upload-config-kubeadm",
+			Client: c,
+			Log:    log,
+			Phase:  resources.PhaseUploadConfigKubeadm,
+		},
+		&resources.KubeadmPhase{
+			Name:   "upload-config-kubelet",
+			Client: c,
+			Log:    log,
+			Phase:  resources.PhaseUploadConfigKubelet,
+		},
+		&resources.KubeadmPhase{
+			Name:   "bootstrap-token",
+			Client: c,
+			Log:    log,
+			Phase:  resources.PhaseBootstrapToken,
+		},
+	}
+}
+
+func getKubeadmAddonResources(c client.Client, log logr.Logger, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource {
+	return []resources.Resource{
+		&resources.KubeadmAddonResource{
+			Name:         "coredns",
+			Client:       c,
+			Log:          log,
+			KubeadmAddon: resources.AddonCoreDNS,
+		},
+		&resources.KubeadmAddonResource{
+			Name:         "kubeproxy",
+			Client:       c,
+			Log:          log,
+			KubeadmAddon: resources.AddonKubeProxy,
+		},
+	}
+}
+
+func getExternalKonnectivityResources(c client.Client, log logr.Logger, tcpReconcilerConfig TenantControlPlaneReconcilerConfig, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource {
+	return []resources.Resource{
+		&konnectivity.ServiceAccountResource{
+			Client: c,
+			Name:   "konnectivity-sa",
+		},
+		&konnectivity.ClusterRoleBindingResource{
+			Client: c,
+			Name:   "konnectivity-clusterrolebinding",
+		},
+		&konnectivity.KubernetesDeploymentResource{
+			Client: c,
+			Name:   "konnectivity-deployment",
+		},
+		&konnectivity.ServiceResource{
+			Client: c,
+			Name:   "konnectivity-service",
+		},
+		&konnectivity.Agent{
+			Client: c,
+			Name:   "konnectivity-agent",
+		},
+	}
+}
+
+func getInternalKonnectivityResources(c client.Client, log logr.Logger, tcpReconcilerConfig TenantControlPlaneReconcilerConfig, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource {
+	return []resources.Resource{
+		&konnectivity.EgressSelectorConfigurationResource{
+			Client: c,
+			Name:   "konnectivity-egress-selector-configuration",
+		},
+		&konnectivity.CertificateResource{
+			Client: c,
+			Log:    log,
+			Name:   "konnectivity-certificate",
+		},
+		&konnectivity.KubeconfigResource{
+			Client: c,
+			Name:   "konnectivity-kubeconfig",
+		},
+	}
+}
+
+func getArrayFromString(s string) []string {
+	var a []string
+	a = append(a, strings.Split(s, separator)...)
+
+	return a
+}
+
+func getNamespacedName(namespace string, name string) k8stypes.NamespacedName {
+	return k8stypes.NamespacedName{Namespace: namespace, Name: name}
+}
+
+func getTmpDirectory(base string, tenantControlPlane kamajiv1alpha1.TenantControlPlane) string {
+	return fmt.Sprintf("%s/%s/%s", base, tenantControlPlane.GetName(), uuid.New())
+}

controllers/storage.go

@@ -0,0 +1,57 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package controllers
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"fmt"
+
+	corev1 "k8s.io/api/core/v1"
+	k8stypes "k8s.io/apimachinery/pkg/types"
+
+	"github.com/clastix/kamaji/internal/sql"
+	"github.com/clastix/kamaji/internal/types"
+)
+
+func (r *TenantControlPlaneReconciler) getStorageConnection(ctx context.Context) (sql.DBConnection, error) {
+	// TODO: https://github.com/clastix/kamaji/issues/67
+	switch r.Config.ETCDStorageType {
+	case types.KineMySQL:
+		secret := &corev1.Secret{}
+		namespacedName := k8stypes.NamespacedName{Namespace: r.Config.KineMySQLSecretNamespace, Name: r.Config.KineMySQLSecretName}
+		if err := r.Client.Get(ctx, namespacedName, secret); err != nil {
+			return nil, err
+		}
+
+		rootCAs := x509.NewCertPool()
+		if ok := rootCAs.AppendCertsFromPEM(secret.Data["ca.crt"]); !ok {
+			return nil, fmt.Errorf("error creating root ca for mysql db connector")
+		}
+
+		certificate, err := tls.X509KeyPair(secret.Data["server.crt"], secret.Data["server.key"])
+		if err != nil {
+			return nil, err
+		}
+
+		return sql.GetDBConnection(
+			sql.ConnectionConfig{
+				SQLDriver: sql.MySQL,
+				User:      "root",
+				Password:  string(secret.Data["MYSQL_ROOT_PASSWORD"]),
+				Host:      r.Config.KineMySQLHost,
+				Port:      r.Config.KineMySQLPort,
+				DBName:    "mysql",
+				TLSConfig: &tls.Config{
+					ServerName:   r.Config.KineMySQLHost,
+					RootCAs:      rootCAs,
+					Certificates: []tls.Certificate{certificate},
+				},
+			},
+		)
+	default:
+		return nil, nil
+	}
+}

controllers/tenantcontrolplane_controller.go

@@ -6,9 +6,7 @@ package controllers
 import (
 	"context"
 	"fmt"
-	"strings"
 
-	"github.com/google/uuid"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	networkingv1 "k8s.io/api/networking/v1"
@@ -21,11 +19,13 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/log"
 
 	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+	kamajierrors "github.com/clastix/kamaji/internal/errors"
 	"github.com/clastix/kamaji/internal/resources"
+	"github.com/clastix/kamaji/internal/sql"
+	"github.com/clastix/kamaji/internal/types"
 )
 
 const (
-	separator = ","
 	finalizer = "finalizer.kamaji.clastix.io"
 )
 
@@ -38,6 +38,7 @@ type TenantControlPlaneReconciler struct {
 
 // TenantControlPlaneReconcilerConfig gives the necessary configuration for TenantControlPlaneReconciler.
 type TenantControlPlaneReconcilerConfig struct {
+	ETCDStorageType           types.ETCDStorageType
 	ETCDCASecretName          string
 	ETCDCASecretNamespace     string
 	ETCDClientSecretName      string
@@ -45,6 +46,12 @@ type TenantControlPlaneReconcilerConfig struct {
 	ETCDEndpoints             string
 	ETCDCompactionInterval    string
 	TmpBaseDirectory          string
+	DBConnection              sql.DBConnection
+	KineMySQLSecretName       string
+	KineMySQLSecretNamespace  string
+	KineMySQLHost             string
+	KineMySQLPort             int
+	KineContainerImage        string
 }
 
 //+kubebuilder:rbac:groups=kamaji.clastix.io,resources=tenantcontrolplanes,verbs=get;list;watch;create;update;patch;delete
@@ -75,190 +82,71 @@ func (r *TenantControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R
 		return ctrl.Result{}, nil
 	}
 
-	if markedToBeDeleted {
-		registeredDeleteableResources := []resources.DeleteableResource{
-			&resources.ETCDSetupResource{
-				Name:                  "etcd-setup",
-				Client:                r.Client,
-				Scheme:                r.Scheme,
-				Log:                   log,
-				ETCDClientCertsSecret: getNamespacedName(r.Config.ETCDClientSecretNamespace, r.Config.ETCDClientSecretName),
-				ETCDCACertsSecret:     getNamespacedName(r.Config.ETCDCASecretNamespace, r.Config.ETCDCASecretName),
-				Endpoints:             getArrayFromString(r.Config.ETCDEndpoints),
-			},
+	dbConnection, err := r.getStorageConnection(ctx)
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+	defer func() {
+		// TODO: Currently, etcd is not accessed using this dbConnection. For that reason we need this check
+		// Check: https://github.com/clastix/kamaji/issues/67
+		if dbConnection != nil {
+			dbConnection.Close()
 		}
+	}()
 
-		for _, resource := range registeredDeleteableResources {
-			if err := resource.Delete(ctx, tenantControlPlane); err != nil {
+	if markedToBeDeleted {
+		log.Info("marked for deletion, performing clean-up")
+
+		groupDeleteableResourceBuilderConfiguration := GroupDeleteableResourceBuilderConfiguration{
+			client:              r.Client,
+			log:                 log,
+			tcpReconcilerConfig: r.Config,
+			tenantControlPlane:  *tenantControlPlane,
+			DBConnection:        dbConnection,
+		}
+		registeredDeletableResources := GetDeletableResources(groupDeleteableResourceBuilderConfiguration)
+
+		for _, resource := range registeredDeletableResources {
+			if err := resources.HandleDeletion(ctx, resource, tenantControlPlane); err != nil {
 				return ctrl.Result{}, err
 			}
 		}
 
 		if hasFinalizer {
-			controllerutil.RemoveFinalizer(tenantControlPlane, finalizer)
-			if err := r.Update(ctx, tenantControlPlane); err != nil {
+			log.Info("removing finalizer")
+
+			if err := r.RemoveFinalizer(ctx, tenantControlPlane); err != nil {
 				return ctrl.Result{}, err
 			}
 		}
 
+		log.Info("resource deletion has been completed")
+
 		return ctrl.Result{}, nil
 	}
 
 	if !hasFinalizer {
-		controllerutil.AddFinalizer(tenantControlPlane, finalizer)
-		if err := r.Update(ctx, tenantControlPlane); err != nil {
-			return ctrl.Result{}, err
-		}
-
-		return ctrl.Result{}, nil
+		return ctrl.Result{}, r.AddFinalizer(ctx, tenantControlPlane)
 	}
 
-	registeredResources := []resources.Resource{
-		&resources.KubernetesServiceResource{
-			Client: r.Client,
-		},
-		&resources.KubeadmConfigResource{
-			Name:                   "kubeadmconfig",
-			Port:                   tenantControlPlane.Spec.NetworkProfile.Port,
-			KubernetesVersion:      tenantControlPlane.Spec.Kubernetes.Version,
-			PodCIDR:                tenantControlPlane.Spec.NetworkProfile.PodCIDR,
-			ServiceCIDR:            tenantControlPlane.Spec.NetworkProfile.ServiceCIDR,
-			Domain:                 tenantControlPlane.Spec.NetworkProfile.Domain,
-			ETCDs:                  getArrayFromString(r.Config.ETCDEndpoints),
-			ETCDCompactionInterval: r.Config.ETCDCompactionInterval,
-			Client:                 r.Client,
-			Scheme:                 r.Scheme,
-			Log:                    log,
-			TmpDirectory:           getTmpDirectory(r.Config.TmpBaseDirectory, *tenantControlPlane),
-		},
-		&resources.CACertificate{
-			Name:         "ca",
-			Client:       r.Client,
-			Log:          log,
-			TmpDirectory: getTmpDirectory(r.Config.TmpBaseDirectory, *tenantControlPlane),
-		},
-		&resources.FrontProxyCACertificate{
-			Name:         "front-proxy-ca-certificate",
-			Client:       r.Client,
-			Log:          log,
-			TmpDirectory: getTmpDirectory(r.Config.TmpBaseDirectory, *tenantControlPlane),
-		},
-		&resources.SACertificate{
-			Name:         "sa-certificate",
-			Client:       r.Client,
-			Log:          log,
-			TmpDirectory: getTmpDirectory(r.Config.TmpBaseDirectory, *tenantControlPlane),
-		},
-		&resources.APIServerCertificate{
-			Name:         "api-server-certificate",
-			Client:       r.Client,
-			Log:          log,
-			TmpDirectory: getTmpDirectory(r.Config.TmpBaseDirectory, *tenantControlPlane),
-		},
-		&resources.APIServerKubeletClientCertificate{
-			Name:         "api-server-kubelet-client-certificate",
-			Client:       r.Client,
-			Log:          log,
-			TmpDirectory: getTmpDirectory(r.Config.TmpBaseDirectory, *tenantControlPlane),
-		},
-		&resources.FrontProxyClientCertificate{
-			Name:         "front-proxy-client-certificate",
-			Client:       r.Client,
-			Log:          log,
-			TmpDirectory: getTmpDirectory(r.Config.TmpBaseDirectory, *tenantControlPlane),
-		},
-		&resources.KubeconfigResource{
-			Name:               "admin-kubeconfig",
-			Client:             r.Client,
-			Scheme:             r.Scheme,
-			Log:                log,
-			KubeConfigFileName: resources.AdminKubeConfigFileName,
-			TmpDirectory:       getTmpDirectory(r.Config.TmpBaseDirectory, *tenantControlPlane),
-		},
-		&resources.KubeconfigResource{
-			Name:               "controller-manager-kubeconfig",
-			Client:             r.Client,
-			Scheme:             r.Scheme,
-			Log:                log,
-			KubeConfigFileName: resources.ControllerManagerKubeConfigFileName,
-			TmpDirectory:       getTmpDirectory(r.Config.TmpBaseDirectory, *tenantControlPlane),
-		},
-		&resources.KubeconfigResource{
-			Name:               "scheduler-kubeconfig",
-			Client:             r.Client,
-			Scheme:             r.Scheme,
-			Log:                log,
-			KubeConfigFileName: resources.SchedulerKubeConfigFileName,
-			TmpDirectory:       getTmpDirectory(r.Config.TmpBaseDirectory, *tenantControlPlane),
-		},
-		&resources.ETCDCACertificatesResource{
-			Name:                  "etcd-ca-certificates",
-			Client:                r.Client,
-			Log:                   log,
-			ETCDCASecretName:      r.Config.ETCDCASecretName,
-			ETCDCASecretNamespace: r.Config.ETCDCASecretNamespace,
-		},
-		&resources.ETCDCertificatesResource{
-			Name:   "etcd-certificates",
-			Client: r.Client,
-			Log:    log,
-		},
-		&resources.ETCDSetupResource{
-			Name:                  "etcd-setup",
-			Client:                r.Client,
-			Scheme:                r.Scheme,
-			Log:                   log,
-			ETCDClientCertsSecret: getNamespacedName(r.Config.ETCDClientSecretNamespace, r.Config.ETCDClientSecretName),
-			ETCDCACertsSecret:     getNamespacedName(r.Config.ETCDCASecretNamespace, r.Config.ETCDCASecretName),
-			Endpoints:             getArrayFromString(r.Config.ETCDEndpoints),
-		},
-		&resources.KubernetesUpgrade{
-			Name:   "upgrade",
-			Client: r.Client,
-		},
-		&resources.KubernetesDeploymentResource{
-			Client:                 r.Client,
-			ETCDEndpoints:          getArrayFromString(r.Config.ETCDEndpoints),
-			ETCDCompactionInterval: r.Config.ETCDCompactionInterval,
-		},
-		&resources.KubernetesIngressResource{
-			Client: r.Client,
-		},
-		&resources.KubeadmPhaseResource{
-			Name:         "upload-config-kubeadm",
-			Client:       r.Client,
-			Log:          log,
-			KubeadmPhase: resources.PhaseUploadConfigKubeadm,
-		},
-		&resources.KubeadmPhaseResource{
-			Name:         "upload-config-kubelet",
-			Client:       r.Client,
-			Log:          log,
-			KubeadmPhase: resources.PhaseUploadConfigKubelet,
-		},
-		&resources.KubeadmPhaseResource{
-			Name:         "addon-coredns",
-			Client:       r.Client,
-			Log:          log,
-			KubeadmPhase: resources.PhaseAddonCoreDNS,
-		},
-		&resources.KubeadmPhaseResource{
-			Name:         "addon-kubeproxy",
-			Client:       r.Client,
-			Log:          log,
-			KubeadmPhase: resources.PhaseAddonKubeProxy,
-		},
-		&resources.KubeadmPhaseResource{
-			Name:         "bootstrap-token",
-			Client:       r.Client,
-			Log:          log,
-			KubeadmPhase: resources.PhaseBootstrapToken,
-		},
+	groupResourceBuilderConfiguration := GroupResourceBuilderConfiguration{
+		client:              r.Client,
+		log:                 log,
+		tcpReconcilerConfig: r.Config,
+		tenantControlPlane:  *tenantControlPlane,
+		DBConnection:        dbConnection,
 	}
+	registeredResources := GetResources(groupResourceBuilderConfiguration)
 
 	for _, resource := range registeredResources {
 		result, err := resources.Handle(ctx, resource, tenantControlPlane)
 		if err != nil {
+			if kamajierrors.ShouldReconcileErrorBeIgnored(err) {
+				log.V(1).Info("sentinel error, enqueuing back request", "error", err.Error())
+
+				return ctrl.Result{Requeue: true}, nil
+			}
+
 			return ctrl.Result{}, err
 		}
 
@@ -275,6 +163,8 @@ func (r *TenantControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R
 		return ctrl.Result{}, nil
 	}
 
+	log.Info(fmt.Sprintf("%s has been reconciled", tenantControlPlane.GetName()))
+
 	return ctrl.Result{}, nil
 }
 
@@ -324,21 +214,6 @@ func (r *TenantControlPlaneReconciler) updateStatus(ctx context.Context, namespa
 	return nil
 }
 
-func getArrayFromString(s string) []string {
-	var a []string
-	a = append(a, strings.Split(s, separator)...)
-
-	return a
-}
-
-func getNamespacedName(namespace string, name string) k8stypes.NamespacedName {
-	return k8stypes.NamespacedName{Namespace: namespace, Name: name}
-}
-
-func getTmpDirectory(base string, tenantControlPlane kamajiv1alpha1.TenantControlPlane) string {
-	return fmt.Sprintf("%s/%s/%s", base, tenantControlPlane.GetName(), uuid.New())
-}
-
 func hasFinalizer(tenantControlPlane kamajiv1alpha1.TenantControlPlane) bool {
 	for _, f := range tenantControlPlane.GetFinalizers() {
 		if f == finalizer {
@@ -348,3 +223,15 @@ func hasFinalizer(tenantControlPlane kamajiv1alpha1.TenantControlPlane) bool {
 
 	return false
 }
+
+func (r *TenantControlPlaneReconciler) AddFinalizer(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+	controllerutil.AddFinalizer(tenantControlPlane, finalizer)
+
+	return r.Update(ctx, tenantControlPlane)
+}
+
+func (r *TenantControlPlaneReconciler) RemoveFinalizer(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+	controllerutil.RemoveFinalizer(tenantControlPlane, finalizer)
+
+	return r.Update(ctx, tenantControlPlane)
+}

deploy/Makefile

@@ -0,0 +1,16 @@
+include etcd/Makefile
+
+deploy_path := $(patsubst %/,%,$(dir $(abspath $(lastword $(MAKEFILE_LIST)))))
+
+.DEFAULT_GOAL := kamaji
+
+.PHONY: etcd-cluster
+reqs: etcd-cluster
+
+.PHONY: kamaji
+kamaji: reqs
+	@kubectl apply -f $(deploy_path)/../../config/install.yaml
+
+.PHONY: destroy
+destroy: etcd-certificates/cleanup
+	@kubectl delete -f $(deploy_path)/../../config/install.yaml

deploy/README.md

@@ -0,0 +1,21 @@
+# Deploy Kamaji
+
+## Quickstart with KinD
+
+```sh
+make -C kind
+```
+
+## Multi-tenant etcd cluster
+
+> This assumes you already have a running Kubernetes cluster and kubeconfig.
+
+```sh
+make -C etcd
+```
+
+## Multi-tenant MySQL-MariaDB cluster
+
+> This assumes you already have a running Kubernetes cluster and kubeconfig.
+
+Read [this](./mysql/README.md) in order to know more about.

deploy/etcd/etcd-cluster.yaml

@@ -1,113 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: etcd
-  namespace:
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: etcd-server
-  namespace:
-spec:
-  type: ClusterIP
-  ports:
-    - name: client
-      port: 2379
-      protocol: TCP
-      targetPort: 2379
-  selector:
-    app: etcd
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: etcd
-  namespace:
-spec:
-  clusterIP: None
-  ports:
-    - port: 2379
-      name: client
-    - port: 2380
-      name: peer
-  selector:
-    app: etcd
----
-apiVersion: apps/v1
-kind: StatefulSet
-metadata:
-  name: etcd
-  labels:
-    app: etcd
-  namespace:
-spec:
-  serviceName: etcd
-  selector:
-    matchLabels:
-      app: etcd
-  replicas: 3
-  template:
-    metadata:
-      name: etcd
-      labels:
-        app: etcd
-    spec:
-      serviceAccountName: etcd
-      volumes:
-      - name: certs
-        secret:
-          secretName: etcd-certs
-      containers:
-        - name: etcd
-          image: quay.io/coreos/etcd:v3.5.1
-          ports:
-            - containerPort: 2379
-              name: client
-            - containerPort: 2380
-              name: peer
-          volumeMounts:
-          - name: data
-            mountPath: /var/run/etcd
-          - name: certs
-            mountPath: /etc/etcd/pki
-          command:
-          - etcd 
-          - --data-dir=/var/run/etcd
-          - --name=$(POD_NAME)
-          - --initial-cluster-state=new
-          - --initial-cluster=etcd-0=https://etcd-0.etcd.$(POD_NAMESPACE).svc.cluster.local:2380,etcd-1=https://etcd-1.etcd.$(POD_NAMESPACE).svc.cluster.local:2380,etcd-2=https://etcd-2.etcd.$(POD_NAMESPACE).svc.cluster.local:2380
-          - --initial-advertise-peer-urls=https://$(POD_NAME).etcd.$(POD_NAMESPACE).svc.cluster.local:2380
-          - --initial-cluster-token=kamaji
-          - --listen-client-urls=https://0.0.0.0:2379
-          - --advertise-client-urls=https://etcd-0.etcd.$(POD_NAMESPACE).svc.cluster.local:2379,https://etcd-1.etcd.$(POD_NAMESPACE).svc.cluster.local:2379,https://etcd-2.etcd.$(POD_NAMESPACE).svc.cluster.local:2379,https://etcd-server.$(POD_NAMESPACE).svc.cluster.local:2379
-          - --client-cert-auth=true
-          - --trusted-ca-file=/etc/etcd/pki/ca.crt
-          - --cert-file=/etc/etcd/pki/server.pem
-          - --key-file=/etc/etcd/pki/server-key.pem
-          - --listen-peer-urls=https://0.0.0.0:2380
-          - --peer-client-cert-auth=true
-          - --peer-trusted-ca-file=/etc/etcd/pki/ca.crt
-          - --peer-cert-file=/etc/etcd/pki/peer.pem 
-          - --peer-key-file=/etc/etcd/pki/peer-key.pem
-          - --auto-compaction-mode=periodic
-          - --auto-compaction-retention=5m
-          - --v=8
-          env:
-          - name: POD_NAME
-            valueFrom:
-              fieldRef:
-                fieldPath: metadata.name
-          - name: POD_NAMESPACE
-            valueFrom:
-              fieldRef:
-                fieldPath: metadata.namespace
-  volumeClaimTemplates:
-  - metadata:
-      name: data
-    spec:
-      accessModes: ["ReadWriteOnce"]
-      resources:
-        requests:
-          storage: 2Gi
-

deploy/kind/Makefile

@@ -11,7 +11,10 @@ prometheus-stack:
 	helm repo update
 	helm install prometheus-stack --create-namespace -n monitoring prometheus-community/kube-prometheus-stack
 
-kamaji: kind ingress-nginx etcd-cluster
+reqs: kind ingress-nginx etcd-cluster
+
+kamaji: reqs
+	@kubectl apply -f $(kind_path)/../../config/install.yaml
 
 destroy: kind/destroy etcd-certificates/cleanup
 
@@ -27,10 +30,13 @@ ingress-nginx-install:
 	kubectl apply -f $(kind_path)/nginx-deploy.yaml
 
 kamaji-kind-worker-build:
-	docker build -f $(kind_path)/kamaji-kind-worker.dockerfile -t quay.io/clastix/kamaji-kind-worker:$${WORKER_VERSION:-latest} .
+	docker build -f $(kind_path)/kamaji-kind-worker.dockerfile -t clastix/kamaji-kind-worker:$${WORKER_VERSION:-latest} .
 
 kamaji-kind-worker-push: kamaji-kind-worker-build
-	docker push quay.io/clastix/kamaji-kind-worker:$${WORKER_VERSION:-latest}
+	docker push clastix/kamaji-kind-worker:$${WORKER_VERSION:-latest}
 
 kamaji-kind-worker-join:
 	$(kind_path)/join-node.bash
+
+kamaji-kind-worker-join-through-konnectivity:
+	$(kind_path)/join-node-konnectivity.bash

deploy/kind/README.md

@@ -1,138 +0,0 @@
-# Setup a minimal Kamaji for development
-
-This document explains how to deploy a minimal Kamaji setup on [KinD](https://kind.sigs.k8s.io/) for development scopes. Please refer to the [Kamaji documentation](../../README.md) for understanding all the terms used in this guide, as for example: `admin cluster` and `tenant control plane`.
-
-## Tools
-
-We assume you have installed on your workstation:
-
-- [Docker](https://docs.docker.com/engine/install/)
-- [KinD](https://kind.sigs.k8s.io/)
-- [kubectl](https://kubernetes.io/docs/tasks/tools/)
-- [kubeadm](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/install-kubeadm/)
-- [jq](https://stedolan.github.io/jq/)
-- [openssl](https://www.openssl.org/)
-- [cfssl](https://github.com/cloudflare/cfssl)
-- [cfssljson](https://github.com/cloudflare/cfssl)
-
-## Setup Kamaji on KinD
-
-The instance of Kamaji is made of a single node hosting:
-
-- admin control-plane
-- admin worker
-- multi-tenant etcd cluster
-
-The multi-tenant etcd cluster is deployed as statefulset into the Kamaji node.
-
-Run `make kamaji` to setup Kamaji on KinD.
-
-```bash
-cd ./deploy/kind
-make kamaji
-```
-
-At this moment you will have your KinD up and running and ETCD cluster in multitenant mode. 
-
-### Install Kamaji
-
-```bash
-$ kubectl apply -f ../../config/install.yaml
-```
-
-### Deploy Tenant Control Plane
-
-Now it is the moment of deploying your first tenant control plane.
-
-```bash
-$ kubectl apply -f - <<EOF
-apiVersion: kamaji.clastix.io/v1alpha1
-kind: TenantControlPlane
-metadata:
-  name: tenant1
-spec:
-  controlPlane:
-    deployment:
-      replicas: 2
-      additionalMetadata:
-        annotations:
-          environment.clastix.io: tenant1
-          tier.clastix.io: "0"
-        labels:
-          tenant.clastix.io: tenant1
-          kind.clastix.io: deployment
-    service:
-      additionalMetadata:
-        annotations:
-          environment.clastix.io: tenant1
-          tier.clastix.io: "0"
-        labels:
-          tenant.clastix.io: tenant1
-          kind.clastix.io: service
-      serviceType: NodePort
-    ingress:
-      enabled: false
-  kubernetes:
-    version: "v1.23.4"
-    kubelet:
-      cgroupfs: cgroupfs
-    admissionControllers:
-    - LimitRanger
-    - ResourceQuota
-  networkProfile:
-    address: "172.18.0.2"
-    port: 31443
-    domain: "clastix.labs"
-    serviceCidr: "10.96.0.0/16"
-    podCidr: "10.244.0.0/16"
-    dnsServiceIPs: 
-    - "10.96.0.10"
-EOF
-```
-
-> Check networkProfile fields according to your installation
-> To let Kamaji works in kind, you have indicate that the service must be [NodePort](https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport)
-
-### Get Kubeconfig
-
-Let's retrieve kubeconfig and store in `/tmp/kubeconfig`
-
-```bash
-$ kubectl get secrets tenant1-admin-kubeconfig -o json \
- | jq -r '.data["admin.conf"]' \
- | base64 -d > /tmp/kubeconfig
- ```
-
-It can be export it, to facilitate the next tasks:
-
-```bash
-$ export KUBECONFIG=/tmp/kubeconfig
-```
-
-### Install CNI
-
-We highly recommend to install [kindnet](https://github.com/aojea/kindnet) as CNI for your kamaji TCP.
-
-```bash
-$ kubectl create -f https://raw.githubusercontent.com/aojea/kindnet/master/install-kindnet.yaml
-```
-
-### Join worker nodes
-
-```bash
-$ make kamaji-kind-worker-join
-```
-
-> To add more worker nodes, run again the command above.
-
-Check out the node:
-
-```bash
-$ kubectl get nodes
-NAME           STATUS   ROLES    AGE   VERSION
-d2d4b468c9de   Ready    <none>   44s   v1.23.4
-```
-
-> For more complex scenarios (exposing port, different version and so on), run `join-node.bash`
-
-Tenant control plane provision has been finished in a minimal Kamaji setup based on KinD. Therefore, you could develop, test and make your own experiments with Kamaji.

deploy/kind/etcd-cluster.yaml

@@ -1,177 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  creationTimestamp: "2021-12-08T17:49:38Z"
-  name: etcd
-  namespace: kamaji-system
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: etcd-server
-  namespace: kamaji-system
-spec:
-  type: ClusterIP
-  ports:
-    - name: client
-      port: 2379
-      protocol: TCP
-      targetPort: 2379
-  selector:
-    app: etcd
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: etcd
-  namespace: kamaji-system
-spec:
-  clusterIP: None
-  ports:
-    - port: 2379
-      name: client
-    - port: 2380
-      name: peer
-  selector:
-    app: etcd
----
-apiVersion: apps/v1
-kind: StatefulSet
-metadata:
-  name: etcd
-  labels:
-    app: etcd
-  namespace: kamaji-system
-spec:
-  serviceName: etcd
-  selector:
-    matchLabels:
-      app: etcd
-  replicas: 3
-  template:
-    metadata:
-      name: etcd
-      labels:
-        app: etcd
-    spec:
-      serviceAccountName: etcd
-      volumes:
-      - name: certs
-        secret:
-          secretName: etcd-certs
-      containers:
-        - name: etcd
-          image: quay.io/coreos/etcd:v3.5.1
-          ports:
-            - containerPort: 2379
-              name: client
-            - containerPort: 2380
-              name: peer
-          volumeMounts:
-            - name: data
-              mountPath: /var/run/etcd
-            - name: certs
-              mountPath: /etc/etcd/pki
-          command:
-          - etcd 
-          - --data-dir=/var/run/etcd
-          - --name=$(POD_NAME)
-          - --initial-cluster-state=new
-          - --initial-cluster=etcd-0=https://etcd-0.etcd.$(POD_NAMESPACE).svc.cluster.local:2380,etcd-1=https://etcd-1.etcd.$(POD_NAMESPACE).svc.cluster.local:2380,etcd-2=https://etcd-2.etcd.$(POD_NAMESPACE).svc.cluster.local:2380
-          - --initial-advertise-peer-urls=https://$(POD_NAME).etcd.$(POD_NAMESPACE).svc.cluster.local:2380
-          - --initial-cluster-token=kamaji
-          - --listen-client-urls=https://0.0.0.0:2379
-          - --advertise-client-urls=https://etcd-0.etcd.$(POD_NAMESPACE).svc.cluster.local:2379,https://etcd-1.etcd.$(POD_NAMESPACE).svc.cluster.local:2379,https://etcd-2.etcd.$(POD_NAMESPACE).svc.cluster.local:2379,https://etcd-server.$(POD_NAMESPACE).svc.cluster.local:2379
-          - --client-cert-auth=true
-          - --trusted-ca-file=/etc/etcd/pki/ca.crt
-          - --cert-file=/etc/etcd/pki/server.pem
-          - --key-file=/etc/etcd/pki/server-key.pem
-          - --listen-peer-urls=https://0.0.0.0:2380
-          - --peer-client-cert-auth=true
-          - --peer-trusted-ca-file=/etc/etcd/pki/ca.crt
-          - --peer-cert-file=/etc/etcd/pki/peer.pem 
-          - --peer-key-file=/etc/etcd/pki/peer-key.pem
-          - --auto-compaction-mode=periodic
-          - --auto-compaction-retention=5m
-          - --v=8
-          env:
-          - name: POD_NAME
-            valueFrom:
-              fieldRef:
-                fieldPath: metadata.name
-          - name: POD_NAMESPACE
-            valueFrom:
-              fieldRef:
-                fieldPath: metadata.namespace
-          livenessProbe:
-            failureThreshold: 8
-            httpGet:
-              host: 127.0.0.1
-              path: /health
-              port: 2381
-              scheme: HTTP
-            initialDelaySeconds: 10
-            periodSeconds: 10
-            timeoutSeconds: 15
-          startupProbe:
-            failureThreshold: 24
-            httpGet:
-              host: 127.0.0.1
-              path: /health
-              port: 2381
-              scheme: HTTP
-            initialDelaySeconds: 10
-            periodSeconds: 10
-            timeoutSeconds: 15
-  volumeClaimTemplates:
-    - metadata:
-        name: data
-      spec:
-        accessModes: ["ReadWriteOnce"]
-        resources:
-          requests:
-            storage: 8Gi
----
-apiVersion: v1
-kind: Pod
-metadata:
-  labels:
-    app: etcd
-  name: etcd-root-client
-  namespace: kamaji-system
-spec:
-  serviceAccountName: etcd
-  containers:
-  - command:
-    - sleep
-    - infinity
-    env:
-    - name: POD_NAMESPACE
-      valueFrom:
-        fieldRef:
-          fieldPath: metadata.namespace    
-    - name: ETCDCTL_ENDPOINTS
-      value: https://etcd-server.$(POD_NAMESPACE).svc.cluster.local:2379
-    - name: ETCDCTL_CACERT
-      value: /opt/certs/ca/ca.crt
-    - name: ETCDCTL_CERT
-      value: /opt/certs/root-client-certs/tls.crt
-    - name: ETCDCTL_KEY
-      value: /opt/certs/root-client-certs/tls.key
-    image: quay.io/coreos/etcd:v3.5.1
-    imagePullPolicy: IfNotPresent
-    name: etcd-client
-    resources: {}
-    volumeMounts:
-    - name: root-client-certs
-      mountPath: /opt/certs/root-client-certs
-    - name: ca
-      mountPath: /opt/certs/ca     
-  volumes:
-  - name: root-client-certs
-    secret:
-      secretName: root-client-certs
-  - name: ca
-    secret:
-      secretName: etcd-certs     
-

deploy/kind/join-node-konnectivity.bash

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+set -e
+
+# Constants
+export DOCKER_IMAGE_NAME="clastix/kamaji-kind-worker"
+
+# Variables
+export KUBERNETES_VERSION=${1:-latest}
+export KUBECONFIG="${KUBECONFIG:-/tmp/kubeconfig}"
+
+if [ -z $2 ]
+then
+    MAPPING_PORT=""
+else
+    MAPPING_PORT="-p ${2}:80"
+fi
+
+export KONNECTIVITY_PROXY_HOST=${3:-konnectiviy.local}
+
+clear
+echo "Welcome to join a new node through Konnectivity"
+
+echo -ne "\nChecking right kubeconfig\n"
+kubectl cluster-info
+echo "Are you pointing to the right tenant control plane? (Type return to continue)"
+read
+
+JOIN_CMD="$(kubeadm --kubeconfig=${KUBECONFIG} token create --print-join-command) --ignore-preflight-errors=SystemVerification"
+echo "Deploying new node..."
+KIND_IP=$(docker inspect kamaji-control-plane --format='{{.NetworkSettings.Networks.kind.IPAddress}}')
+NODE=$(docker run -d --add-host $KONNECTIVITY_PROXY_HOST:$KIND_IP --privileged -v /lib/modules:/lib/modules:ro -v /var --net host $MAPPING_PORT $DOCKER_IMAGE_NAME:$KUBERNETES_VERSION)
+sleep 10
+echo "Joining new node..."
+docker exec -e JOIN_CMD="$JOIN_CMD" $NODE /bin/bash -c "$JOIN_CMD"

deploy/kind/join-node.bash

@@ -3,11 +3,12 @@
 set -e
 
 # Constants
-export DOCKER_IMAGE_NAME="quay.io/clastix/kamaji-kind-worker"
+export DOCKER_IMAGE_NAME="clastix/kamaji-kind-worker"
 export DOCKER_NETWORK="kind"
 
 # Variables
 export KUBERNETES_VERSION=${1:-latest}
+export KUBECONFIG="${KUBECONFIG:-/tmp/kubeconfig}"
 
 if [ -z $2 ]
 then
@@ -24,7 +25,7 @@ kubectl cluster-info
 echo "Are you pointing to the right tenant control plane? (Type return to continue)"
 read
 
-JOIN_CMD="$(kubeadm --kubeconfig=/tmp/kubeconfig token create --print-join-command) --ignore-preflight-errors=SystemVerification"
+JOIN_CMD="$(kubeadm --kubeconfig=${KUBECONFIG} token create --print-join-command) --ignore-preflight-errors=SystemVerification"
 echo "Deploying new node..."
 NODE=$(docker run -d --privileged -v /lib/modules:/lib/modules:ro -v /var --net $DOCKER_NETWORK $MAPPING_PORT $DOCKER_IMAGE_NAME:$KUBERNETES_VERSION)
 sleep 10

deploy/kind/kind-kamaji.yaml

@@ -10,6 +10,10 @@ nodes:
     nodeRegistration:
       kubeletExtraArgs:
         node-labels: "ingress-ready=true"
+  ## required for Cluster API local development
+  extraMounts:
+    - hostPath: /var/run/docker.sock
+      containerPath: /var/run/docker.sock
   extraPortMappings:
     ## expose port 80 of the node to port 80 on the host
   - containerPort: 80
@@ -19,12 +23,15 @@ nodes:
   - containerPort: 443
     hostPort: 443
     protocol: TCP
+    ## expose port 31132 of the node to port 31132 on the host for konnectivity
+  - containerPort: 31132
+    hostPort: 31132
+    protocol: TCP   
     ## expose port 31443 of the node to port 31443 on the host
   - containerPort: 31443
     hostPort: 31443
-    protocol: TCP    
+    protocol: TCP
     ## expose port 6443 of the node to port 8443 on the host
   - containerPort: 6443
     hostPort: 8443
     protocol: TCP
-

deploy/mysql/Makefile

@@ -0,0 +1,31 @@
+mariadb_path := $(patsubst %/,%,$(dir $(abspath $(lastword $(MAKEFILE_LIST)))))
+
+.PHONY: mariadb mariadb-certificates mariadb-secrets
+
+mariadb: mariadb-certificates mariadb-secrets mariadb-deployment
+
+mariadb-certificates:
+	rm -rf $(mariadb_path)/certs && mkdir $(mariadb_path)/certs
+	cfssl gencert -initca $(mariadb_path)/ca-csr.json | cfssljson -bare $(mariadb_path)/certs/ca
+	@mv $(mariadb_path)/certs/ca.pem $(mariadb_path)/certs/ca.crt
+	@mv $(mariadb_path)/certs/ca-key.pem $(mariadb_path)/certs/ca.key
+	cfssl gencert -ca=$(mariadb_path)/certs/ca.crt -ca-key=$(mariadb_path)/certs/ca.key \
+		-config=$(mariadb_path)/config.json -profile=server \
+		$(mariadb_path)/server-csr.json | cfssljson -bare $(mariadb_path)/certs/server
+	@mv $(mariadb_path)/certs/server.pem $(mariadb_path)/certs/server.crt
+	@mv $(mariadb_path)/certs/server-key.pem $(mariadb_path)/certs/server.key
+	chmod 644 $(mariadb_path)/certs/*
+
+mariadb-secrets:
+	@kubectl -n kamaji-system create secret generic mysql-config \
+		--from-file=$(mariadb_path)/certs/ca.crt --from-file=$(mariadb_path)/certs/ca.key \
+		--from-file=$(mariadb_path)/certs/server.key --from-file=$(mariadb_path)/certs/server.crt \
+		--from-file=$(mariadb_path)/mysql-ssl.cnf \
+		--from-literal=MYSQL_ROOT_PASSWORD=root
+
+mariadb-deployment:
+	@kubectl -n kamaji-system apply -f $(mariadb_path)/mariadb.yaml
+
+destroy:
+	@kubectl delete -n kamaji-system -f $(mariadb_path)/mariadb.yaml
+	@kubectl delete -n kamaji-system secret mysql-config

deploy/mysql/README.md

@@ -0,0 +1,43 @@
+# MySQL as Kubernetes Storage
+
+Kamaji offers the possibility of having a different storage system than `ETCD` thanks to [kine](https://github.com/k3s-io/kine). One of the implementations is [MySQL](https://www.mysql.com/).
+
+Kamaji project is developed using [kind](https://kind.sigs.k8s.io), therefore, MySQL (or [MariaDB](https://mariadb.org/) in this case) will be deployed into the local kubernetes cluster in order to be used as storage for the tenants.
+
+There is a Makefile to help with the process:
+
+* **Full Installation**
+
+```bash
+$ make mariadb
+```
+
+This action will perform all the necessary stuffs to have MariaDB as kubernetes storage backend using kine.
+
+* **Certificate creation**
+
+```bash
+$ make mariadb-certificates
+```
+
+Communication between kine and the backend is encrypted, therefore, some certificates must be created.
+
+* **Secret Deployment**
+
+```bash
+$ make mariadb-secrets
+```
+
+Previous certificates and MySQL configuration have to be available in order to be used. They will be under the secret `kamaji-system:mysql-config`.
+
+* **Deployment**
+
+```bash
+$ make mariadb-deployment
+```
+
+* **Uninstall Everything**
+
+```bash
+$ make destroy
+```

deploy/mysql/ca-csr.json

@@ -0,0 +1,18 @@
+{
+  "CN": "Clastix CA",
+  "key": {
+    "algo": "rsa",
+    "size": 2048
+  },
+  "names": [
+    {
+      "C": "IT",
+      "ST": "Italy",
+      "L": "Milan"
+    }
+  ],
+  "hosts": [
+    "127.0.0.1",
+    "localhost"
+  ]
+}

deploy/mysql/config.json

@@ -0,0 +1,18 @@
+{
+  "signing": {
+      "default": {
+          "expiry": "8760h"
+      },
+      "profiles": {
+          "server": {
+              "expiry": "8760h",
+              "usages": [
+                  "signing",
+                  "key encipherment",
+                  "server auth",
+                  "client auth"
+              ]
+          }
+      }
+  }
+}

deploy/mysql/kine.yaml

@@ -0,0 +1,61 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kine-tenant
+  namespace:
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: kine-tenant
+  namespace:
+spec:
+  type: ClusterIP
+  ports:
+    - name: server
+      port: 2379
+      protocol: TCP
+      targetPort: 2379
+  selector:
+    app: kine-tenant
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: kine-tenant
+  labels:
+    app: kine-tenant
+  namespace:
+spec:
+  selector:
+    matchLabels:
+      app: kine-tenant
+  replicas: 1
+  template:
+    metadata:
+      name: kine-tenant
+      labels:
+        app: kine-tenant
+    spec:
+      serviceAccountName: kine-tenant
+      volumes:
+      - name: certs
+        secret:
+          secretName: mysql-certs
+      containers:
+        - name: kine-tenant
+          image: rancher/kine:v0.9.2-amd64
+          ports:
+            - containerPort: 2379
+              name: server
+          volumeMounts:
+          - name: certs
+            mountPath: /kine
+          env:    
+          - name: GODEBUG
+            value: "x509ignoreCN=0"
+          args:
+          - --endpoint=mysql://tenant1:tenant1@tcp(mysql:3306)/tenant1
+          - --ca-file=/kine/ca.crt
+          - --cert-file=/kine/server.crt
+          - --key-file=/kine/server.key

deploy/mysql/mariadb.yaml

@@ -0,0 +1,77 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: mariadb
+  namespace:
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: mariadb
+  namespace:
+spec:
+  type: ClusterIP
+  ports:
+    - name: server
+      port: 3306
+      protocol: TCP
+      targetPort: 3306
+  selector:
+    app: mariadb
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: mariadb
+  labels:
+    app: mariadb
+  namespace:
+spec:
+  selector:
+    matchLabels:
+      app: mariadb
+  replicas: 1
+  template:
+    metadata:
+      name: mariadb
+      labels:
+        app: mariadb
+    spec:
+      serviceAccountName: mariadb
+      volumes:
+      - name: certs
+        secret:
+          secretName: mysql-config
+      - name: data
+        persistentVolumeClaim:
+          claimName: pvc-mariadb
+      containers:
+        - name: mariadb
+          image: mariadb:10.7.4
+          ports:
+            - containerPort: 3306
+              name: server
+          volumeMounts:
+          - name: data
+            mountPath: /var/lib/mariadb
+          - name: certs
+            mountPath: /etc/mysql/conf.d/
+          env:
+          - name: MYSQL_ROOT_PASSWORD
+            valueFrom:
+              secretKeyRef:
+                name: mysql-config
+                key: MYSQL_ROOT_PASSWORD                
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: pvc-mariadb
+  namespace:
+spec:
+  accessModes:
+  - ReadWriteOnce
+  resources:
+    requests:
+      storage: 2Gi
+  storageClassName: standard

deploy/mysql/mysql-ssl.cnf

@@ -0,0 +1,5 @@
+[mysqld]
+ssl-ca=/etc/mysql/conf.d/ca.crt
+ssl-cert=/etc/mysql/conf.d/server.crt
+ssl-key=/etc/mysql/conf.d/server.key
+require_secure_transport=ON

deploy/mysql/server-csr.json

@@ -0,0 +1,19 @@
+{
+  "CN": "mariadb.kamaji-system.svc.cluster.local",
+  "key": {
+    "algo": "rsa",
+    "size": 2048
+  },
+  "hosts": [
+    "127.0.0.1",
+    "localhost",
+    "mariadb",
+    "mariadb.kamaji-system",
+    "mariadb.kamaji-system.svc",
+    "mariadb.kamaji-system.svc.cluster.local",
+    "mysql",
+    "mysql.kamaji-system",
+    "mysql.kamaji-system.svc",
+    "mysql.kamaji-system.svc.cluster.local"
+  ]
+}

docs/README.md

@@ -0,0 +1,10 @@
+# Kamaji documentation
+
+- [Architecture](./architecture.md)
+- [Concepts](./concepts.md)
+- [Getting started](./getting-started-with-kamaji.md)
+- [Kamaji Deployment](./kamaji-deployment-guide.md)
+- [Tenant deployment](./kamaji-tenant-deployment-guide.md)
+- Deployment on cloud providers:
+  - [Azure](./kamaji-azure-deployment-guide.md)
+- [Reference](./reference.md)

docs/getting-started-with-kamaji.md

@@ -1,604 +1,212 @@
-# Setup a Kamaji environment
-This getting started guide will lead you through the process of creating a basic working Kamaji setup.
+# Setup a minimal Kamaji for development
 
-Kamaji requires:
+This document explains how to deploy a minimal Kamaji setup on [KinD](https://kind.sigs.k8s.io/) for development scopes. Please refer to the [Kamaji documentation](../README.md) for understanding all the terms used in this guide, as for example: `admin cluster` and `tenant control plane`.
 
-- (optional) a bootstrap node;
-- a multi-tenant `etcd` cluster made of 3 nodes hosting the datastore for the `Tenant`s' clusters
-- a Kubernetes cluster, running the admin and Tenant Control Planes
-- an arbitrary number of machines hosting `Tenant`s' workloads
+## Pre-requisites
 
-> In this guide, we assume all machines are running `Ubuntu 20.04`.
+We assume you have installed on your workstation:
 
-  * [Prepare the bootstrap workspace](#prepare-the-bootstrap-workspace)
-  * [Access Admin cluster](#access-admin-cluster)
-  * [Setup external multi-tenant etcd](#setup-external-multi-tenant-etcd)
-  * [Setup internal multi-tenant etcd](#setup-internal-multi-tenant-etcd)
-  * [Install Kamaji controller](#install-kamaji-controller)
-  * [Setup Tenant cluster](#setup-tenant-cluster)
+- [Docker](https://docs.docker.com/engine/install/)
+- [KinD](https://kind.sigs.k8s.io/)
+- [kubectl](https://kubernetes.io/docs/tasks/tools/)
+- [kubeadm](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/install-kubeadm/)
+- [jq](https://stedolan.github.io/jq/)
+- [openssl](https://www.openssl.org/)
+- [cfssl](https://github.com/cloudflare/cfssl)
+- [cfssljson](https://github.com/cloudflare/cfssl)
 
-## Prepare the bootstrap workspace
-This getting started guide is supposed to be run from a remote or local bootstrap machine.
-First, prepare the workspace directory:
+## Setup Kamaji on KinD
 
-```
-git clone https://github.com/clastix/kamaji
-cd kamaji/deploy
-```
+The instance of Kamaji is made of a single node hosting:
 
-Throughout the instructions, shell variables are used to indicate values that you should adjust to your own environment.
+- admin control-plane
+- admin worker
+- multi-tenant etcd cluster
 
-### Install required tools
-On the bootstrap machine, install all the required tools to work with a Kamaji setup.
+### Standard
 
-#### cfssl and cfssljson
-The `cfssl` and `cfssljson` command line utilities will be used in addition to `kubeadm` to provision the PKI Infrastructure and generate TLS certificates.
-
-```
-wget -q --show-progress --https-only --timestamping \
-https://storage.googleapis.com/kubernetes-the-hard-way/cfssl/1.4.1/linux/cfssl \
-https://storage.googleapis.com/kubernetes-the-hard-way/cfssl/1.4.1/linux/cfssljson
-
-chmod +x cfssl cfssljson
-sudo mv cfssl cfssljson /usr/local/bin/
-```
-
-#### Kubernetes tools
-Install `kubeadm` and `kubectl`
+You can install your KinD cluster, ETCD multi-tenant cluster and Kamaji operator with a **single command**:
 
 ```bash
-sudo apt update && sudo apt install -y apt-transport-https ca-certificates curl && \
-sudo curl -fsSLo /usr/share/keyrings/kubernetes-archive-keyring.gpg https://packages.cloud.google.com/apt/doc/apt-key.gpg && \
-echo "deb [signed-by=/usr/share/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" | sudo tee /etc/apt/sources.list.d/kubernetes.list && \
-sudo apt update && sudo apt install -y kubeadm kubectl --allow-change-held-packages && \
-sudo apt-mark hold kubeadm kubectl
+$ make -C deploy/kind
 ```
 
-#### etcdctl
-For administration of the `etcd` cluster, download and install the `etcdctl` CLI utility on the bootstrap machine
+Now you can [create your first `TenantControlPlane`](#deploy-tenant-control-plane).
+
+### Data store-specific
+
+#### ETCD
+
+The multi-tenant etcd cluster is deployed as statefulset into the Kamaji node.
+
+Run `make reqs` to setup Kamaji's requisites on KinD:
 
 ```bash
-ETCD_VER=v3.5.1
-ETCD_URL=https://storage.googleapis.com/etcd
-curl -L ${ETCD_URL}/${ETCD_VER}/etcd-${ETCD_VER}-linux-amd64.tar.gz -o etcd-${ETCD_VER}-linux-amd64.tar.gz
-tar xzvf etcd-${ETCD_VER}-linux-amd64.tar.gz etcd-${ETCD_VER}-linux-amd64/etcdctl
-sudo cp etcd-${ETCD_VER}-linux-amd64/etcdctl /usr/bin/etcdctl
-rm -rf etcd-${ETCD_VER}-linux-amd64*
+$ make -C deploy/kind reqs
 ```
 
-Verify `etcdctl` version is installed
+At this moment you will have your KinD up and running and ETCD cluster in multitenant mode.
+
+Now you're ready to [install Kamaji operator](#install-kamaji).
+
+#### Kine MySQL
+
+> The MySQL-compatible cluster provisioning is omitted here.
+
+Kamaji offers the possibility of using a different storage system than `ETCD` for the tenants, like MySQL compatible databases.
+
+Once a compatible-mysql database is running, we need to provide information about it to kamaji:
+
+```
+--etcd-storage-type=kine-mysql
+--kine-mysql-host=<database host>
+--kine-mysql-port=<database port>
+--kine-mysql-secret-name=<secret name>
+--kine-mysql-secret-namespace=<secret namespace>
+```
+
+The secret with the configuration and certificates for mysql should look like:
+```yaml
+apiVersion: v1
+data:
+  MYSQL_ROOT_PASSWORD: ...
+  ca.crt: ...
+  ca.key: ...
+  mysql-ssl.cnf: ...
+  server.crt:  ...
+  server.key:  ...
+kind: Secret
+metadata:
+  creationTimestamp: "2022-06-30T08:03:15Z"
+  name: mysql-config
+  namespace: kamaji-system
+  resourceVersion: "32228"
+  uid: 51b155a1-426c-42d2-8147-be680bf458a6
+type: Opaque
+```
+
+and `mysql-ssl.cnf`:
+```
+[mysqld]
+ssl-ca=/etc/mysql/conf.d/ca.crt
+ssl-cert=/etc/mysql/conf.d/server.crt
+ssl-key=/etc/mysql/conf.d/server.key
+require_secure_transport=ON
+```
+
+You can read more about it [here](../deploy/mysql/README.md).
+
+Assuming you adjusted the [Kamaji manifest](./config/install.yaml) to connect to the MySQL-compatible database, you can now install it.
+
+### Install Kamaji
 
 ```bash
-etcdctl version
-etcdctl version: 3.5.1
-API version: 3.5
+$ kubectl apply -f config/install.yaml
 ```
 
+### Deploy Tenant Control Plane
 
-## Access Admin cluster
-In Kamaji, an Admin Cluster is a regular Kubernetes cluster which hosts zero to many Tenant Cluster Control Planes running as pods. The admin cluster acts as management cluster for all the Tenant clusters and implements Monitoring, Logging, and Governance of all the Kamaji setup, including all Tenant clusters. 
-
-Any regular and conformant Kubernetes v1.22+ cluster can be turned into a Kamaji setup. Currently we tested:
-
-- [Kubernetes installed with `kubeadm`](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/).
-- [Azure AKS managed service](./kamaji-on-azure.md).
-- [KinD for local development](./kind/README.md).
-
-The admin cluster should provide:
-
-- CNI module installed, eg. Calico
-- Support for LoadBalancer Service Type, eg. MetalLB
-- Ingress Controller
-- CSI module installed with StorageClass for multi-tenant `etcd`
-- Monitoring Stack, eg. Prometheus and Grafana
-
-Make sure you have a `kubeconfig` file with admin permissions on the cluster you want to turn into a Kamaji Admin Cluster.
-
-## Setup external multi-tenant etcd
-In this section, we're going to setup a multi-tenant `etcd` cluster on dedicated nodes. Alternatively, if you want to use an internal `etcd` cluster as Kubernetes StatefulSet, jump [here](#setup-internal-multi-tenant-etcd).
-
-### Ensure host access
-From the bootstrap machine load the environment for external `etcd` setup:
+Now it is the moment of deploying your first tenant control plane.
 
 ```bash
-source kamaji-external-etcd.env
-```
-
-The installer requires a user that has access to all hosts. In order to run the installer as a non-root user, first configure passwordless sudo rights each host:
-
-Generate an SSH key on the host you run the installer on:
-
-```bash
-ssh-keygen -t rsa
-```
-
-> Do not use a password.
-
-Distribute the key to the other cluster hosts.
-
-Depending on your environment, use a bash loop:
-
-```bash
-HOSTS=(${ETCD0} ${ETCD1} ${ETCD2})
-for i in "${!HOSTS[@]}"; do
-  HOST=${HOSTS[$i]}
-  ssh-copy-id -i ~/.ssh/id_rsa.pub $HOST;
-done
-```
-
-> Alternatively, inject the generated public key into machines metadata.
-
-Confirm that you can access each host from bootstrap machine:
-
-```bash
-HOSTS=(${ETCD0} ${ETCD1} ${ETCD2})
-for i in "${!HOSTS[@]}"; do
-  HOST=${HOSTS[$i]}
-  ssh ${USER}@${HOST} -t 'hostname';
-done
-```
-
-### Configure disk layout
-As per `etcd` [requirements](https://etcd.io/docs/v3.5/op-guide/hardware/#disks), back `etcd`’s storage with a SSD. A SSD usually provides lower write latencies and with less variance than a spinning disk, thus improving the stability and reliability of `etcd`.
-
-For each `etcd` machine, we assume an additional `sdb` disk of 10GB:
-
-```
-clastix@kamaji-etcd-00:~$ lsblk
-NAME    MAJ:MIN RM  SIZE RO TYPE MOUNTPOINT
-sda       8:0    0   16G  0 disk 
-├─sda1    8:1    0 15.9G  0 part /
-├─sda14   8:14   0    4M  0 part 
-└─sda15   8:15   0  106M  0 part /boot/efi
-sdb       8:16   0   10G  0 disk 
-sr0      11:0    1    4M  0 rom  
-```
-
-Create partition, format, and mount the `etcd` disk, by running the script below from the bootstrap machine:
-
-> If you already used the `etcd` disks, please make sure to wipe the partitions with `sudo wipefs --all --force /dev/sdb` before to attempt to recreate them.
-
-```bash
-for i in "${!ETCDHOSTS[@]}"; do
-  HOST=${ETCDHOSTS[$i]}
-  ssh ${USER}@${HOST} -t 'echo type=83 | sudo sfdisk -f -q /dev/sdb'
-  ssh ${USER}@${HOST} -t 'sudo mkfs -F -q -t ext4 /dev/sdb1'
-  ssh ${USER}@${HOST} -t 'sudo mkdir -p /var/lib/etcd'
-  ssh ${USER}@${HOST} -t 'sudo e2label /dev/sdb1 ETCD'
-  ssh ${USER}@${HOST} -t 'echo LABEL=ETCD /var/lib/etcd ext4 defaults 0 1 | sudo tee -a /etc/fstab'
-  ssh ${USER}@${HOST} -t 'sudo mount -a'
-  ssh ${USER}@${HOST} -t 'sudo lsblk -f'
-done
-```
-
-### Install prerequisites
-Use bash script `nodes-prerequisites.sh` to install all the dependencies on all the cluster nodes:
-
-- Install `containerd` as container runtime
-- Install `crictl`, the command line for working with `containerd`
-- Install `kubectl`, `kubelet`, and `kubeadm` in the desired version, eg. `v1.24.0`
-
-Run the installation script:
-
-```bash
-VERSION=v1.24.0
-./nodes-prerequisites.sh ${VERSION:1} ${HOSTS[@]}
-```
-
-### Configure kubelet
-
-On each `etcd` node, configure the `kubelet` service to start `etcd` static pods using `containerd` as container runtime, by running the script below from the bootstrap machine:
-
-```bash
-cat << EOF > 20-etcd-service-manager.conf
-[Service]
-ExecStart=
-ExecStart=/usr/bin/kubelet --address=127.0.0.1 --pod-manifest-path=/etc/kubernetes/manifests --cgroup-driver=systemd --container-runtime=remote --container-runtime-endpoint=/run/containerd/containerd.sock
-Restart=always
-EOF
-```
-
-```
-for i in "${!ETCDHOSTS[@]}"; do
-  HOST=${ETCDHOSTS[$i]}
-  scp 20-etcd-service-manager.conf ${USER}@${HOST}:
-  ssh ${USER}@${HOST} -t 'sudo chown -R root:root 20-etcd-service-manager.conf && sudo mv 20-etcd-service-manager.conf /etc/systemd/system/kubelet.service.d/20-etcd-service-manager.conf'
-  ssh ${USER}@${HOST} -t 'sudo systemctl daemon-reload'
-  ssh ${USER}@${HOST} -t 'sudo systemctl start kubelet'
-  ssh ${USER}@${HOST} -t 'sudo systemctl enable kubelet'
-done
-
-rm -f 20-etcd-service-manager.conf
-```
-
-### Create configuration
-Create temp directories to store files that will end up on `etcd` hosts:
-
-```bash
-mkdir -p /tmp/${ETCD0}/ /tmp/${ETCD1}/ /tmp/${ETCD2}/
-NAMES=("etcd00" "etcd01" "etcd02")
-
-for i in "${!ETCDHOSTS[@]}"; do
-HOST=${ETCDHOSTS[$i]}
-NAME=${NAMES[$i]}
-
-cat <<EOF | sudo tee /tmp/${HOST}/kubeadmcfg.yaml
-apiVersion: "kubeadm.k8s.io/v1beta2"
-kind: ClusterConfiguration
-etcd:
-  local:
-    serverCertSANs:
-    - "${HOST}"
-    peerCertSANs:
-    - "${HOST}"
-    extraArgs:
-      initial-cluster: ${NAMES[0]}=https://${ETCDHOSTS[0]}:2380,${NAMES[1]}=https://${ETCDHOSTS[1]}:2380,${NAMES[2]}=https://${ETCDHOSTS[2]}:2380
-      initial-cluster-state: new
-      name: ${NAME}
-      listen-peer-urls: https://${HOST}:2380
-      listen-client-urls: https://${HOST}:2379
-      advertise-client-urls: https://${HOST}:2379
-      initial-advertise-peer-urls: https://${HOST}:2380
-      auto-compaction-mode: periodic
-      auto-compaction-retention: 5m
-      quota-backend-bytes: '8589934592'
-EOF
-done
-```
-> Note:
->
-> ##### Etcd compaction
->
-> By enabling `etcd` authentication, it prevents the tenant apiservers (clients of `etcd`) to issue compaction requests. We set `etcd` to automatically compact the keyspace with the `--auto-compaction-*` option with a period of hours or minutes. When `--auto-compaction-mode=periodic` and `--auto-compaction-retention=5m` and writes per minute are about 1000, `etcd` compacts revision 5000 for every 5 minute. 
-> 
-> ##### Etcd storage quota
->
-> Currently, `etcd` is limited in storage size, defaulted to `2GB` and configurable with `--quota-backend-bytes` flag up to `8GB`. In Kamaji, we use a single `etcd` to store multiple tenant clusters, so we need to increase this size. Please, note `etcd` warns at startup if the configured value exceeds `8GB`.
-
-### Generate certificates 
-On the bootstrap machine, using `kubeadm` init phase, create and distribute `etcd` CA certificates:
-
-```bash
-sudo kubeadm init phase certs etcd-ca
-mkdir kamaji
-sudo cp -r /etc/kubernetes/pki/etcd kamaji
-sudo chown -R ${USER}. kamaji/etcd
-```
-
-For each `etcd` host: 
-
-```bash
-for i in "${!ETCDHOSTS[@]}"; do
-  HOST=${ETCDHOSTS[$i]}
-  sudo kubeadm init phase certs etcd-server --config=/tmp/${HOST}/kubeadmcfg.yaml
-  sudo kubeadm init phase certs etcd-peer --config=/tmp/${HOST}/kubeadmcfg.yaml
-  sudo kubeadm init phase certs etcd-healthcheck-client --config=/tmp/${HOST}/kubeadmcfg.yaml
-  sudo cp -R /etc/kubernetes/pki /tmp/${HOST}/
-  sudo find /etc/kubernetes/pki -not -name ca.crt -not -name ca.key -type f -delete
-done
-```
-
-### Startup the cluster
-Upload certificates on each `etcd` node and restart the `kubelet`
-
-```bash
-for i in "${!ETCDHOSTS[@]}"; do
-  HOST=${ETCDHOSTS[$i]}
-  sudo chown -R ${USER}. /tmp/${HOST}
-  scp -r /tmp/${HOST}/* ${USER}@${HOST}:
-  ssh ${USER}@${HOST} -t 'sudo chown -R root:root pki'
-  ssh ${USER}@${HOST} -t 'sudo mv pki /etc/kubernetes/'
-  ssh ${USER}@${HOST} -t 'sudo kubeadm init phase etcd local --config=kubeadmcfg.yaml'
-  ssh ${USER}@${HOST} -t 'sudo systemctl daemon-reload'
-  ssh ${USER}@${HOST} -t 'sudo systemctl restart kubelet'
-done
-```
-
-This will start the static `etcd` pod on each node and then the cluster gets formed.
-
-Generate certificates for the `root` user
-
-```bash
-cat > root-csr.json <<EOF  
-{
-  "CN": "root",
-  "key": {
-    "algo": "rsa",
-    "size": 2048
-  }
-}
-EOF
-```
-
-```bash
-cfssl gencert \
-  -ca=kamaji/etcd/ca.crt \
-  -ca-key=kamaji/etcd/ca.key \
-  -config=cfssl-cert-config.json \
-  -profile=client-authentication \
-  root-csr.json | cfssljson -bare root
-```
-
-```bash
-cp root.pem kamaji/etcd/root.crt
-cp root-key.pem kamaji/etcd/root.key
-rm root*
-```
-
-The result should be:
-
-```bash
-$ tree kamaji
-kamaji
-└── etcd
-    ├── ca.crt
-    ├── ca.key
-    ├── root.crt
-    └── root.key
-```
-
-Use the `root` user to check the just formed `etcd` cluster is in health state 
-
-```bash
-export ETCDCTL_CACERT=kamaji/etcd/ca.crt
-export ETCDCTL_CERT=kamaji/etcd/root.crt
-export ETCDCTL_KEY=kamaji/etcd/root.key
-export ETCDCTL_ENDPOINTS=https://${ETCD0}:2379
-
-etcdctl member list -w table
-```
-
-The result should be something like this:
-
-```
-+------------------+---------+--------+----------------------------+----------------------------+------------+
-|        ID        | STATUS  |  NAME  |         PEER ADDRS         |        CLIENT ADDRS        | IS LEARNER |
-+------------------+---------+--------+----------------------------+----------------------------+------------+
-| 72657d6307364226 | started | etcd01 | https://192.168.32.11:2380 | https://192.168.32.11:2379 |      false |
-| 91eb892c5ee87610 | started | etcd00 | https://192.168.32.10:2380 | https://192.168.32.10:2379 |      false |
-| e9971c576949c34e | started | etcd02 | https://192.168.32.12:2380 | https://192.168.32.12:2379 |      false |
-+------------------+---------+--------+----------------------------+----------------------------+------------+
-```
-
-### Enable multi-tenancy
-The `root` user has full access to `etcd`, must be created before activating authentication. The `root` user must have the `root` role and is allowed to change anything inside `etcd`.
-
-```bash
-etcdctl user add --no-password=true root
-etcdctl role add root
-etcdctl user grant-role root root
-etcdctl auth enable
-```
-
-### Cleanup
-If you want to get rid of the etcd cluster, for each node, login and clean it:
-
-```bash
-HOSTS=(${ETCD0} ${ETCD1} ${ETCD2})
-for i in "${!HOSTS[@]}"; do
-  HOST=${HOSTS[$i]}
-  ssh ${USER}@${HOST} -t 'sudo kubeadm reset -f';
-  ssh ${USER}@${HOST} -t 'sudo systemctl reboot';
-done
-```
-
-## Setup internal multi-tenant etcd
-If you opted for an internal etcd cluster running in the Kamaji admin cluster, follow steps below.
-
-From the bootstrap machine load the environment for internal `etcd` setup:
-
-```bash
-source kamaji-internal-etcd.env
-```
-
-### Generate certificates 
-On the bootstrap machine, using `kubeadm` init phase, create the `etcd` CA certificates:
-
-```bash
-sudo kubeadm init phase certs etcd-ca
-mkdir kamaji
-sudo cp -r /etc/kubernetes/pki/etcd kamaji
-sudo chown -R ${USER}. kamaji/etcd
-```
-
-Generate the `etcd` certificates for peers:
-
-```
-cat << EOF | tee kamaji/etcd/peer-csr.json
-{
-  "CN": "etcd",
-  "key": {
-    "algo": "rsa",
-    "size": 2048
-  },
-  "hosts": [
-    "127.0.0.1",
-    "etcd-0",
-    "etcd-0.etcd",
-    "etcd-0.etcd.${ETCD_NAMESPACE}.svc",
-    "etcd-0.etcd.${ETCD_NAMESPACE}.svc.cluster.local",
-    "etcd-1",
-    "etcd-1.etcd",
-    "etcd-1.etcd.${ETCD_NAMESPACE}.svc",
-    "etcd-1.etcd.${ETCD_NAMESPACE}.svc.cluster.local",
-    "etcd-2",
-    "etcd-2.etcd",
-    "etcd-2.etcd.${ETCD_NAMESPACE}.svc",
-    "etcd-2.etcd.${ETCD_NAMESPACE}.cluster.local"
-  ]
-}
-EOF
-
-cfssl gencert -ca=kamaji/etcd/ca.crt -ca-key=kamaji/etcd/ca.key \
-  -config=cfssl-cert-config.json \
-  -profile=peer-authentication kamaji/etcd/peer-csr.json | cfssljson -bare kamaji/etcd/peer
-
-```
-
-Generate the `etcd` certificates for server:
-
-```
-cat << EOF | tee kamaji/etcd/server-csr.json
-{
-  "CN": "etcd",
-  "key": {
-    "algo": "rsa",
-    "size": 2048
-  },
-  "hosts": [
-    "127.0.0.1",
-    "etcd-server",
-    "etcd-server.${ETCD_NAMESPACE}.svc",
-    "etcd-server.${ETCD_NAMESPACE}.svc.cluster.local",
-    "etcd-0.etcd.${ETCD_NAMESPACE}.svc.cluster.local",
-    "etcd-1.etcd.${ETCD_NAMESPACE}.svc.cluster.local",
-    "etcd-2.etcd.${ETCD_NAMESPACE}.svc.cluster.local"
-  ]
-}
-EOF
-
-cfssl gencert -ca=kamaji/etcd/ca.crt -ca-key=kamaji/etcd/ca.key \
-  -config=cfssl-cert-config.json \
-  -profile=peer-authentication kamaji/etcd/server-csr.json | cfssljson -bare kamaji/etcd/server
-```
-
-Generate certificates for the `root` user of the `etcd`
-
-```
-cat << EOF | tee kamaji/etcd/root-csr.json
-{
-  "CN": "root",
-  "key": {
-    "algo": "rsa",
-    "size": 2048
-  }
-}
-EOF
-
-cfssl gencert -ca=kamaji/etcd/ca.crt -ca-key=kamaji/etcd/ca.key \
-  -config=cfssl-cert-config.json \
-  -profile=client-authentication kamaji/etcd/root-csr.json | cfssljson -bare kamaji/etcd/root	
-```
-
-Install the `etcd` in the Kamaji admin cluster
-
-```bash
-kubectl create namespace ${ETCD_NAMESPACE}
-
-kubectl -n ${ETCD_NAMESPACE} create secret generic etcd-certs \
-  --from-file=kamaji/etcd/ca.crt \
-  --from-file=kamaji/etcd/ca.key \
-  --from-file=kamaji/etcd/peer-key.pem --from-file=kamaji/etcd/peer.pem \
-  --from-file=kamaji/etcd/server-key.pem --from-file=kamaji/etcd/server.pem
-
-kubectl -n ${ETCD_NAMESPACE} apply -f etcd/etcd-cluster.yaml
-```
-
-Install an `etcd` client to interact with the `etcd` server
-
-```bash
-kubectl -n ${ETCD_NAMESPACE} create secret tls root-certs \
-  --key=kamaji/etcd/root-key.pem \
-  --cert=kamaji/etcd/root.pem
-
-kubectl -n ${ETCD_NAMESPACE} apply -f etcd/etcd-client.yaml
-```
-
-Wait the etcd instances discover each other and the cluster is formed:
-
-```bash
-kubectl -n ${ETCD_NAMESPACE} wait pod --for=condition=ready -l app=etcd --timeout=120s
-echo -n "\nChecking endpoint's health..."
-
-kubectl -n ${ETCD_NAMESPACE} exec etcd-root-client -- /bin/bash -c "etcdctl endpoint health 1>/dev/null 2>/dev/null; until [ \$$? -eq 0 ]; do sleep 10; printf "."; etcdctl endpoint health 1>/dev/null 2>/dev/null; done;"
-echo -n "\netcd cluster's health:\n"
-
-kubectl -n ${ETCD_NAMESPACE} exec etcd-root-client -- /bin/bash -c "etcdctl endpoint health"
-echo -n "\nWaiting for all members..."
-
-kubectl -n ${ETCD_NAMESPACE} exec etcd-root-client -- /bin/bash -c "until [ \$$(etcdctl member list 2>/dev/null | wc -l) -eq 3 ]; do sleep 10; printf '.'; done;"
-	@echo -n "\netcd's members:\n"
-
-kubectl -n ${ETCD_NAMESPACE} exec etcd-root-client -- /bin/bash -c "etcdctl member list -w table"
-```
-
-### Enable multi-tenancy
-The `root` user has full access to `etcd`, must be created before activating authentication. The `root` user must have the `root` role and is allowed to change anything inside `etcd`.
-
-```bash
-kubectl -n ${ETCD_NAMESPACE} exec etcd-root-client -- etcdctl user add --no-password=true root
-kubectl -n ${ETCD_NAMESPACE} exec etcd-root-client -- etcdctl role add root
-kubectl -n ${ETCD_NAMESPACE} exec etcd-root-client -- etcdctl user grant-role root root
-kubectl -n ${ETCD_NAMESPACE} exec etcd-root-client -- etcdctl auth enable
-```
-
-
-## Install Kamaji controller
-Currently, the behaviour of the Kamaji controller for Tenant Control Plane is controlled by (in this order):
-
-- CLI flags
-- Environment variables
-- Configuration file `kamaji.yaml` built into the image
-
-By default Kamaji search for the configuration file and uses parameters found inside of it. In case some environment variable are passed, this will override configuration file parameters. In the end, if also a CLI flag is passed, this will override both env vars and config file as well.
-
-There are multiple ways to deploy the Kamaji controller:
-
-- Use the single YAML file installer
-- Use Kustomize with Makefile
-- Use the Kamaji Helm Chart
-
-The Kamaji controller needs to access the multi-tenant `etcd` in order to provision the access for tenant `kube-apiserver`.
-
-Create the secrets containing the `etcd` certificates
-
-```bash
-kubectl create namespace kamaji-system
-kubectl -n kamaji-system create secret generic etcd-certs \
-  --from-file=kamaji/etcd/ca.crt \
-  --from-file=kamaji/etcd/ca.key
-
-kubectl -n kamaji-system create secret tls root-client-certs \
-  --cert=kamaji/etcd/root.crt \
-  --key=kamaji/etcd/root.key
-```
-
-### Install with a single manifest
-Install with the single YAML file installer:
-
-```bash
-kubectl -n kamaji-system apply -f ../config/install.yaml
-```
-
-Make sure to patch the `etcd` endpoints of the Kamaji controller, according to your environment:
-
-```bash
-cat > patch-deploy.yaml <<EOF 
+$ kubectl apply -f - <<EOF
+apiVersion: kamaji.clastix.io/v1alpha1
+kind: TenantControlPlane
+metadata:
+  name: tenant1
 spec:
-  template:
-    spec:
-      containers:
-      - name: manager
-        args:
-        - --health-probe-bind-address=:8081
-        - --metrics-bind-address=127.0.0.1:8080
-        - --leader-elect
-        - --etcd-endpoints=${ETCD0}:2379,${ETCD1}:2379,${ETCD2}:2379
+  controlPlane:
+    deployment:
+      replicas: 2
+      additionalMetadata:
+        annotations:
+          environment.clastix.io: tenant1
+          tier.clastix.io: "0"
+        labels:
+          tenant.clastix.io: tenant1
+          kind.clastix.io: deployment
+    service:
+      additionalMetadata:
+        annotations:
+          environment.clastix.io: tenant1
+          tier.clastix.io: "0"
+        labels:
+          tenant.clastix.io: tenant1
+          kind.clastix.io: service
+      serviceType: NodePort
+    ingress:
+      enabled: false
+  kubernetes:
+    version: "v1.23.4"
+    kubelet:
+      cgroupfs: cgroupfs
+    admissionControllers:
+    - LimitRanger
+    - ResourceQuota
+  networkProfile:
+    address: "172.18.0.2"
+    port: 31443
+    certSANs:
+    - "test.clastixlabs.io"
+    serviceCidr: "10.96.0.0/16"
+    podCidr: "10.244.0.0/16"
+    dnsServiceIPs: 
+    - "10.96.0.10"
+  addons:
+    coreDNS: {}
+    kubeProxy: {}
 EOF
-
-kubectl -n kamaji-system patch \
-  deployment kamaji-controller-manager \
-  --patch-file patch-deploy.yaml
 ```
 
-The Kamaji Tenant Control Plane controller is now running on the Admin Cluster:
+> Check networkProfile fields according to your installation
+> To let Kamaji works in kind, you have indicate that the service must be [NodePort](https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport)
+
+### Get Kubeconfig
+
+Let's retrieve kubeconfig and store in `/tmp/kubeconfig`
 
 ```bash
-kubectl -n kamaji-system get deploy
-NAME                          READY   UP-TO-DATE   AVAILABLE   AGE
-operator-controller-manager   1/1     1            1           14h
+$ kubectl get secrets tenant1-admin-kubeconfig -o json \
+ | jq -r '.data["admin.conf"]' \
+ | base64 -d > /tmp/kubeconfig
+ ```
+
+It can be export it, to facilitate the next tasks:
+
+```bash
+$ export KUBECONFIG=/tmp/kubeconfig
 ```
 
-## Setup Tenant Cluster
-Now you are getting an Admin Cluster available to run multiple Tenant Control Planes, deployed by the Kamaji controller. Please, refer to the Kamaji Tenant Deployment [guide](./kamaji-tenant-deployment-guide.md).
+### Install CNI
 
+We highly recommend to install [kindnet](https://github.com/aojea/kindnet) as CNI for your kamaji TCP.
+
+```bash
+$ kubectl create -f https://raw.githubusercontent.com/aojea/kindnet/master/install-kindnet.yaml
+```
+
+### Join worker nodes
+
+```bash
+$ make kamaji-kind-worker-join
+```
+
+> To add more worker nodes, run again the command above.
+
+Check out the node:
+
+```bash
+$ kubectl get nodes
+NAME           STATUS   ROLES    AGE   VERSION
+d2d4b468c9de   Ready    <none>   44s   v1.23.4
+```
+
+> For more complex scenarios (exposing port, different version and so on), run `join-node.bash`
+
+Tenant control plane provision has been finished in a minimal Kamaji setup based on KinD. Therefore, you could develop, test and make your own experiments with Kamaji.
+
+## Cleanup
+
+```bash
+$ make destroy
+```

docs/kamaji-azure-deployment-guide.md

@@ -67,10 +67,10 @@ kubectl cluster-info
 ```
 
 ## Setup internal multi-tenant etcd
-Follow the instructions [here](./getting-started-with-kamaji.md#setup-internal-multi-tenant-etcd).
+Follow the instructions [here](./kamaji-deployment-guide.md#setup-internal-multi-tenant-etcd).
 
 ## Install Kamaji controller
-Follow the instructions [here](./getting-started-with-kamaji.md#install-kamaji-controller).
+Follow the instructions [here](./kamaji-deployment-guide.md#install-kamaji-controller).
 
 ## Create Tenant Clusters
 To create a Tenant Cluster in Kamaji on AKS, we have to work on both the Kamaji and Azure infrastructure sides.
@@ -82,21 +82,10 @@ source kamaji-tenant-azure.env
 ### On Kamaji side
 With Kamaji on AKS, the tenant control plane is accessible:
 
-- from tenant work nodes through an internal loadbalancer as `https://${TENANT_ADDR}:${TENANT_PORT}`
+- from tenant work nodes through an internal loadbalancer as `https://${TENANT_ADDR}:6443`
 - from tenant admin user through an external loadbalancer `https://${TENANT_NAME}.${KAMAJI_REGION}.cloudapp.azure.com:443`
 
-#### Allocate an internal IP address for the Tenant Control Plane
-Currently, Kamaji has a known limitation, meaning the address `${TENANT_ADDR}:${TENANT_PORT}` must be known in advance before to create the Tenant Control Plane. Given this limitation, let's to reserve an IP address and port in the same virtual subnet used by the Kamaji admin cluster:
-
-
-```bash
-export TENANT_ADDR=10.240.0.100
-export TENANT_PORT=6443
-export TENANT_DOMAIN=$KAMAJI_REGION.cloudapp.azure.com
-```
-
-> Make sure the `TENANT_ADDR` value does not overlap with already allocated IP addresses in the AKS virtual network. In the future, Kamaji will implement a dynamic IP allocation.
-
+Where `TENANT_ADDR` is the Azure internal IP address assigned to the LoadBalancer service created by Kamaji to expose the Tenant Control Plane endpoint.
 
 #### Create the Tenant Control Plane
 
@@ -139,13 +128,16 @@ spec:
       - ResourceQuota
       - LimitRanger
   networkProfile:
-    address: ${TENANT_ADDR}
-    port: ${TENANT_PORT}
-    domain: ${TENANT_DOMAIN}
+    port: 6443
+    certSANs:
+    - ${TENANT_NAME}.${KAMAJI_REGION}.cloudapp.azure.com
     serviceCidr: ${TENANT_SVC_CIDR}
     podCidr: ${TENANT_POD_CIDR}
     dnsServiceIPs:
     - ${TENANT_DNS_SERVICE}
+  addons:
+    coreDNS: {}
+    kubeProxy: {}
 ---
 apiVersion: v1
 kind: Service
@@ -195,6 +187,11 @@ NAME        READY   UP-TO-DATE   AVAILABLE   AGE
 tenant-00   2/2     2            2           47m
 ```
 
+Collect the internal IP address of Azure loadbalancer where the Tenant control Plane is exposed:
+
+```bash
+TENANT_ADDR=$(kubectl -n ${TENANT_NAMESPACE} get svc ${TENANT_NAME} -o json | jq -r ."status.loadBalancer.ingress[].ip")
+```
 
 #### Working with Tenant Control Plane
 Check the access to the Tenant Control Plane:
@@ -225,7 +222,7 @@ NAMESPACE     NAME         TYPE        CLUSTER-IP   EXTERNAL-IP   PORT(S)     AG
 default       kubernetes   ClusterIP   10.32.0.1    <none>        443/TCP     6m
 ```
 
-Check out how the Tenant Control Plane advertises itself to workloads:
+Check out how the Tenant Control Plane advertises itself:
 
 ```
 kubectl --kubeconfig=${TENANT_NAMESPACE}-${TENANT_NAME}.kubeconfig get ep
@@ -234,7 +231,7 @@ NAME         ENDPOINTS           AGE
 kubernetes   10.240.0.100:6443   57m
 ```
 
-Make sure it's `${TENANT_ADDR}:${TENANT_PORT}`.
+Make sure it's `${TENANT_ADDR}:6443`.
 
 ### Prepare the Infrastructure for the Tenant virtual machines
 Kamaji provides Control Plane as a Service, so the tenant user can join his own virtual machines as worker nodes. Each tenant can place his virtual machines in a dedicated Azure virtual network.
@@ -332,7 +329,7 @@ az vmss scale \
 The current approach for joining nodes is to use the `kubeadm` one therefore, we will create a bootstrap token to perform the action:
 
 ```bash
-JOIN_CMD=$(echo "sudo kubeadm join ${TENANT_ADDR}:${TENANT_PORT} ")$(kubeadm --kubeconfig=${TENANT_NAMESPACE}-${TENANT_NAME}.kubeconfig token create --print-join-command |cut -d" " -f4-)
+JOIN_CMD=$(echo "sudo kubeadm join ${TENANT_ADDR}:6443 ")$(kubeadm --kubeconfig=${TENANT_NAMESPACE}-${TENANT_NAME}.kubeconfig token create --print-join-command |cut -d" " -f4-)
 ```
 
 A bash loop will be used to join all the available nodes.
@@ -398,6 +395,16 @@ kamaji-tenant-worker-02   Ready       <none>   10m    v1.23.4
 
 
 ## Cleanup
-To get rid of the Tenant infrastructure, remove the RESOURCE_GROUP:  `az group delete --name $TENANT_RG --yes --no-wait`.
+To get rid of the Tenant infrastructure, remove the RESOURCE_GROUP:
 
-To get rid of the Kamaji infrastructure, remove the RESOURCE_GROUP:  `az group delete --name $KAMAJI_RG --yes --no-wait`.
+```
+az group delete --name $TENANT_RG --yes --no-wait
+```
+
+To get rid of the Kamaji infrastructure, remove the RESOURCE_GROUP:
+
+```
+az group delete --name $KAMAJI_RG --yes --no-wait
+```
+
+That's all folks!

docs/kamaji-deployment-guide.md

@@ -0,0 +1,603 @@
+# Install a Kamaji environment
+This guide will lead you through the process of creating a basic working Kamaji setup.
+
+Kamaji requires:
+
+- (optional) a bootstrap node;
+- a multi-tenant `etcd` cluster made of 3 nodes hosting the datastore for the `Tenant`s' clusters
+- a Kubernetes cluster, running the admin and Tenant Control Planes
+- an arbitrary number of machines hosting `Tenant`s' workloads
+
+> In this guide, we assume all machines are running `Ubuntu 20.04`.
+
+  * [Prepare the bootstrap workspace](#prepare-the-bootstrap-workspace)
+  * [Access Admin cluster](#access-admin-cluster)
+  * [Setup external multi-tenant etcd](#setup-external-multi-tenant-etcd)
+  * [Setup internal multi-tenant etcd](#setup-internal-multi-tenant-etcd)
+  * [Install Kamaji controller](#install-kamaji-controller)
+  * [Setup Tenant cluster](#setup-tenant-cluster)
+
+## Prepare the bootstrap workspace
+This guide is supposed to be run from a remote or local bootstrap machine.
+First, prepare the workspace directory:
+
+```
+git clone https://github.com/clastix/kamaji
+cd kamaji/deploy
+```
+
+Throughout the instructions, shell variables are used to indicate values that you should adjust to your own environment.
+
+### Install required tools
+On the bootstrap machine, install all the required tools to work with a Kamaji setup.
+
+#### cfssl and cfssljson
+The `cfssl` and `cfssljson` command line utilities will be used in addition to `kubeadm` to provision the PKI Infrastructure and generate TLS certificates.
+
+```
+wget -q --show-progress --https-only --timestamping \
+https://storage.googleapis.com/kubernetes-the-hard-way/cfssl/1.4.1/linux/cfssl \
+https://storage.googleapis.com/kubernetes-the-hard-way/cfssl/1.4.1/linux/cfssljson
+
+chmod +x cfssl cfssljson
+sudo mv cfssl cfssljson /usr/local/bin/
+```
+
+#### Kubernetes tools
+Install `kubeadm` and `kubectl`
+
+```bash
+sudo apt update && sudo apt install -y apt-transport-https ca-certificates curl && \
+sudo curl -fsSLo /usr/share/keyrings/kubernetes-archive-keyring.gpg https://packages.cloud.google.com/apt/doc/apt-key.gpg && \
+echo "deb [signed-by=/usr/share/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" | sudo tee /etc/apt/sources.list.d/kubernetes.list && \
+sudo apt update && sudo apt install -y kubeadm kubectl --allow-change-held-packages && \
+sudo apt-mark hold kubeadm kubectl
+```
+
+#### etcdctl
+For administration of the `etcd` cluster, download and install the `etcdctl` CLI utility on the bootstrap machine
+
+```bash
+ETCD_VER=v3.5.1
+ETCD_URL=https://storage.googleapis.com/etcd
+curl -L ${ETCD_URL}/${ETCD_VER}/etcd-${ETCD_VER}-linux-amd64.tar.gz -o etcd-${ETCD_VER}-linux-amd64.tar.gz
+tar xzvf etcd-${ETCD_VER}-linux-amd64.tar.gz etcd-${ETCD_VER}-linux-amd64/etcdctl
+sudo cp etcd-${ETCD_VER}-linux-amd64/etcdctl /usr/bin/etcdctl
+rm -rf etcd-${ETCD_VER}-linux-amd64*
+```
+
+Verify `etcdctl` version is installed
+
+```bash
+etcdctl version
+etcdctl version: 3.5.1
+API version: 3.5
+```
+
+
+## Access Admin cluster
+In Kamaji, an Admin Cluster is a regular Kubernetes cluster which hosts zero to many Tenant Cluster Control Planes running as pods. The admin cluster acts as management cluster for all the Tenant clusters and implements Monitoring, Logging, and Governance of all the Kamaji setup, including all Tenant clusters. 
+
+Any regular and conformant Kubernetes v1.22+ cluster can be turned into a Kamaji setup. Currently we tested:
+
+- [Kubernetes installed with `kubeadm`](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/).
+- [Azure AKS managed service](./kamaji-on-azure.md).
+- [KinD for local development](./getting-started-with-kamaji.md ).
+
+The admin cluster should provide:
+
+- CNI module installed, eg. Calico
+- Support for LoadBalancer Service Type, eg. MetalLB or, alternatively, an Ingress Controller
+- CSI module installed with StorageClass for multi-tenant `etcd`
+- Monitoring Stack, eg. Prometheus and Grafana
+
+Make sure you have a `kubeconfig` file with admin permissions on the cluster you want to turn into Kamaji Admin Cluster.
+
+## Setup external multi-tenant etcd
+In this section, we're going to setup a multi-tenant `etcd` cluster on dedicated nodes. Alternatively, if you want to use an internal `etcd` cluster as Kubernetes StatefulSet, jump [here](#setup-internal-multi-tenant-etcd).
+
+### Ensure host access
+From the bootstrap machine load the environment for external `etcd` setup:
+
+```bash
+source kamaji-external-etcd.env
+```
+
+The installer requires a user that has access to all hosts. In order to run the installer as a non-root user, first configure passwordless sudo rights each host:
+
+Generate an SSH key on the host you run the installer on:
+
+```bash
+ssh-keygen -t rsa
+```
+
+> Do not use a password.
+
+Distribute the key to the other cluster hosts.
+
+Depending on your environment, use a bash loop:
+
+```bash
+HOSTS=(${ETCD0} ${ETCD1} ${ETCD2})
+for i in "${!HOSTS[@]}"; do
+  HOST=${HOSTS[$i]}
+  ssh-copy-id -i ~/.ssh/id_rsa.pub $HOST;
+done
+```
+
+> Alternatively, inject the generated public key into machines metadata.
+
+Confirm that you can access each host from bootstrap machine:
+
+```bash
+HOSTS=(${ETCD0} ${ETCD1} ${ETCD2})
+for i in "${!HOSTS[@]}"; do
+  HOST=${HOSTS[$i]}
+  ssh ${USER}@${HOST} -t 'hostname';
+done
+```
+
+### Configure disk layout
+As per `etcd` [requirements](https://etcd.io/docs/v3.5/op-guide/hardware/#disks), back `etcd`’s storage with a SSD. A SSD usually provides lower write latencies and with less variance than a spinning disk, thus improving the stability and reliability of `etcd`.
+
+For each `etcd` machine, we assume an additional `sdb` disk of 10GB:
+
+```
+clastix@kamaji-etcd-00:~$ lsblk
+NAME    MAJ:MIN RM  SIZE RO TYPE MOUNTPOINT
+sda       8:0    0   16G  0 disk 
+├─sda1    8:1    0 15.9G  0 part /
+├─sda14   8:14   0    4M  0 part 
+└─sda15   8:15   0  106M  0 part /boot/efi
+sdb       8:16   0   10G  0 disk 
+sr0      11:0    1    4M  0 rom  
+```
+
+Create partition, format, and mount the `etcd` disk, by running the script below from the bootstrap machine:
+
+> If you already used the `etcd` disks, please make sure to wipe the partitions with `sudo wipefs --all --force /dev/sdb` before to attempt to recreate them.
+
+```bash
+for i in "${!ETCDHOSTS[@]}"; do
+  HOST=${ETCDHOSTS[$i]}
+  ssh ${USER}@${HOST} -t 'echo type=83 | sudo sfdisk -f -q /dev/sdb'
+  ssh ${USER}@${HOST} -t 'sudo mkfs -F -q -t ext4 /dev/sdb1'
+  ssh ${USER}@${HOST} -t 'sudo mkdir -p /var/lib/etcd'
+  ssh ${USER}@${HOST} -t 'sudo e2label /dev/sdb1 ETCD'
+  ssh ${USER}@${HOST} -t 'echo LABEL=ETCD /var/lib/etcd ext4 defaults 0 1 | sudo tee -a /etc/fstab'
+  ssh ${USER}@${HOST} -t 'sudo mount -a'
+  ssh ${USER}@${HOST} -t 'sudo lsblk -f'
+done
+```
+
+### Install prerequisites
+Use bash script `nodes-prerequisites.sh` to install all the dependencies on all the cluster nodes:
+
+- Install `containerd` as container runtime
+- Install `crictl`, the command line for working with `containerd`
+- Install `kubectl`, `kubelet`, and `kubeadm` in the desired version, eg. `v1.24.0`
+
+Run the installation script:
+
+```bash
+VERSION=v1.24.0
+./nodes-prerequisites.sh ${VERSION:1} ${HOSTS[@]}
+```
+
+### Configure kubelet
+
+On each `etcd` node, configure the `kubelet` service to start `etcd` static pods using `containerd` as container runtime, by running the script below from the bootstrap machine:
+
+```bash
+cat << EOF > 20-etcd-service-manager.conf
+[Service]
+ExecStart=
+ExecStart=/usr/bin/kubelet --address=127.0.0.1 --pod-manifest-path=/etc/kubernetes/manifests --cgroup-driver=systemd --container-runtime=remote --container-runtime-endpoint=/run/containerd/containerd.sock
+Restart=always
+EOF
+```
+
+```
+for i in "${!ETCDHOSTS[@]}"; do
+  HOST=${ETCDHOSTS[$i]}
+  scp 20-etcd-service-manager.conf ${USER}@${HOST}:
+  ssh ${USER}@${HOST} -t 'sudo chown -R root:root 20-etcd-service-manager.conf && sudo mv 20-etcd-service-manager.conf /etc/systemd/system/kubelet.service.d/20-etcd-service-manager.conf'
+  ssh ${USER}@${HOST} -t 'sudo systemctl daemon-reload'
+  ssh ${USER}@${HOST} -t 'sudo systemctl start kubelet'
+  ssh ${USER}@${HOST} -t 'sudo systemctl enable kubelet'
+done
+
+rm -f 20-etcd-service-manager.conf
+```
+
+### Create configuration
+Create temp directories to store files that will end up on `etcd` hosts:
+
+```bash
+mkdir -p /tmp/${ETCD0}/ /tmp/${ETCD1}/ /tmp/${ETCD2}/
+NAMES=("etcd00" "etcd01" "etcd02")
+
+for i in "${!ETCDHOSTS[@]}"; do
+HOST=${ETCDHOSTS[$i]}
+NAME=${NAMES[$i]}
+
+cat <<EOF | sudo tee /tmp/${HOST}/kubeadmcfg.yaml
+apiVersion: "kubeadm.k8s.io/v1beta2"
+kind: ClusterConfiguration
+etcd:
+  local:
+    serverCertSANs:
+    - "${HOST}"
+    peerCertSANs:
+    - "${HOST}"
+    extraArgs:
+      initial-cluster: ${NAMES[0]}=https://${ETCDHOSTS[0]}:2380,${NAMES[1]}=https://${ETCDHOSTS[1]}:2380,${NAMES[2]}=https://${ETCDHOSTS[2]}:2380
+      initial-cluster-state: new
+      name: ${NAME}
+      listen-peer-urls: https://${HOST}:2380
+      listen-client-urls: https://${HOST}:2379
+      advertise-client-urls: https://${HOST}:2379
+      initial-advertise-peer-urls: https://${HOST}:2380
+      auto-compaction-mode: periodic
+      auto-compaction-retention: 5m
+      quota-backend-bytes: '8589934592'
+EOF
+done
+```
+> Note:
+>
+> ##### Etcd compaction
+>
+> By enabling `etcd` authentication, it prevents the tenant apiservers (clients of `etcd`) to issue compaction requests. We set `etcd` to automatically compact the keyspace with the `--auto-compaction-*` option with a period of hours or minutes. When `--auto-compaction-mode=periodic` and `--auto-compaction-retention=5m` and writes per minute are about 1000, `etcd` compacts revision 5000 for every 5 minute. 
+> 
+> ##### Etcd storage quota
+>
+> Currently, `etcd` is limited in storage size, defaulted to `2GB` and configurable with `--quota-backend-bytes` flag up to `8GB`. In Kamaji, we use a single `etcd` to store multiple tenant clusters, so we need to increase this size. Please, note `etcd` warns at startup if the configured value exceeds `8GB`.
+
+### Generate certificates 
+On the bootstrap machine, using `kubeadm` init phase, create and distribute `etcd` CA certificates:
+
+```bash
+sudo kubeadm init phase certs etcd-ca
+mkdir kamaji
+sudo cp -r /etc/kubernetes/pki/etcd kamaji
+sudo chown -R ${USER}. kamaji/etcd
+```
+
+For each `etcd` host: 
+
+```bash
+for i in "${!ETCDHOSTS[@]}"; do
+  HOST=${ETCDHOSTS[$i]}
+  sudo kubeadm init phase certs etcd-server --config=/tmp/${HOST}/kubeadmcfg.yaml
+  sudo kubeadm init phase certs etcd-peer --config=/tmp/${HOST}/kubeadmcfg.yaml
+  sudo kubeadm init phase certs etcd-healthcheck-client --config=/tmp/${HOST}/kubeadmcfg.yaml
+  sudo cp -R /etc/kubernetes/pki /tmp/${HOST}/
+  sudo find /etc/kubernetes/pki -not -name ca.crt -not -name ca.key -type f -delete
+done
+```
+
+### Startup the cluster
+Upload certificates on each `etcd` node and restart the `kubelet`
+
+```bash
+for i in "${!ETCDHOSTS[@]}"; do
+  HOST=${ETCDHOSTS[$i]}
+  sudo chown -R ${USER}. /tmp/${HOST}
+  scp -r /tmp/${HOST}/* ${USER}@${HOST}:
+  ssh ${USER}@${HOST} -t 'sudo chown -R root:root pki'
+  ssh ${USER}@${HOST} -t 'sudo mv pki /etc/kubernetes/'
+  ssh ${USER}@${HOST} -t 'sudo kubeadm init phase etcd local --config=kubeadmcfg.yaml'
+  ssh ${USER}@${HOST} -t 'sudo systemctl daemon-reload'
+  ssh ${USER}@${HOST} -t 'sudo systemctl restart kubelet'
+done
+```
+
+This will start the static `etcd` pod on each node and then the cluster gets formed.
+
+Generate certificates for the `root` user
+
+```bash
+cat > root-csr.json <<EOF  
+{
+  "CN": "root",
+  "key": {
+    "algo": "rsa",
+    "size": 2048
+  }
+}
+EOF
+```
+
+```bash
+cfssl gencert \
+  -ca=kamaji/etcd/ca.crt \
+  -ca-key=kamaji/etcd/ca.key \
+  -config=cfssl-cert-config.json \
+  -profile=client-authentication \
+  root-csr.json | cfssljson -bare root
+```
+
+```bash
+cp root.pem kamaji/etcd/root.crt
+cp root-key.pem kamaji/etcd/root.key
+rm root*
+```
+
+The result should be:
+
+```bash
+$ tree kamaji
+kamaji
+└── etcd
+    ├── ca.crt
+    ├── ca.key
+    ├── root.crt
+    └── root.key
+```
+
+Use the `root` user to check the just formed `etcd` cluster is in health state 
+
+```bash
+export ETCDCTL_CACERT=kamaji/etcd/ca.crt
+export ETCDCTL_CERT=kamaji/etcd/root.crt
+export ETCDCTL_KEY=kamaji/etcd/root.key
+export ETCDCTL_ENDPOINTS=https://${ETCD0}:2379
+
+etcdctl member list -w table
+```
+
+The result should be something like this:
+
+```
++------------------+---------+--------+----------------------------+----------------------------+------------+
+|        ID        | STATUS  |  NAME  |         PEER ADDRS         |        CLIENT ADDRS        | IS LEARNER |
++------------------+---------+--------+----------------------------+----------------------------+------------+
+| 72657d6307364226 | started | etcd01 | https://192.168.32.11:2380 | https://192.168.32.11:2379 |      false |
+| 91eb892c5ee87610 | started | etcd00 | https://192.168.32.10:2380 | https://192.168.32.10:2379 |      false |
+| e9971c576949c34e | started | etcd02 | https://192.168.32.12:2380 | https://192.168.32.12:2379 |      false |
++------------------+---------+--------+----------------------------+----------------------------+------------+
+```
+
+### Enable multi-tenancy
+The `root` user has full access to `etcd`, must be created before activating authentication. The `root` user must have the `root` role and is allowed to change anything inside `etcd`.
+
+```bash
+etcdctl user add --no-password=true root
+etcdctl role add root
+etcdctl user grant-role root root
+etcdctl auth enable
+```
+
+### Cleanup
+If you want to get rid of the etcd cluster, for each node, login and clean it:
+
+```bash
+HOSTS=(${ETCD0} ${ETCD1} ${ETCD2})
+for i in "${!HOSTS[@]}"; do
+  HOST=${HOSTS[$i]}
+  ssh ${USER}@${HOST} -t 'sudo kubeadm reset -f';
+  ssh ${USER}@${HOST} -t 'sudo systemctl reboot';
+done
+```
+
+## Setup internal multi-tenant etcd
+If you opted for an internal etcd cluster running in the Kamaji admin cluster, follow steps below.
+
+From the bootstrap machine load the environment for internal `etcd` setup:
+
+```bash
+source kamaji-internal-etcd.env
+```
+
+### Generate certificates 
+On the bootstrap machine, using `kubeadm` init phase, create the `etcd` CA certificates:
+
+```bash
+sudo kubeadm init phase certs etcd-ca
+mkdir kamaji
+sudo cp -r /etc/kubernetes/pki/etcd kamaji
+sudo chown -R ${USER}. kamaji/etcd
+```
+
+Generate the `etcd` certificates for peers:
+
+```
+cat << EOF | tee kamaji/etcd/peer-csr.json
+{
+  "CN": "etcd",
+  "key": {
+    "algo": "rsa",
+    "size": 2048
+  },
+  "hosts": [
+    "127.0.0.1",
+    "etcd-0",
+    "etcd-0.etcd",
+    "etcd-0.etcd.${ETCD_NAMESPACE}.svc",
+    "etcd-0.etcd.${ETCD_NAMESPACE}.svc.cluster.local",
+    "etcd-1",
+    "etcd-1.etcd",
+    "etcd-1.etcd.${ETCD_NAMESPACE}.svc",
+    "etcd-1.etcd.${ETCD_NAMESPACE}.svc.cluster.local",
+    "etcd-2",
+    "etcd-2.etcd",
+    "etcd-2.etcd.${ETCD_NAMESPACE}.svc",
+    "etcd-2.etcd.${ETCD_NAMESPACE}.cluster.local"
+  ]
+}
+EOF
+
+cfssl gencert -ca=kamaji/etcd/ca.crt -ca-key=kamaji/etcd/ca.key \
+  -config=cfssl-cert-config.json \
+  -profile=peer-authentication kamaji/etcd/peer-csr.json | cfssljson -bare kamaji/etcd/peer
+
+```
+
+Generate the `etcd` certificates for server:
+
+```
+cat << EOF | tee kamaji/etcd/server-csr.json
+{
+  "CN": "etcd",
+  "key": {
+    "algo": "rsa",
+    "size": 2048
+  },
+  "hosts": [
+    "127.0.0.1",
+    "etcd-server",
+    "etcd-server.${ETCD_NAMESPACE}.svc",
+    "etcd-server.${ETCD_NAMESPACE}.svc.cluster.local",
+    "etcd-0.etcd.${ETCD_NAMESPACE}.svc.cluster.local",
+    "etcd-1.etcd.${ETCD_NAMESPACE}.svc.cluster.local",
+    "etcd-2.etcd.${ETCD_NAMESPACE}.svc.cluster.local"
+  ]
+}
+EOF
+
+cfssl gencert -ca=kamaji/etcd/ca.crt -ca-key=kamaji/etcd/ca.key \
+  -config=cfssl-cert-config.json \
+  -profile=peer-authentication kamaji/etcd/server-csr.json | cfssljson -bare kamaji/etcd/server
+```
+
+Generate certificates for the `root` user of the `etcd`
+
+```
+cat << EOF | tee kamaji/etcd/root-csr.json
+{
+  "CN": "root",
+  "key": {
+    "algo": "rsa",
+    "size": 2048
+  }
+}
+EOF
+
+cfssl gencert -ca=kamaji/etcd/ca.crt -ca-key=kamaji/etcd/ca.key \
+  -config=cfssl-cert-config.json \
+  -profile=client-authentication kamaji/etcd/root-csr.json | cfssljson -bare kamaji/etcd/root	
+```
+
+Install the `etcd` in the Kamaji admin cluster
+
+```bash
+kubectl create namespace ${ETCD_NAMESPACE}
+
+kubectl -n ${ETCD_NAMESPACE} create secret generic etcd-certs \
+  --from-file=kamaji/etcd/ca.crt \
+  --from-file=kamaji/etcd/ca.key \
+  --from-file=kamaji/etcd/peer-key.pem --from-file=kamaji/etcd/peer.pem \
+  --from-file=kamaji/etcd/server-key.pem --from-file=kamaji/etcd/server.pem
+
+kubectl -n ${ETCD_NAMESPACE} apply -f etcd/etcd-cluster.yaml
+```
+
+Install an `etcd` client to interact with the `etcd` server
+
+```bash
+kubectl -n ${ETCD_NAMESPACE} create secret tls root-client-certs \
+  --key=kamaji/etcd/root-key.pem \
+  --cert=kamaji/etcd/root.pem
+
+kubectl -n ${ETCD_NAMESPACE} apply -f etcd/etcd-client.yaml
+```
+
+Wait the etcd instances discover each other and the cluster is formed:
+
+```bash
+kubectl -n ${ETCD_NAMESPACE} wait pod --for=condition=ready -l app=etcd --timeout=120s
+echo -n "\nChecking endpoint's health..."
+
+kubectl -n ${ETCD_NAMESPACE} exec etcd-root-client -- /bin/bash -c "etcdctl endpoint health 1>/dev/null 2>/dev/null; until [ \$$? -eq 0 ]; do sleep 10; printf "."; etcdctl endpoint health 1>/dev/null 2>/dev/null; done;"
+echo -n "\netcd cluster's health:\n"
+
+kubectl -n ${ETCD_NAMESPACE} exec etcd-root-client -- /bin/bash -c "etcdctl endpoint health"
+echo -n "\nWaiting for all members..."
+
+kubectl -n ${ETCD_NAMESPACE} exec etcd-root-client -- /bin/bash -c "until [ \$$(etcdctl member list 2>/dev/null | wc -l) -eq 3 ]; do sleep 10; printf '.'; done;"
+	@echo -n "\netcd's members:\n"
+
+kubectl -n ${ETCD_NAMESPACE} exec etcd-root-client -- /bin/bash -c "etcdctl member list -w table"
+```
+
+### Enable multi-tenancy
+The `root` user has full access to `etcd`, must be created before activating authentication. The `root` user must have the `root` role and is allowed to change anything inside `etcd`.
+
+```bash
+kubectl -n ${ETCD_NAMESPACE} exec etcd-root-client -- etcdctl user add --no-password=true root
+kubectl -n ${ETCD_NAMESPACE} exec etcd-root-client -- etcdctl role add root
+kubectl -n ${ETCD_NAMESPACE} exec etcd-root-client -- etcdctl user grant-role root root
+kubectl -n ${ETCD_NAMESPACE} exec etcd-root-client -- etcdctl auth enable
+```
+
+
+## Install Kamaji controller
+Currently, the behaviour of the Kamaji controller for Tenant Control Plane is controlled by (in this order):
+
+- CLI flags
+- Environment variables
+- Configuration file `kamaji.yaml` built into the image
+
+By default Kamaji search for the configuration file and uses parameters found inside of it. In case some environment variable are passed, this will override configuration file parameters. In the end, if also a CLI flag is passed, this will override both env vars and config file as well.
+
+There are multiple ways to deploy the Kamaji controller:
+
+- Use the single YAML file installer
+- Use Kustomize with Makefile
+- Use the Kamaji Helm Chart
+
+The Kamaji controller needs to access the multi-tenant `etcd` in order to provision the access for tenant `kube-apiserver`.
+
+Create the secrets containing the `etcd` certificates
+
+```bash
+kubectl create namespace kamaji-system
+kubectl -n kamaji-system create secret generic etcd-certs \
+  --from-file=kamaji/etcd/ca.crt \
+  --from-file=kamaji/etcd/ca.key
+
+kubectl -n kamaji-system create secret tls root-client-certs \
+  --cert=kamaji/etcd/root.crt \
+  --key=kamaji/etcd/root.key
+```
+
+### Install with a single manifest
+Install with the single YAML file installer:
+
+```bash
+kubectl -n kamaji-system apply -f ../config/install.yaml
+```
+
+Make sure to patch the `etcd` endpoints of the Kamaji controller, according to your environment:
+
+```bash
+cat > patch-deploy.yaml <<EOF 
+spec:
+  template:
+    spec:
+      containers:
+      - name: manager
+        args:
+        - --health-probe-bind-address=:8081
+        - --metrics-bind-address=127.0.0.1:8080
+        - --leader-elect
+        - --etcd-endpoints=${ETCD0}:2379,${ETCD1}:2379,${ETCD2}:2379
+EOF
+
+kubectl -n kamaji-system patch \
+  deployment kamaji-controller-manager \
+  --patch-file patch-deploy.yaml
+```
+
+The Kamaji Tenant Control Plane controller is now running on the Admin Cluster:
+
+```bash
+kubectl -n kamaji-system get deploy
+NAME                          READY   UP-TO-DATE   AVAILABLE   AGE
+operator-controller-manager   1/1     1            1           14h
+```
+
+## Setup Tenant Cluster
+Now you are getting an Admin Cluster available to run multiple Tenant Control Planes, deployed by the Kamaji controller. Please, refer to the Kamaji Tenant Deployment [guide](./kamaji-tenant-deployment-guide.md).
+

docs/kamaji-tenant-deployment-guide.md

@@ -62,14 +62,35 @@ spec:
   networkProfile:
     address: ${TENANT_ADDR}
     port: ${TENANT_PORT}
-    domain: ${TENANT_DOMAIN}
+    certSANs:
+    - ${TENANT_NAME}.${TENANT_DOMAIN}
     serviceCidr: ${TENANT_SVC_CIDR}
     podCidr: ${TENANT_POD_CIDR}
     dnsServiceIPs:
     - ${TENANT_DNS_SERVICE}
+  addons:
+    coreDNS: {}
+    kubeProxy: {}
 EOF
 ```
 
+If workers are not reachable from tenant control plane, konnectivity can be enabled (it is by default):
+
+```yaml
+...
+  addons:
+    konnectivity:
+      enabled: true
+      proxyHost: "172.18.0.2"
+      proxyPort: 31132
+...   
+```
+
+`proxyHost` is the address where konnectivity proxy server will be running. Konnectivity works as sidecar container into the tenant control plane pod. If no value is specified, it will take tenant IP.
+
+`proxyPort` is the port where konnectivity proxy server will be running. (default `8132`)
+
+
 ```bash
 kubectl create namespace ${TENANT_NAMESPACE}
 kubectl apply -f ${TENANT_NAMESPACE}-${TENANT_NAME}-tcp.yaml

docs/reference.md

@@ -21,8 +21,13 @@ Available flags are the following:
 --etcd-client-secret-name            Name of the secret which contains ETCD client certificates. (default: "root-client-certs")
 --etcd-client-secret-namespace       Name of the namespace where the secret which contains ETCD client certificates is. (default: "kamaji")
 --etcd-compaction-interval           ETCD Compaction interval (i.e. "5m0s"). (default: "0" (disabled))
---etcd-endpoints                     Comma-separated list with ETCD endpoints (i.e. etcd-0.etcd.kamaji.svc.cluster.local,etcd-1.etcd.kamaji.svc.cluster.local,etcd-2.etcd.kamaji.svc.cluster.local)
+--etcd-endpoints                     Comma-separated list with ETCD endpoints (i.e. https://etcd-0.etcd.kamaji.svc.cluster.local,https://etcd-1.etcd.kamaji.svc.cluster.local,https://etcd-2.etcd.kamaji.svc.cluster.local)
+--etcd-storage-type                  ETCD Storage type (i.e. "etcd", "kine-mysql"). (default: "etcd")
 --health-probe-bind-address string   The address the probe endpoint binds to. (default ":8081")
+--kine-mysql-host                    Host where MySQL is running (default: "localhost")
+--kine-mysql-port int                Port where MySQL is running (default: 3306)
+--kine-mysql-secret-name             Name of the secret where the necessary configuration and certificates are. (default: "mysql-config")
+--kine-mysql-secret-name             Name of the namespace of the secret where the necessary configuration and certificates are. (default: "kamaji-system")
 --kubeconfig string                  Paths to a kubeconfig. Only required if out-of-cluster.
 --leader-elect                       Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.
 --metrics-bind-address string        The address the metric endpoint binds to. (default ":8080")
@@ -43,9 +48,14 @@ Available environment variables are:
 | `KAMAJI_ETCD_CLIENT_SECRET_NAMESPACE`      | Name of the namespace where the secret which contains ETCD client certificates is. (default: "kamaji")  |
 | `KAMAJI_ETCD_COMPACTION_INTERVAL`      | ETCD Compaction interval (i.e. "5m0s"). (default: "0" (disabled))  |
 | `KAMAJI_ETCD_ENDPOINTS`      | Comma-separated list with ETCD endpoints (i.e. etcd-server-1:2379,etcd-server-2:2379). (default: "etcd-server:2379")  |
+| `KAMAJI_ETCD_STORAGE_TYPE`      | ETCD Storage type (i.e. "etcd", "kine-mysql"). (default: "etcd")  |
 | `KAMAJI_ETCD_SERVERS`      | Comma-separated list with ETCD servers (i.e. etcd-0.etcd.kamaji.svc.cluster.local,etcd-1.etcd.kamaji.svc.cluster.local,etcd-2.etcd.kamaji.svc.cluster.local)  |
 | `KAMAJI_METRICS_BIND_ADDRESS`      | The address the metric endpoint binds to. (default ":8080")  |
 | `KAMAJI_HEALTH_PROBE_BIND_ADDRESS` | The address the probe endpoint binds to. (default ":8081")   |
+| `KAMAJI_KINE_MYSQL_HOST` | Host where MySQL is running(default "localhost")   |
+| `KAMAJI_KINE_MYSQL_PORT` | Port where MySQL is running (default: 3306)   |
+| `KAMAJI_KINE_MYSQL_SECRET_NAME` | Name of the secret where the necessary configuration and certificates are. (default: "mysql-config")   |
+| `KAMAJI_KINE_MYSQL_SECRET_NAMESPACE` | Name of the namespace of the secret where the necessary configuration and certificates are. (default: "kamaji-system")   |
 | `KAMAJI_LEADER_ELECTION`           | Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager. |
 | `KAMAJI_TMP_DIRECTORY`           | Directory which will be used to work with temporary files. (default "/tmp/kamaji") |
 
@@ -80,4 +90,38 @@ $ make yaml-installation-file
 
 ```
 
-It will generate a yaml installation file at `config/installation.yaml`. It should be customize accordingly.
+It will generate a yaml installation file at `config/install.yaml`. It should be customize accordingly.
+
+
+## Tenant Control Planes
+
+### Add-ons
+
+Kamaji provides optional installations into the deployed tenant control plane through add-ons. Is it possible to enable/disable them through the `tcp` definition.
+
+### Core DNS
+
+```yaml
+addons:
+    coreDNS: {}
+```
+
+### Kube-Proxy
+
+```yaml
+addons:
+    kubeProxy: {}
+```
+
+### Konnectivity
+
+```yaml
+addons:
+    konnectivity:
+        proxyPort: 31132 # mandatory
+        proxyHost: "172.18.0.2"  
+        allowAddressAsExternalIP: false
+        serviceType: NodePort # mandatory
+        version: v0.0.31
+        serverImage: us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-server
+        agentImage: us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-agent

e2e/suite_test.go

@@ -1,16 +1,16 @@
 // Copyright 2022 Clastix Labs
 // SPDX-License-Identifier: Apache-2.0
 
-package controllers
+package e2e
 
 import (
-	"path/filepath"
 	"testing"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 	"k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/rest"
+	"k8s.io/utils/pointer"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/envtest"
 	"sigs.k8s.io/controller-runtime/pkg/envtest/printer"
@@ -23,9 +23,11 @@ import (
 // These tests use Ginkgo (BDD-style Go testing framework). Refer to
 // http://onsi.github.io/ginkgo/ to learn more about Ginkgo.
 
-var cfg *rest.Config // nolint
-var k8sClient client.Client
-var testEnv *envtest.Environment
+var (
+	cfg       *rest.Config
+	k8sClient client.Client
+	testEnv   *envtest.Environment
+)
 
 func TestAPIs(t *testing.T) {
 	RegisterFailHandler(Fail)
@@ -40,11 +42,12 @@ var _ = BeforeSuite(func() {
 
 	By("bootstrapping test environment")
 	testEnv = &envtest.Environment{
-		CRDDirectoryPaths:     []string{filepath.Join("..", "config", "crd", "bases")},
-		ErrorIfCRDPathMissing: true,
+		UseExistingCluster: pointer.Bool(true),
 	}
 
-	cfg, err := testEnv.Start()
+	var err error
+
+	cfg, err = testEnv.Start()
 	Expect(err).NotTo(HaveOccurred())
 	Expect(cfg).NotTo(BeNil())
 

e2e/tenant_control_plane_ready_test.go

@@ -0,0 +1,86 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package e2e
+
+import (
+	"context"
+	"time"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+)
+
+var _ = Describe("Deploy a TenantControlPlane resource", func() {
+	// Fill TenantControlPlane object
+	tcp := kamajiv1alpha1.TenantControlPlane{
+		TypeMeta: metav1.TypeMeta{},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "tcp-clusterip",
+			Namespace: "default",
+		},
+		Spec: kamajiv1alpha1.TenantControlPlaneSpec{
+			ControlPlane: kamajiv1alpha1.ControlPlane{
+				Deployment: kamajiv1alpha1.DeploymentSpec{
+					Replicas: 1,
+				},
+				Ingress: kamajiv1alpha1.IngressSpec{
+					Enabled: false,
+				},
+				Service: kamajiv1alpha1.ServiceSpec{
+					ServiceType: "ClusterIP",
+				},
+			},
+			NetworkProfile: kamajiv1alpha1.NetworkProfileSpec{
+				Address: "172.18.0.2",
+			},
+			Kubernetes: kamajiv1alpha1.KubernetesSpec{
+				Version: "v1.23.6",
+				Kubelet: kamajiv1alpha1.KubeletSpec{
+					CGroupFS: "cgroupfs",
+				},
+				AdmissionControllers: kamajiv1alpha1.AdmissionControllers{
+					"LimitRanger",
+					"ResourceQuota",
+				},
+			},
+			Addons: kamajiv1alpha1.AddonsSpec{},
+		},
+	}
+
+	// Create a TenantControlPlane resource into the cluster
+	JustBeforeEach(func() {
+		Expect(k8sClient.Create(context.Background(), &tcp)).NotTo(HaveOccurred())
+	})
+
+	// Delete the TenantControlPlane resource after test is finished
+	JustAfterEach(func() {
+		PrintTenantControlPlaneInfo(&tcp)
+		PrintKamajiLogs()
+		Expect(k8sClient.Delete(context.Background(), &tcp)).Should(Succeed())
+	})
+
+	// Check if TenantControlPlane resource has been created
+	It("Should be Ready", func() {
+		Eventually(func() kamajiv1alpha1.KubernetesVersionStatus {
+			err := k8sClient.Get(context.Background(), types.NamespacedName{
+				Name:      tcp.GetName(),
+				Namespace: tcp.GetNamespace(),
+			}, &tcp)
+			if err != nil {
+				return ""
+			}
+
+			// Check if Status field has been created on TenantControlPlane struct
+			if tcp.Status.Kubernetes.Version.Status == nil {
+				return ""
+			}
+
+			return *tcp.Status.Kubernetes.Version.Status
+		}, 5*time.Minute, time.Second).Should(Equal(kamajiv1alpha1.VersionReady))
+	})
+})

e2e/utils_test.go

@@ -0,0 +1,110 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package e2e
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io/ioutil"
+	"os/exec"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/kubernetes"
+
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+)
+
+func GetKindIPAddress() string {
+	ep := &corev1.Endpoints{}
+	Expect(k8sClient.Get(context.Background(), types.NamespacedName{Name: "kubernetes", Namespace: "default"}, ep)).ToNot(HaveOccurred())
+
+	return ep.Subsets[0].Addresses[0].IP
+}
+
+func PrintTenantControlPlaneInfo(tcp *kamajiv1alpha1.TenantControlPlane) {
+	kubectlExec := func(args ...string) {
+		cmd := exec.Command("kubectl")
+
+		var out bytes.Buffer
+		cmd.Stdout = &out
+		cmd.Args = args
+
+		Expect(cmd.Run()).ToNot(HaveOccurred())
+
+		for {
+			line, err := out.ReadString('\n')
+			if err != nil {
+				return
+			}
+
+			_, _ = fmt.Fprint(GinkgoWriter, ">>> ", line)
+		}
+	}
+
+	if CurrentGinkgoTestDescription().Failed {
+		_, _ = fmt.Fprintln(GinkgoWriter, "DEBUG: Tenant Control Plane definition")
+		kubectlExec(
+			fmt.Sprintf("--namespace=%s", tcp.GetNamespace()),
+			"get",
+			"tcp",
+			tcp.GetName(),
+		)
+		_, _ = fmt.Fprintln(GinkgoWriter, "DEBUG: Tenant Control Plane resources")
+		kubectlExec(
+			fmt.Sprintf("--namespace=%s", tcp.GetNamespace()),
+			"get",
+			"svc,deployment,pods,ep,configmap,secrets",
+		)
+		_, _ = fmt.Fprintln(GinkgoWriter, "DEBUG: Tenant Control Plane pods")
+		kubectlExec(
+			fmt.Sprintf("--namespace=%s", tcp.GetNamespace()),
+			"describe",
+			"pods",
+		)
+	}
+}
+
+func PrintKamajiLogs() {
+	if CurrentGinkgoTestDescription().Failed {
+		clientset, err := kubernetes.NewForConfig(cfg)
+		Expect(err).ToNot(HaveOccurred())
+
+		list, err := clientset.CoreV1().Pods("kamaji-system").List(context.Background(), metav1.ListOptions{
+			LabelSelector: "app.kubernetes.io/component=controller-manager",
+		})
+		Expect(err).ToNot(HaveOccurred())
+		Expect(list.Items).To(HaveLen(1))
+
+		request := clientset.CoreV1().Pods("kamaji-system").GetLogs(list.Items[0].GetName(), &corev1.PodLogOptions{
+			Container: "manager",
+			SinceSeconds: func() *int64 {
+				seconds := int64(CurrentGinkgoTestDescription().Duration.Seconds())
+
+				return &seconds
+			}(),
+			Timestamps: true,
+		})
+
+		podLogs, err := request.Stream(context.Background())
+		Expect(err).ToNot(HaveOccurred())
+
+		defer podLogs.Close()
+
+		podBytes, err := ioutil.ReadAll(podLogs)
+		Expect(err).ToNot(HaveOccurred())
+
+		_, _ = fmt.Fprintln(GinkgoWriter, "DEBUG: retrieving Kamaji Pod logs")
+
+		for _, line := range bytes.Split(podBytes, []byte("\n")) {
+			_, _ = fmt.Fprintln(GinkgoWriter, ">>> ", string(line))
+		}
+
+		_, _ = fmt.Fprintln(GinkgoWriter, "DEBUG: end of Kamaji Pod logs")
+	}
+}

e2e/worker_kubeadm_join_test.go

@@ -0,0 +1,170 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package e2e
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"strings"
+	"time"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/testcontainers/testcontainers-go"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/kubernetes/cmd/kubeadm/app/cmd"
+	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/util"
+
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+)
+
+var _ = Describe("starting a kind worker with kubeadm", func() {
+	ctx := context.Background()
+
+	var tcp kamajiv1alpha1.TenantControlPlane
+
+	var workerContainer testcontainers.Container
+
+	var kubeconfigFile *os.File
+
+	JustBeforeEach(func() {
+		tcp = kamajiv1alpha1.TenantControlPlane{
+			TypeMeta: metav1.TypeMeta{},
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "worker-nodes-join",
+				Namespace: "default",
+			},
+			Spec: kamajiv1alpha1.TenantControlPlaneSpec{
+				ControlPlane: kamajiv1alpha1.ControlPlane{
+					Deployment: kamajiv1alpha1.DeploymentSpec{
+						Replicas: 1,
+					},
+					Ingress: kamajiv1alpha1.IngressSpec{
+						Enabled: false,
+					},
+					Service: kamajiv1alpha1.ServiceSpec{
+						ServiceType: "NodePort",
+					},
+				},
+				NetworkProfile: kamajiv1alpha1.NetworkProfileSpec{
+					Address: GetKindIPAddress(),
+					Port:    31443,
+				},
+				Kubernetes: kamajiv1alpha1.KubernetesSpec{
+					Version: "v1.23.6",
+					Kubelet: kamajiv1alpha1.KubeletSpec{
+						CGroupFS: "cgroupfs",
+					},
+					AdmissionControllers: kamajiv1alpha1.AdmissionControllers{
+						"LimitRanger",
+						"ResourceQuota",
+					},
+				},
+				Addons: kamajiv1alpha1.AddonsSpec{},
+			},
+		}
+		Expect(k8sClient.Create(ctx, &tcp)).NotTo(HaveOccurred())
+
+		var err error
+
+		workerContainer, err = testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
+			ContainerRequest: testcontainers.ContainerRequest{
+				Name:       fmt.Sprintf("%s-worker-node", tcp.GetName()),
+				Image:      fmt.Sprintf("kindest/node:%s", tcp.Spec.Kubernetes.Version),
+				Mounts:     testcontainers.ContainerMounts{testcontainers.BindMount("/lib/modules", "/lib/modules")},
+				Networks:   []string{"kind"},
+				Privileged: true,
+			},
+			Started: true,
+		})
+		Expect(err).ToNot(HaveOccurred())
+
+		kubeconfigFile, err = ioutil.TempFile("", "kamaji")
+		Expect(err).ToNot(HaveOccurred())
+	})
+
+	JustAfterEach(func() {
+		PrintTenantControlPlaneInfo(&tcp)
+		PrintKamajiLogs()
+		Expect(workerContainer.Terminate(ctx)).ToNot(HaveOccurred())
+		Expect(k8sClient.Delete(ctx, &tcp)).Should(Succeed())
+		Expect(os.Remove(kubeconfigFile.Name())).ToNot(HaveOccurred())
+	})
+
+	It("should join the Tenant Control Plane cluster", func() {
+		By("waiting for the Tenant Control Plane being ready", func() {
+			Eventually(func() kamajiv1alpha1.KubernetesVersionStatus {
+				err := k8sClient.Get(ctx, types.NamespacedName{Name: tcp.GetName(), Namespace: tcp.GetNamespace()}, &tcp)
+				if err != nil {
+					return ""
+				}
+
+				if tcp.Status.Kubernetes.Version.Status == nil {
+					return ""
+				}
+
+				return *tcp.Status.Kubernetes.Version.Status
+			}, 5*time.Minute, time.Second).Should(Equal(kamajiv1alpha1.VersionReady))
+		})
+
+		By("downloading Tenant Control Plane kubeconfig", func() {
+			secret := &corev1.Secret{}
+
+			Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: tcp.GetNamespace(), Name: tcp.Status.KubeConfig.Admin.SecretName}, secret)).NotTo(HaveOccurred())
+
+			_, err := kubeconfigFile.Write(secret.Data["admin.conf"])
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		var joinCommandBuffer *bytes.Buffer
+
+		By("generating kubeadm join command", func() {
+			joinCommandBuffer = bytes.NewBuffer([]byte(""))
+
+			config, err := clientcmd.BuildConfigFromFlags("", kubeconfigFile.Name())
+			Expect(err).ToNot(HaveOccurred())
+
+			clientset, err := kubernetes.NewForConfig(config)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(cmd.RunCreateToken(joinCommandBuffer, clientset, "", util.DefaultInitConfiguration(), true, "", kubeconfigFile.Name())).ToNot(HaveOccurred())
+		})
+
+		By("executing the command in the worker node", func() {
+			cmds := append(strings.Split(strings.TrimSpace(joinCommandBuffer.String()), " "), "--ignore-preflight-errors", "SystemVerification")
+
+			exitCode, err := workerContainer.Exec(ctx, cmds)
+			Expect(exitCode).To(Equal(0))
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		By("waiting for nodes", func() {
+			config, err := clientcmd.BuildConfigFromFlags("", kubeconfigFile.Name())
+			Expect(err).ToNot(HaveOccurred())
+
+			clientset, err := kubernetes.NewForConfig(config)
+			Expect(err).ToNot(HaveOccurred())
+
+			Eventually(func() string {
+				nodes, err := clientset.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
+				if err != nil {
+					return ""
+				}
+
+				if len(nodes.Items) == 0 {
+					return ""
+				}
+
+				return nodes.Items[0].GetName()
+			}, time.Minute, time.Second).Should(Equal(workerContainer.GetContainerID()[:12]))
+		})
+	})
+})

e2e/worker_tcp_change_port_test.go

@@ -0,0 +1,165 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package e2e
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"time"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/version"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
+
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+)
+
+var _ = Describe("validating kubeconfig", func() {
+	ctx := context.Background()
+
+	var tcp *kamajiv1alpha1.TenantControlPlane
+
+	var kubeconfigFile *os.File
+
+	JustBeforeEach(func() {
+		tcp = &kamajiv1alpha1.TenantControlPlane{
+			TypeMeta: metav1.TypeMeta{},
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "kubeconfig",
+				Namespace: "default",
+			},
+			Spec: kamajiv1alpha1.TenantControlPlaneSpec{
+				ControlPlane: kamajiv1alpha1.ControlPlane{
+					Deployment: kamajiv1alpha1.DeploymentSpec{
+						Replicas: 1,
+					},
+					Ingress: kamajiv1alpha1.IngressSpec{
+						Enabled: false,
+					},
+					Service: kamajiv1alpha1.ServiceSpec{
+						ServiceType: "NodePort",
+					},
+				},
+				NetworkProfile: kamajiv1alpha1.NetworkProfileSpec{
+					Address: GetKindIPAddress(),
+					Port:    30001,
+				},
+				Kubernetes: kamajiv1alpha1.KubernetesSpec{
+					Version: "v1.23.6",
+					Kubelet: kamajiv1alpha1.KubeletSpec{
+						CGroupFS: "cgroupfs",
+					},
+					AdmissionControllers: kamajiv1alpha1.AdmissionControllers{
+						"LimitRanger",
+						"ResourceQuota",
+					},
+				},
+				Addons: kamajiv1alpha1.AddonsSpec{},
+			},
+		}
+		Expect(k8sClient.Create(ctx, tcp)).NotTo(HaveOccurred())
+
+		var err error
+
+		kubeconfigFile, err = ioutil.TempFile("", "kamaji")
+		Expect(err).ToNot(HaveOccurred())
+	})
+
+	JustAfterEach(func() {
+		PrintKamajiLogs()
+		PrintTenantControlPlaneInfo(tcp)
+		Expect(k8sClient.Delete(ctx, tcp)).Should(Succeed())
+		Expect(os.Remove(kubeconfigFile.Name())).ToNot(HaveOccurred())
+	})
+
+	It("return kubernetes version", func() {
+		for _, port := range []int32{30002, 30003, 30004} {
+			Eventually(func() string {
+				By(fmt.Sprintf("ensuring TCP port is set to %d", port), func() {
+					Eventually(func() (err error) {
+						if err = k8sClient.Get(ctx, types.NamespacedName{Namespace: tcp.GetNamespace(), Name: tcp.GetName()}, tcp); err != nil {
+							_, _ = fmt.Fprintln(GinkgoWriter, "DEBUG: cannot retrieve TCP:", err.Error())
+
+							return err
+						}
+
+						tcp.Spec.NetworkProfile.Port = port
+
+						return k8sClient.Update(ctx, tcp)
+					}, time.Minute, 5*time.Second).ShouldNot(HaveOccurred())
+				})
+
+				By("ensuring port change is defined in the TCP status", func() {
+					Eventually(func() int32 {
+						if err := k8sClient.Get(ctx, types.NamespacedName{Namespace: tcp.GetNamespace(), Name: tcp.GetName()}, tcp); err != nil {
+							_, _ = fmt.Fprintln(GinkgoWriter, "DEBUG: cannot retrieve TCP:", err.Error())
+
+							return 0
+						}
+
+						return tcp.Status.Kubernetes.Service.Port
+					}, time.Minute, 5*time.Second).Should(Equal(port))
+				})
+
+				By("ensuring downloading the updated kubeconfig", func() {
+					Eventually(func() (err error) {
+						if err := k8sClient.Get(ctx, types.NamespacedName{Namespace: tcp.GetNamespace(), Name: tcp.GetName()}, tcp); err != nil {
+							_, _ = fmt.Fprintln(GinkgoWriter, "DEBUG: cannot retrieve TCP:", err.Error())
+
+							return err
+						}
+
+						secret := &corev1.Secret{}
+
+						if err = k8sClient.Get(ctx, types.NamespacedName{Namespace: tcp.GetNamespace(), Name: tcp.Status.KubeConfig.Admin.SecretName}, secret); err != nil {
+							_, _ = fmt.Fprintln(GinkgoWriter, "DEBUG: cannot retrieve kubeconfig secret name:", err.Error())
+
+							return err
+						}
+
+						_, err = kubeconfigFile.Write(secret.Data["admin.conf"])
+
+						return err
+					}, time.Minute, 5*time.Second).ShouldNot(HaveOccurred())
+				})
+
+				var version version.Info
+
+				By("retrieving TCP version using the kubeconfig", func() {
+					config, err := clientcmd.BuildConfigFromFlags("", kubeconfigFile.Name())
+					if err != nil {
+						_, _ = fmt.Fprintln(GinkgoWriter, "DEBUG: cannot generate REST configuration:", err.Error())
+
+						return
+					}
+
+					clientset, err := kubernetes.NewForConfig(config)
+					if err != nil {
+						_, _ = fmt.Fprintln(GinkgoWriter, "DEBUG: cannot generate clientset:", err.Error())
+
+						return
+					}
+
+					serverVersion, err := clientset.ServerVersion()
+					if err != nil {
+						_, _ = fmt.Fprintln(GinkgoWriter, "DEBUG: cannot retrieve server version:", err.Error())
+
+						return
+					}
+
+					version = *serverVersion
+				})
+
+				return version.GitVersion
+			}, 3*time.Minute, 5*time.Second).Should(Equal(tcp.Spec.Kubernetes.Version))
+		}
+	})
+})

go.mod

@@ -1,20 +1,23 @@
 module github.com/clastix/kamaji
 
-go 1.17
+go 1.18
 
 require (
 	github.com/go-logr/logr v1.2.0
-	github.com/google/uuid v1.1.2
+	github.com/go-sql-driver/mysql v1.6.0
+	github.com/google/uuid v1.3.0
 	github.com/onsi/ginkgo v1.16.5
 	github.com/onsi/gomega v1.17.0
 	github.com/pkg/errors v0.9.1
 	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.10.1
+	github.com/testcontainers/testcontainers-go v0.13.0
 	go.etcd.io/etcd/api/v3 v3.5.1
 	go.etcd.io/etcd/client/v3 v3.5.1
 	google.golang.org/grpc v1.43.0
 	k8s.io/api v0.23.5
 	k8s.io/apimachinery v0.23.5
+	k8s.io/apiserver v0.23.5
 	k8s.io/client-go v0.23.5
 	k8s.io/cluster-bootstrap v0.0.0
 	k8s.io/component-base v0.23.5
@@ -28,6 +31,7 @@ require (
 require (
 	cloud.google.com/go v0.99.0 // indirect
 	cloud.google.com/go/storage v1.18.2 // indirect
+	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
 	github.com/Azure/go-autorest v14.2.0+incompatible // indirect
 	github.com/Azure/go-autorest/autorest v0.11.18 // indirect
 	github.com/Azure/go-autorest/autorest/adal v0.9.13 // indirect
@@ -35,53 +39,88 @@ require (
 	github.com/Azure/go-autorest/logger v0.2.1 // indirect
 	github.com/Azure/go-autorest/tracing v0.6.0 // indirect
 	github.com/Microsoft/go-winio v0.4.17 // indirect
+	github.com/Microsoft/hcsshim v0.8.23 // indirect
+	github.com/PuerkitoBio/purell v1.1.1 // indirect
+	github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/blang/semver v3.5.1+incompatible // indirect
+	github.com/cenkalti/backoff/v4 v4.1.2 // indirect
 	github.com/census-instrumentation/opencensus-proto v0.3.0 // indirect
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
 	github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4 // indirect
 	github.com/cncf/xds/go v0.0.0-20211216145620-d92e9ce0af51 // indirect
+	github.com/containerd/cgroups v1.0.1 // indirect
+	github.com/containerd/containerd v1.5.9 // indirect
 	github.com/coredns/caddy v1.1.0 // indirect
 	github.com/coredns/corefile-migration v1.0.14 // indirect
 	github.com/coreos/go-semver v0.3.0 // indirect
 	github.com/coreos/go-systemd/v22 v22.3.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/docker/distribution v2.7.1+incompatible // indirect
+	github.com/docker/docker v20.10.11+incompatible // indirect
+	github.com/docker/go-connections v0.4.0 // indirect
+	github.com/docker/go-units v0.4.0 // indirect
 	github.com/envoyproxy/go-control-plane v0.10.1 // indirect
 	github.com/envoyproxy/protoc-gen-validate v0.6.2 // indirect
 	github.com/evanphx/json-patch v4.12.0+incompatible // indirect
 	github.com/form3tech-oss/jwt-go v3.2.3+incompatible // indirect
 	github.com/fsnotify/fsnotify v1.5.1 // indirect
+	github.com/go-errors/errors v1.0.1 // indirect
 	github.com/go-logr/zapr v1.2.0 // indirect
+	github.com/go-openapi/jsonpointer v0.19.5 // indirect
+	github.com/go-openapi/jsonreference v0.19.5 // indirect
+	github.com/go-openapi/swag v0.19.14 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/google/btree v1.0.1 // indirect
 	github.com/google/go-cmp v0.5.6 // indirect
 	github.com/google/gofuzz v1.1.0 // indirect
+	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
 	github.com/googleapis/gax-go/v2 v2.1.1 // indirect
 	github.com/googleapis/gnostic v0.5.5 // indirect
 	github.com/googleapis/google-cloud-go-testing v0.0.0-20210719221736-1c9a4c676720 // indirect
+	github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/imdario/mergo v0.3.12 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
+	github.com/lithammer/dedent v1.1.0 // indirect
 	github.com/magiconair/properties v1.8.5 // indirect
+	github.com/mailru/easyjson v0.7.6 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
 	github.com/mitchellh/mapstructure v1.4.3 // indirect
+	github.com/moby/sys/mount v0.2.0 // indirect
+	github.com/moby/sys/mountinfo v0.5.0 // indirect
+	github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
+	github.com/morikuni/aec v1.0.0 // indirect
 	github.com/nxadm/tail v1.4.8 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/image-spec v1.0.2 // indirect
+	github.com/opencontainers/runc v1.0.2 // indirect
 	github.com/pelletier/go-toml v1.9.4 // indirect
+	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_golang v1.12.1 // indirect
 	github.com/prometheus/client_model v0.2.0 // indirect
 	github.com/prometheus/common v0.32.1 // indirect
 	github.com/prometheus/procfs v0.7.3 // indirect
+	github.com/sirupsen/logrus v1.8.1 // indirect
 	github.com/spf13/afero v1.7.0 // indirect
 	github.com/spf13/cast v1.4.1 // indirect
 	github.com/spf13/cobra v1.4.0 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
+	github.com/stretchr/testify v1.7.0 // indirect
 	github.com/subosito/gotenv v1.2.0 // indirect
+	github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca // indirect
 	go.etcd.io/etcd/client/pkg/v3 v3.5.1 // indirect
 	go.opencensus.io v0.23.0 // indirect
+	go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect
 	go.uber.org/atomic v1.7.0 // indirect
 	go.uber.org/multierr v1.6.0 // indirect
 	go.uber.org/zap v1.19.1 // indirect
@@ -100,15 +139,18 @@ require (
 	google.golang.org/protobuf v1.27.1 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/ini.v1 v1.66.2 // indirect
+	gopkg.in/square/go-jose.v2 v2.5.1 // indirect
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
 	k8s.io/apiextensions-apiserver v0.23.5 // indirect
-	k8s.io/apiserver v0.23.5 // indirect
+	k8s.io/cli-runtime v0.23.5 // indirect
 	k8s.io/klog/v2 v2.60.1 // indirect
 	k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 // indirect
 	k8s.io/system-validators v1.6.0 // indirect
 	sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 // indirect
+	sigs.k8s.io/kustomize/api v0.10.1 // indirect
+	sigs.k8s.io/kustomize/kyaml v0.13.0 // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect
 	sigs.k8s.io/yaml v1.3.0 // indirect
 )

helm/kamaji/Chart.yaml

@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.0
+version: 0.1.1
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

helm/kamaji/README.md

@@ -1,36 +1,24 @@
 # kamaji
 
-![Version: 0.1.0](https://img.shields.io/badge/Version-0.1.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.1.0](https://img.shields.io/badge/AppVersion-0.1.0-informational?style=flat-square)
+![Version: 0.1.1](https://img.shields.io/badge/Version-0.1.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.1.0](https://img.shields.io/badge/AppVersion-0.1.0-informational?style=flat-square)
 
 A Kubernetes distribution aimed to build and operate a Managed Kubernetes service with a fraction of operational burde.
 
 **Homepage:** <https://github.com/clastix/kamaji-internal/tree/master/helm/kamaji>
 
-## Installing the Chart
-
-To install the chart with the release name `kamaji`:
-
 ### Pre-requisites
 
-1. Deploy a [multi-tenant Etcd cluster](https://github.com/clastix/kamaji-internal/blob/master/deploy/getting-started-with-kamaji.md#setup-internal-multi-tenant-etcd)
-2. Create the `Secret` containing the Etcd CA cert keypair:
+Kamaji requires a [multi-tenant etcd cluster](https://github.com/clastix/kamaji-internal/blob/master/deploy/getting-started-with-kamaji.md#setup-internal-multi-tenant-etcd) cluster.
+The installation and provisioning processes are already put in place by the Helm Chart starting from v0.1.1 in order to streamline the local test.
 
-```
-kubectl -n kamaji-system create secret generic etcd-certs \
-  --from-file=/path/to/etcd/ca.crt \
-  --from-file=/path/to/etcd/ca.key
-```
+> For production use an externally managed etcd is highly recommended, the etcd addon offered by this chart is not considered production-grade.
 
-3. Create a `Secret` containing the Etcd root user client cert keypair:
-
-```
-kubectl -n kamaji-system create secret tls root-client-certs \
-  --cert=/path/to/etcd/root.pem \
-  --key=/path/to/etcd/root-key.pem
-```
+If you'd like to use an externally managed etcd instance, you can specify the overrides and by setting the value `etcd.deploy=false`.
 
 ### Install Kamaji
 
+To install the chart with the release name `kamaji`:
+
 ```console
 helm upgrade --install --namespace kamaji-system --create-namespace kamaji .
 ```
@@ -57,26 +45,22 @@ Kubernetes: `>=1.18`
 |-----|------|---------|-------------|
 | affinity | object | `{}` | Kubernetes affinity rules to apply to Kamaji controller pods |
 | configPath | string | `"./kamaji.yaml"` | Configuration file path alternative. (default "./kamaji.yaml") |
-| etcd.caSecret.name | string | `"etcd-certs"` | Name of the secret which contains CA's certificate and private key. (default: "etcd-certs") |
-| etcd.caSecret.namespace | string | `"kamaji-system"` | Namespace of the secret which contains CA's certificate and private key. (default: "kamaji") |
-| etcd.clientSecret.name | string | `"root-client-certs"` | Name of the secret which contains ETCD client certificates. (default: "root-client-certs") |
-| etcd.clientSecret.namespace | string | `"kamaji-system"` | Name of the namespace where the secret which contains ETCD client certificates is. (default: "kamaji") |
 | etcd.compactionInterval | int | `0` | ETCD Compaction interval (e.g. "5m0s"). (default: "0" (disabled)) |
-| etcd.endpoints | string | `"etcd-0.etcd.kamaji-system.svc.cluster.local:2379,etcd-1.etcd.kamaji-system.svc.cluster.local:2379,etcd-2.etcd.kamaji-system.svc.cluster.local:2379"` | (string) Comma-separated list of the endpoints of the etcd cluster's members. |
+| etcd.deploy | bool | `true` | Install an etcd 3.5 with enabled multi-tenancy along with Kamaji |
+| etcd.overrides.caSecret.name | string | `"etcd-certs"` | Name of the secret which contains CA's certificate and private key. (default: "etcd-certs") |
+| etcd.overrides.caSecret.namespace | string | `"kamaji-system"` | Namespace of the secret which contains CA's certificate and private key. (default: "kamaji-system") |
+| etcd.overrides.clientSecret.name | string | `"root-client-certs"` | Name of the secret which contains ETCD client certificates. (default: "root-client-certs") |
+| etcd.overrides.clientSecret.namespace | string | `"kamaji-system"` | Name of the namespace where the secret which contains ETCD client certificates is. (default: "kamaji-system") |
+| etcd.overrides.endpoints | string | `"https://etcd-0.etcd.kamaji-system.svc.cluster.local:2379,https://etcd-1.etcd.kamaji-system.svc.cluster.local:2379,https://etcd-2.etcd.kamaji-system.svc.cluster.local:2379"` | (string) Comma-separated list of the endpoints of the etcd cluster's members. |
+| etcd.serviceAccount.create | bool | `true` | Create a ServiceAccount, required to install and provision the etcd backing storage (default: true) |
+| etcd.serviceAccount.name | string | `""` | Define the ServiceAccount name to use during the setup and provision of the etcd backing storage (default: "") |
 | extraArgs | list | `[]` | A list of extra arguments to add to the kamaji controller default ones |
 | fullnameOverride | string | `""` |  |
 | healthProbeBindAddress | string | `":8081"` | The address the probe endpoint binds to. (default ":8081") |
-| image.pullPolicy | string | `"IfNotPresent"` |  |
-| image.repository | string | `"quay.io/clastix/kamaji"` | The container image of the Kamaji controller. |
+| image.pullPolicy | string | `"Always"` |  |
+| image.repository | string | `"clastix/kamaji"` | The container image of the Kamaji controller. |
 | image.tag | string | `"latest"` |  |
 | imagePullSecrets | list | `[]` |  |
-| ingress.annotations | object | `{}` |  |
-| ingress.className | string | `""` | Name of the ingress class to route through this controller. |
-| ingress.enabled | bool | `false` | Whether to expose the Kamaji controller through an Ingress. |
-| ingress.hosts[0].host | string | `"chart-example.local"` |  |
-| ingress.hosts[0].paths[0].path | string | `"/"` |  |
-| ingress.hosts[0].paths[0].pathType | string | `"ImplementationSpecific"` |  |
-| ingress.tls | list | `[]` |  |
 | livenessProbe | object | `{"httpGet":{"path":"/healthz","port":"healthcheck"},"initialDelaySeconds":15,"periodSeconds":20}` | The livenessProbe for the controller container |
 | loggingDevel.enable | bool | `false` | (string) Development Mode defaults(encoder=consoleEncoder,logLevel=Debug,stackTraceLevel=Warn). Production Mode defaults(encoder=jsonEncoder,logLevel=Info,stackTraceLevel=Error) (default false) |
 | metricsBindAddress | string | `":8080"` | (string) The address the metric endpoint binds to. (default ":8080") |

helm/kamaji/README.md.gotmpl

@@ -7,31 +7,19 @@
 
 {{ template "chart.homepageLine" . }}
 
-## Installing the Chart
-
-To install the chart with the release name `kamaji`:
-
 ### Pre-requisites
 
-1. Deploy a [multi-tenant Etcd cluster](https://github.com/clastix/kamaji-internal/blob/master/deploy/getting-started-with-kamaji.md#setup-internal-multi-tenant-etcd)
-2. Create the `Secret` containing the Etcd CA cert keypair:
+Kamaji requires a [multi-tenant etcd cluster](https://github.com/clastix/kamaji-internal/blob/master/deploy/getting-started-with-kamaji.md#setup-internal-multi-tenant-etcd) cluster.
+The installation and provisioning processes are already put in place by the Helm Chart starting from v0.1.1 in order to streamline the local test.
 
-```
-kubectl -n kamaji-system create secret generic etcd-certs \
-  --from-file=/path/to/etcd/ca.crt \
-  --from-file=/path/to/etcd/ca.key
-```
+> For production use an externally managed etcd is highly recommended, the etcd addon offered by this chart is not considered production-grade.
 
-3. Create a `Secret` containing the Etcd root user client cert keypair:
-
-```
-kubectl -n kamaji-system create secret tls root-client-certs \
-  --cert=/path/to/etcd/root.pem \
-  --key=/path/to/etcd/root-key.pem
-```
+If you'd like to use an externally managed etcd instance, you can specify the overrides and by setting the value `etcd.deploy=false`.
 
 ### Install Kamaji
 
+To install the chart with the release name `kamaji`:
+
 ```console
 helm upgrade --install --namespace kamaji-system --create-namespace kamaji .
 ```

helm/kamaji/templates/NOTES.txt

@@ -1,22 +1,5 @@
-1. Get the application URL by running these commands:
-{{- if .Values.ingress.enabled }}
-{{- range $host := .Values.ingress.hosts }}
-  {{- range .paths }}
-  http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }}
-  {{- end }}
-{{- end }}
-{{- else if contains "NodePort" .Values.service.type }}
-  export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "kamaji.fullname" . }})
-  export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
-  echo http://$NODE_IP:$NODE_PORT
-{{- else if contains "LoadBalancer" .Values.service.type }}
-     NOTE: It may take a few minutes for the LoadBalancer IP to be available.
-           You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "kamaji.fullname" . }}'
-  export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "kamaji.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}")
-  echo http://$SERVICE_IP:{{ .Values.service.port }}
-{{- else if contains "ClusterIP" .Values.service.type }}
-  export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "kamaji.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
-  export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}")
-  echo "Visit http://127.0.0.1:8080 to use your application"
-  kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT
-{{- end }}
+List the available CRDs installed by Kamaji:
+  kubectl get customresourcedefinitions.apiextensions.k8s.io
+
+List all the Tenant Control Plane resources deployed in your cluster:
+  kubectl get tenantcontrolplanes.kamaji.clastix.io --all-namespaces

helm/kamaji/templates/_helpers_etcd.tpl

@@ -0,0 +1,120 @@
+{{/*
+Create a default fully qualified etcd name.
+*/}}
+{{- define "etcd.fullname" -}}
+{{- printf "etcd" }}
+{{- end }}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "etcd.serviceAccountName" -}}
+{{- if .Values.etcd.serviceAccount.create }}
+{{- default (include "etcd.fullname" .) .Values.etcd.serviceAccount.name }}
+{{- else }}
+{{- default "default" .Values.etcd.serviceAccount.name }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create the name of the Service to use
+*/}}
+{{- define "etcd.serviceName" -}}
+{{- printf "%s" (include "etcd.fullname" .) | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "etcd.labels" -}}
+app.kubernetes.io/name: {{ include "kamaji.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+app.kubernetes.io/components: etcd
+{{- end }}
+
+{{/*
+Selector labels.
+*/}}
+{{- define "etcd.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "kamaji.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+app.kubernetes.io/component: etcd
+{{- end }}
+
+{{/*
+Name of the etcd CA secret.
+*/}}
+{{- define "etcd.caSecretName" }}
+{{- if .Values.etcd.deploy }}
+{{- printf "%s-%s" (include "etcd.fullname" .) "certs" | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- required "A valid .Values.etcd.overrides.caSecret.name required!" .Values.etcd.overrides.caSecret.name }}
+{{- end }}
+{{- end }}
+
+{{/*
+Namespace of the etcd CA secret.
+*/}}
+{{- define "etcd.caSecretNamespace" }}
+{{- if .Values.etcd.deploy }}
+{{- .Release.Namespace }}
+{{- else }}
+{{- required "A valid .Values.etcd.overrides.caSecret.namespace required!" .Values.etcd.overrides.caSecret.namespace }}
+{{- end }}
+{{- end }}
+
+{{/*
+Name of the certificate signing requests for the certificates required by etcd.
+*/}}
+{{- define "etcd.csrConfigMapName" }}
+{{- printf "%s-csr" (include "etcd.fullname" .) }}
+{{- end }}
+
+{{/*
+Name of the etcd root-client secret.
+*/}}
+{{- define "etcd.clientSecretName" }}
+{{- if .Values.etcd.deploy }}
+{{- printf "root-client-certs" }}
+{{- else }}
+{{- required "A valid .Values.etcd.overrides.clientSecret.name required!" .Values.etcd.overrides.clientSecret.name }}
+{{- end }}
+{{- end }}
+
+{{/*
+Namespace of the etcd root-client secret.
+*/}}
+{{- define "etcd.clientSecretNamespace" }}
+{{- if .Values.etcd.deploy }}
+{{- .Release.Namespace }}
+{{- else }}
+{{- required "A valid .Values.etcd.overrides.clientSecret.namespace required!" .Values.etcd.overrides.clientSecret.namespace }}
+{{- end }}
+{{- end }}
+
+{{/*
+List the declared etcd endpoints, using the overrides in case of unmanaged etcd.
+*/}}
+{{- define "etcd.endpoints" }}
+{{- if .Values.etcd.deploy }}
+{{- range $count := until 3 -}}
+    {{- printf "https://%s-%d.%s.%s.svc.cluster.local:2379" "etcd" $count ( include "etcd.serviceName" . ) $.Release.Namespace -}}
+    {{- if lt $count  ( sub 3 1 ) -}}
+      {{- printf "," -}}
+    {{- end -}}
+{{- end }}
+{{- else }}
+{{- required "A valid .Values.etcd.overrides.endpoints required!" .Values.etcd.overrides.endpoints }}
+{{- end }}
+{{- end }}
+
+{{/*
+Retrieve the current Kubernetes version to launch a kubectl container with the minimum version skew possible.
+*/}}
+{{- define "etcd.jobsTagKubeVersion" -}}
+{{- if contains "-eks-" .Capabilities.KubeVersion.GitVersion }}
+{{- print "v" .Capabilities.KubeVersion.Major "." (.Capabilities.KubeVersion.Minor | replace "+" "") -}}
+{{- else }}
+{{- print "v" .Capabilities.KubeVersion.Major "." .Capabilities.KubeVersion.Minor -}}
+{{- end }}
+{{- end }}

helm/kamaji/templates/controller.yaml

@@ -40,12 +40,12 @@ spec:
           protocol: TCP
       - args:
         - --config-file={{ .Values.configPath }}
-        - --etcd-ca-secret-name={{ .Values.etcd.caSecret.name }}
-        - --etcd-ca-secret-namespace={{ .Values.etcd.caSecret.namespace }}
-        - --etcd-client-secret-name={{ .Values.etcd.clientSecret.name }}
-        - --etcd-client-secret-namespace={{ .Values.etcd.clientSecret.namespace }}
+        - --etcd-ca-secret-name={{ include "etcd.caSecretName" . }}
+        - --etcd-ca-secret-namespace={{ include "etcd.caSecretNamespace" . }}
+        - --etcd-client-secret-name={{ include "etcd.clientSecretName" . }}
+        - --etcd-client-secret-namespace={{ include "etcd.clientSecretNamespace" . }}
         - --etcd-compaction-interval={{ .Values.etcd.compactionInterval }}
-        - --etcd-endpoints={{ .Values.etcd.endpoints }}
+        - --etcd-endpoints={{ include "etcd.endpoints" . }}
         - --health-probe-bind-address={{ .Values.healthProbeBindAddress }}
         - --leader-elect
         - --metrics-bind-address={{ .Values.metricsBindAddress }}

helm/kamaji/templates/etcd_cm.yaml

@@ -0,0 +1,94 @@
+{{- if .Values.etcd.deploy }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  labels:
+    {{- include "etcd.labels" . | nindent 4 }}
+  name: {{ include "etcd.csrConfigMapName" . }}
+  namespace: {{ .Release.Namespace }}
+data:
+  ca-csr.json: |-
+    {
+      "CN": "Clastix CA",
+      "key": {
+        "algo": "rsa",
+        "size": 2048
+      },
+      "names": [
+        {
+          "C": "IT",
+          "ST": "Italy",
+          "L": "Milan"
+        }
+      ]
+    }
+  config.json: |-
+    {
+      "signing": {
+        "default": {
+          "expiry": "8760h"
+        },
+        "profiles": {
+          "server-authentication": {
+            "usages": ["signing", "key encipherment", "server auth"],
+            "expiry": "8760h"
+          },
+          "client-authentication": {
+            "usages": ["signing", "key encipherment", "client auth"],
+            "expiry": "8760h"
+          },
+          "peer-authentication": {
+            "usages": ["signing", "key encipherment", "server auth", "client auth"],
+            "expiry": "8760h"
+          }
+        }
+      }
+    }
+  server-csr.json: |-
+    {
+      "CN": "etcd",
+      "key": {
+        "algo": "rsa",
+        "size": 2048
+      },
+      "hosts": [
+{{- range $count := until 3 -}}
+        {{ printf "\"etcd-%d.%s.%s.svc.cluster.local\"," $count (include "etcd.serviceName" .) $.Release.Namespace }}
+{{- end }}
+        "etcd-server.{{ .Release.Namespace }}.svc.cluster.local",
+        "etcd-server.{{ .Release.Namespace }}.svc",
+        "etcd-server",
+        "127.0.0.1"
+      ]
+    }
+  peer-csr.json: |-
+    {
+      "CN": "etcd",
+      "key": {
+        "algo": "rsa",
+        "size": 2048
+      },
+      "hosts": [
+{{- range $count := until 3 -}}
+        {{ printf "\"etcd-%d\"," $count }}
+        {{ printf "\"etcd-%d.%s\"," $count (include "etcd.serviceName" .) }}
+        {{ printf "\"etcd-%d.%s.%s.svc\"," $count (include "etcd.serviceName" .) $.Release.Namespace }}
+        {{ printf "\"etcd-%d.%s.%s.svc.cluster.local\"," $count (include "etcd.serviceName" .) $.Release.Namespace }}
+{{- end }}
+        "127.0.0.1"
+      ]
+    }
+  root-client-csr.json: |-
+    {
+      "CN": "root",
+      "key": {
+        "algo": "rsa",
+        "size": 2048
+      },
+      "names": [
+        {
+          "O": "system:masters"
+        }
+      ]
+    }
+{{- end }}

helm/kamaji/templates/etcd_job_postdelete.yaml

@@ -0,0 +1,31 @@
+{{- if .Values.etcd.deploy }}
+apiVersion: batch/v1
+kind: Job
+metadata:
+  labels:
+    {{- include "etcd.labels" . | nindent 4 }}
+  annotations:
+    "helm.sh/hook": pre-delete
+    "helm.sh/hook-weight": "-5"
+    "helm.sh/hook-delete-policy": hook-succeeded
+  name: "{{ .Release.Name }}-etcd-teardown"
+  namespace: {{ .Release.Namespace }}
+spec:
+  template:
+    metadata:
+      name: "{{ .Release.Name }}"
+    spec:
+      serviceAccountName: {{ include "etcd.serviceAccountName" . }}
+      restartPolicy: Never
+      containers:
+        - name: kubectl
+          image: {{ printf "clastix/kubectl:%s" (include "etcd.jobsTagKubeVersion" .) }}
+          command:
+            - kubectl
+            - --namespace={{ .Release.Namespace }}
+            - delete
+            - secret
+            - --ignore-not-found=true
+            - {{ include "etcd.caSecretName" . }}
+            - {{ include "etcd.clientSecretName" . }}
+{{- end }}

helm/kamaji/templates/etcd_job_postinstall.yaml

@@ -0,0 +1,91 @@
+{{- if .Values.etcd.deploy }}
+apiVersion: batch/v1
+kind: Job
+metadata:
+  labels:
+    {{- include "etcd.labels" . | nindent 4 }}
+  annotations:
+    "helm.sh/hook": post-install
+    "helm.sh/hook-weight": "-5"
+    "helm.sh/hook-delete-policy": hook-succeeded
+  name: "{{ .Release.Name }}-etcd-setup"
+  namespace: {{ .Release.Namespace }}
+spec:
+  template:
+    metadata:
+      name: "{{ .Release.Name }}"
+    spec:
+      serviceAccountName: {{ include "etcd.serviceAccountName" . }}
+      restartPolicy: Never
+      initContainers:
+        - name: cfssl
+          image: cfssl/cfssl:latest
+          command:
+            - bash
+            - -c
+            - |-
+              cfssl gencert -initca /csr/ca-csr.json | cfssljson -bare /certs/ca &&
+              mv /certs/ca.pem /certs/ca.crt && mv /certs/ca-key.pem /certs/ca.key &&
+              cfssl gencert -ca=/certs/ca.crt -ca-key=/certs/ca.key -config=/csr/config.json -profile=peer-authentication /csr/peer-csr.json | cfssljson -bare /certs/peer &&
+              cfssl gencert -ca=/certs/ca.crt -ca-key=/certs/ca.key -config=/csr/config.json -profile=peer-authentication /csr/server-csr.json | cfssljson -bare /certs/server &&
+              cfssl gencert -ca=/certs/ca.crt -ca-key=/certs/ca.key -config=/csr/config.json -profile=client-authentication /csr/root-client-csr.json | cfssljson -bare /certs/root-client
+          volumeMounts:
+            - mountPath: /certs
+              name: certs
+            - mountPath: /csr
+              name: csr
+        - name: kubectl
+          image: {{ printf "clastix/kubectl:%s" (include "etcd.jobsTagKubeVersion" .) }}
+          command:
+            - sh
+            - -c
+            - |-
+              kubectl --namespace={{ .Release.Namespace }} delete secret --ignore-not-found=true {{ include "etcd.caSecretName" . }} {{ include "etcd.clientSecretName" . }} &&
+              kubectl --namespace={{ .Release.Namespace }} create secret generic {{ include "etcd.caSecretName" . }} --from-file=/certs/ca.crt --from-file=/certs/ca.key --from-file=/certs/peer-key.pem --from-file=/certs/peer.pem --from-file=/certs/server-key.pem --from-file=/certs/server.pem &&
+              kubectl --namespace={{ .Release.Namespace }} create secret tls {{ include "etcd.clientSecretName" . }} --key=/certs/root-client-key.pem --cert=/certs/root-client.pem &&
+              kubectl --namespace={{ .Release.Namespace }} rollout status sts/etcd --timeout=120s
+          volumeMounts:
+            - mountPath: /certs
+              name: certs
+      containers:
+        - command:
+            - bash
+            - -c
+            - |-
+              etcdctl member list -w table &&
+              etcdctl user add --no-password=true root &&
+              etcdctl role add root &&
+              etcdctl user grant-role root root &&
+              etcdctl auth enable
+          env:
+            - name: ETCDCTL_ENDPOINTS
+              value: https://etcd-0.{{ include "etcd.serviceName" . }}.{{ .Release.Namespace }}.svc.cluster.local:2379
+            - name: ETCDCTL_CACERT
+              value: /opt/certs/ca/ca.crt
+            - name: ETCDCTL_CERT
+              value: /opt/certs/root-certs/tls.crt
+            - name: ETCDCTL_KEY
+              value: /opt/certs/root-certs/tls.key
+          image: quay.io/coreos/etcd:v3.5.1
+          imagePullPolicy: Always
+          name: etcd-client
+          volumeMounts:
+            - name: root-certs
+              mountPath: /opt/certs/root-certs
+            - name: certs
+              mountPath: /opt/certs/ca
+      securityContext:
+        runAsUser: 1000
+        runAsGroup: 1000
+        fsGroup: 1000
+      volumes:
+        - name: root-certs
+          secret:
+            secretName: {{ include "etcd.clientSecretName" . }}
+            optional: true
+        - name: csr
+          configMap:
+            name: {{ include "etcd.csrConfigMapName" . }}
+        - name: certs
+          emptyDir: {}
+{{- end }}

helm/kamaji/templates/etcd_rbac.yaml

@@ -0,0 +1,49 @@
+{{- if .Values.etcd.deploy }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  labels:
+    {{- include "etcd.labels" . | nindent 4 }}
+  name: etcd-gen-certs-role
+  namespace: {{ .Release.Namespace }}
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - secrets
+    verbs:
+      - delete
+    resourceNames:
+      - {{ include "etcd.caSecretName" . }}
+      - {{ include "etcd.clientSecretName" . }}
+  - apiGroups:
+      - ""
+    resources:
+      - secrets
+    verbs:
+      - create
+  - apiGroups:
+      - apps
+    resources:
+      - statefulsets
+    verbs:
+      - get
+      - list
+      - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  labels:
+    {{- include "etcd.labels" . | nindent 4 }}
+  name: etcd-gen-certs-rolebiding
+  namespace: {{ .Release.Namespace }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: etcd-gen-certs-role
+subjects:
+  - kind: ServiceAccount
+    name: {{ include "etcd.serviceAccountName" . }}
+    namespace: {{ .Release.Namespace }}
+{{- end }}

helm/kamaji/templates/etcd_sa.yaml

@@ -0,0 +1,9 @@
+{{- if .Values.etcd.deploy }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    {{- include "etcd.labels" . | nindent 4 }}
+  name: {{ include "etcd.serviceAccountName" . }}
+  namespace: {{ .Release.Namespace }}
+{{- end }}

helm/kamaji/templates/etcd_service.yaml

@@ -0,0 +1,18 @@
+{{- if .Values.etcd.deploy }}
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    {{- include "etcd.labels" . | nindent 4 }}
+  name: {{ include "etcd.serviceName" . }}
+  namespace: {{ .Release.Namespace }}
+spec:
+  clusterIP: None
+  ports:
+    - port: 2379
+      name: client
+    - port: 2380
+      name: peer
+  selector:
+    {{- include "etcd.selectorLabels" . | nindent 4 }}
+{{- end }}

helm/kamaji/templates/etcd_sts.yaml

@@ -0,0 +1,97 @@
+{{- if .Values.etcd.deploy }}
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  labels:
+    {{- include "etcd.labels" . | nindent 4 }}
+  name: {{ include "etcd.fullname" . }}
+  namespace: {{ .Release.Namespace }}
+spec:
+  serviceName: {{ include "etcd.serviceName" . }}
+  selector:
+    matchLabels:
+      {{- include "etcd.selectorLabels" . | nindent 6 }}
+  replicas: 3
+  template:
+    metadata:
+      name: etcd
+      labels:
+        {{- include "etcd.selectorLabels" . | nindent 8 }}
+    spec:
+      volumes:
+        - name: certs
+          secret:
+            secretName: {{ include "etcd.caSecretName" . }}
+      containers:
+        - name: etcd
+          image: quay.io/coreos/etcd:v3.5.1
+          ports:
+            - containerPort: 2379
+              name: client
+            - containerPort: 2380
+              name: peer
+          volumeMounts:
+            - name: data
+              mountPath: /var/run/etcd
+            - name: certs
+              mountPath: /etc/etcd/pki
+          command:
+            - etcd
+            - --data-dir=/var/run/etcd
+            - --name=$(POD_NAME)
+            - --initial-cluster-state=new
+            - --initial-cluster=etcd-0=https://etcd-0.etcd.$(POD_NAMESPACE).svc.cluster.local:2380,etcd-1=https://etcd-1.etcd.$(POD_NAMESPACE).svc.cluster.local:2380,etcd-2=https://etcd-2.etcd.$(POD_NAMESPACE).svc.cluster.local:2380
+            - --initial-advertise-peer-urls=https://$(POD_NAME).etcd.$(POD_NAMESPACE).svc.cluster.local:2380
+            - --initial-cluster-token=kamaji
+            - --listen-client-urls=https://0.0.0.0:2379
+            - --advertise-client-urls={{ include "etcd.endpoints" . }}
+            - --client-cert-auth=true
+            - --trusted-ca-file=/etc/etcd/pki/ca.crt
+            - --cert-file=/etc/etcd/pki/server.pem
+            - --key-file=/etc/etcd/pki/server-key.pem
+            - --listen-peer-urls=https://0.0.0.0:2380
+            - --peer-client-cert-auth=true
+            - --peer-trusted-ca-file=/etc/etcd/pki/ca.crt
+            - --peer-cert-file=/etc/etcd/pki/peer.pem
+            - --peer-key-file=/etc/etcd/pki/peer-key.pem
+            - --auto-compaction-mode=periodic
+            - --auto-compaction-retention=5m
+            - --v=8
+          env:
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+          livenessProbe:
+            failureThreshold: 8
+            httpGet:
+              host: 127.0.0.1
+              path: /health
+              port: 2381
+              scheme: HTTP
+            initialDelaySeconds: 10
+            periodSeconds: 10
+            timeoutSeconds: 15
+          startupProbe:
+            failureThreshold: 24
+            httpGet:
+              host: 127.0.0.1
+              path: /health
+              port: 2381
+              scheme: HTTP
+            initialDelaySeconds: 10
+            periodSeconds: 10
+            timeoutSeconds: 15
+  volumeClaimTemplates:
+    - metadata:
+        name: data
+      spec:
+        accessModes: ["ReadWriteOnce"]
+        resources:
+          requests:
+            storage: 8Gi
+{{- end }}

helm/kamaji/templates/ingress.yaml

@@ -1,61 +0,0 @@
-{{- if .Values.ingress.enabled -}}
-{{- $fullName := include "kamaji.fullname" . -}}
-{{- $svcPort := .Values.service.port -}}
-{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }}
-  {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }}
-  {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}}
-  {{- end }}
-{{- end }}
-{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}}
-apiVersion: networking.k8s.io/v1
-{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}}
-apiVersion: networking.k8s.io/v1beta1
-{{- else -}}
-apiVersion: extensions/v1beta1
-{{- end }}
-kind: Ingress
-metadata:
-  name: {{ $fullName }}
-  labels:
-    {{- include "kamaji.labels" . | nindent 4 }}
-  {{- with .Values.ingress.annotations }}
-  annotations:
-    {{- toYaml . | nindent 4 }}
-  {{- end }}
-spec:
-  {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }}
-  ingressClassName: {{ .Values.ingress.className }}
-  {{- end }}
-  {{- if .Values.ingress.tls }}
-  tls:
-    {{- range .Values.ingress.tls }}
-    - hosts:
-        {{- range .hosts }}
-        - {{ . | quote }}
-        {{- end }}
-      secretName: {{ .secretName }}
-    {{- end }}
-  {{- end }}
-  rules:
-    {{- range .Values.ingress.hosts }}
-    - host: {{ .host | quote }}
-      http:
-        paths:
-          {{- range .paths }}
-          - path: {{ .path }}
-            {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }}
-            pathType: {{ .pathType }}
-            {{- end }}
-            backend:
-              {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }}
-              service:
-                name: {{ $fullName }}
-                port:
-                  number: {{ $svcPort }}
-              {{- else }}
-              serviceName: {{ $fullName }}
-              servicePort: {{ $svcPort }}
-              {{- end }}
-          {{- end }}
-    {{- end }}
-{{- end }}

helm/kamaji/values.sample.yaml

@@ -1,2 +1,2 @@
 etcd:
-  endpoints: "etcd-0.etcd.kamaji-system.svc.cluster.local:2379,etcd-1.etcd.kamaji-system.svc.cluster.local:2379,etcd-2.etcd.kamaji-system.svc.cluster.local:2379"
+  endpoints: "https://etcd-0.etcd.kamaji-system.svc.cluster.local:2379,https://etcd-1.etcd.kamaji-system.svc.cluster.local:2379,https://etcd-2.etcd.kamaji-system.svc.cluster.local:2379"

helm/kamaji/values.yaml

@@ -7,8 +7,8 @@ replicaCount: 1
 
 image:
   # -- The container image of the Kamaji controller.
-  repository: quay.io/clastix/kamaji
-  pullPolicy: IfNotPresent
+  repository: clastix/kamaji
+  pullPolicy: Always
   # Overrides the image tag whose default is the chart appVersion.
   tag: latest
 
@@ -19,24 +19,29 @@ extraArgs: []
 configPath: "./kamaji.yaml"
 
 etcd:
-  caSecret:
-    # -- Name of the secret which contains CA's certificate and private key. (default: "etcd-certs")
-    name: etcd-certs
-    # -- Namespace of the secret which contains CA's certificate and private key. (default: "kamaji")
-    namespace: kamaji-system
-
-  clientSecret:
-    # -- Name of the secret which contains ETCD client certificates. (default: "root-client-certs")
-    name: root-client-certs
-    # -- Name of the namespace where the secret which contains ETCD client certificates is. (default: "kamaji")
-    namespace: kamaji-system
-
+  # -- Install an etcd 3.5 with enabled multi-tenancy along with Kamaji
+  deploy: true
+  serviceAccount:
+    # -- Create a ServiceAccount, required to install and provision the etcd backing storage (default: true)
+    create: true
+    # -- Define the ServiceAccount name to use during the setup and provision of the etcd backing storage (default: "")
+    name: ""
+  overrides:
+    caSecret:
+      # -- Name of the secret which contains CA's certificate and private key. (default: "etcd-certs")
+      name: etcd-certs
+      # -- Namespace of the secret which contains CA's certificate and private key. (default: "kamaji-system")
+      namespace: kamaji-system
+    clientSecret:
+      # -- Name of the secret which contains ETCD client certificates. (default: "root-client-certs")
+      name: root-client-certs
+      # -- Name of the namespace where the secret which contains ETCD client certificates is. (default: "kamaji-system")
+      namespace: kamaji-system
+    # -- (string) Comma-separated list of the endpoints of the etcd cluster's members.
+    endpoints: "https://etcd-0.etcd.kamaji-system.svc.cluster.local:2379,https://etcd-1.etcd.kamaji-system.svc.cluster.local:2379,https://etcd-2.etcd.kamaji-system.svc.cluster.local:2379"
   # -- ETCD Compaction interval (e.g. "5m0s"). (default: "0" (disabled))
   compactionInterval: 0
 
-  # -- (string) Comma-separated list of the endpoints of the etcd cluster's members.
-  endpoints: "etcd-0.etcd.kamaji-system.svc.cluster.local:2379,etcd-1.etcd.kamaji-system.svc.cluster.local:2379,etcd-2.etcd.kamaji-system.svc.cluster.local:2379"
-
 # -- The address the probe endpoint binds to. (default ":8081")
 healthProbeBindAddress: ":8081"
 
@@ -93,24 +98,6 @@ service:
   type: ClusterIP
   port: 8443
 
-ingress:
-  # -- Whether to expose the Kamaji controller through an Ingress.
-  enabled: false
-  # -- Name of the ingress class to route through this controller.
-  className: ""
-  annotations: {}
-    # kubernetes.io/ingress.class: nginx
-    # kubernetes.io/tls-acme: "true"
-  hosts:
-    - host: chart-example.local
-      paths:
-        - path: /
-          pathType: ImplementationSpecific
-  tls: []
-  #  - secretName: chart-example-tls
-  #    hosts:
-  #      - chart-example.local
-
 resources:
   limits:
     cpu: 200m

internal/builders/controlplane/deployment.go

@@ -0,0 +1,731 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package controlplane
+
+import (
+	"fmt"
+	"path"
+	"strconv"
+	"strings"
+
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3"
+	"k8s.io/kubernetes/cmd/kubeadm/app/constants"
+	"k8s.io/utils/pointer"
+
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+	"github.com/clastix/kamaji/internal/types"
+	"github.com/clastix/kamaji/internal/utilities"
+)
+
+type orderedIndex int
+
+const (
+	apiServerIndex orderedIndex = iota
+	schedulerIndex
+	controllerManagerIndex
+	kineIndex
+)
+
+const (
+	etcKubernetesPKIVolume orderedIndex = iota
+	etcCACertificates
+	etcSSLCerts
+	usrShareCACertificates
+	usrLocalShareCACertificates
+	schedulerKubeconfig
+	controllerManagerKubeconfig
+	kineConfig
+)
+
+const (
+	apiServerFlagsAnnotation = "kube-apiserver.kamaji.clastix.io/args"
+	kineVolumeName           = "kine-config"
+)
+
+type Deployment struct {
+	Address                string
+	ETCDEndpoints          []string
+	ETCDCompactionInterval string
+	ETCDStorageType        types.ETCDStorageType
+	KineContainerImage     string
+}
+
+func (d *Deployment) SetContainers(podSpec *corev1.PodSpec, tcp *kamajiv1alpha1.TenantControlPlane, address string) {
+	d.buildKubeAPIServer(podSpec, tcp, address)
+	d.BuildScheduler(podSpec, tcp)
+	d.buildControllerManager(podSpec, tcp)
+	d.buildKine(podSpec, tcp)
+}
+
+func (d *Deployment) SetStrategy(deployment *appsv1.DeploymentSpec) {
+	maxSurge := intstr.FromString("100%")
+
+	maxUnavailable := intstr.FromInt(0)
+
+	deployment.Strategy = appsv1.DeploymentStrategy{
+		Type: appsv1.RollingUpdateDeploymentStrategyType,
+		RollingUpdate: &appsv1.RollingUpdateDeployment{
+			MaxUnavailable: &maxUnavailable,
+			MaxSurge:       &maxSurge,
+		},
+	}
+}
+
+func (d *Deployment) SetVolumes(podSpec *corev1.PodSpec, tcp *kamajiv1alpha1.TenantControlPlane) {
+	for _, fn := range []func(*corev1.PodSpec, *kamajiv1alpha1.TenantControlPlane){
+		d.buildPKIVolume,
+		d.buildCAVolume,
+		d.buildSSLCertsVolume,
+		d.buildShareCAVolume,
+		d.buildLocalShareCAVolume,
+		d.buildSchedulerVolume,
+		d.buildControllerManagerVolume,
+		d.buildKineVolume,
+	} {
+		fn(podSpec, tcp)
+	}
+}
+
+func (d *Deployment) buildPKIVolume(podSpec *corev1.PodSpec, tcp *kamajiv1alpha1.TenantControlPlane) {
+	if index := int(etcKubernetesPKIVolume) + 1; len(podSpec.Volumes) < index {
+		podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{})
+	}
+
+	sources := []corev1.VolumeProjection{
+		{
+			Secret: d.secretProjection(tcp.Status.Certificates.APIServer.SecretName, constants.APIServerCertName, constants.APIServerKeyName),
+		},
+		{
+			Secret: d.secretProjection(tcp.Status.Certificates.CA.SecretName, constants.CACertName, constants.CAKeyName),
+		},
+		{
+			Secret: d.secretProjection(tcp.Status.Certificates.APIServerKubeletClient.SecretName, constants.APIServerKubeletClientCertName, constants.APIServerKubeletClientKeyName),
+		},
+		{
+			Secret: d.secretProjection(tcp.Status.Certificates.FrontProxyCA.SecretName, constants.FrontProxyCACertName, constants.FrontProxyCAKeyName),
+		},
+		{
+			Secret: d.secretProjection(tcp.Status.Certificates.FrontProxyClient.SecretName, constants.FrontProxyClientCertName, constants.FrontProxyClientKeyName),
+		},
+		{
+			Secret: d.secretProjection(tcp.Status.Certificates.SA.SecretName, constants.ServiceAccountPublicKeyName, constants.ServiceAccountPrivateKeyName),
+		},
+	}
+
+	if d.ETCDStorageType == types.ETCD {
+		sources = append(sources, corev1.VolumeProjection{
+			Secret: d.secretProjection(tcp.Status.Certificates.ETCD.APIServer.SecretName, constants.APIServerEtcdClientCertName, constants.APIServerEtcdClientKeyName),
+		})
+		sources = append(sources, corev1.VolumeProjection{
+			Secret: &corev1.SecretProjection{
+				LocalObjectReference: corev1.LocalObjectReference{
+					Name: tcp.Status.Certificates.ETCD.CA.SecretName,
+				},
+				Items: []corev1.KeyToPath{
+					{
+						Key:  constants.CACertName,
+						Path: constants.EtcdCACertName,
+					},
+				},
+			},
+		})
+	}
+
+	podSpec.Volumes[etcKubernetesPKIVolume] = corev1.Volume{
+		Name: "etc-kubernetes-pki",
+		VolumeSource: corev1.VolumeSource{
+			Projected: &corev1.ProjectedVolumeSource{
+				Sources:     sources,
+				DefaultMode: pointer.Int32Ptr(420),
+			},
+		},
+	}
+}
+
+func (d *Deployment) buildCAVolume(podSpec *corev1.PodSpec, tcp *kamajiv1alpha1.TenantControlPlane) {
+	if index := int(etcCACertificates) + 1; len(podSpec.Volumes) < index {
+		podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{})
+	}
+
+	podSpec.Volumes[etcCACertificates] = corev1.Volume{
+		Name: "etc-ca-certificates",
+		VolumeSource: corev1.VolumeSource{
+			Secret: &corev1.SecretVolumeSource{
+				SecretName:  tcp.Status.Certificates.CA.SecretName,
+				DefaultMode: pointer.Int32Ptr(420),
+			},
+		},
+	}
+}
+
+func (d *Deployment) buildSSLCertsVolume(podSpec *corev1.PodSpec, tcp *kamajiv1alpha1.TenantControlPlane) {
+	if index := int(etcSSLCerts) + 1; len(podSpec.Volumes) < index {
+		podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{})
+	}
+
+	podSpec.Volumes[etcSSLCerts] = corev1.Volume{
+		Name: "etc-ssl-certs",
+		VolumeSource: corev1.VolumeSource{
+			Secret: &corev1.SecretVolumeSource{
+				SecretName:  tcp.Status.Certificates.CA.SecretName,
+				DefaultMode: pointer.Int32Ptr(420),
+			},
+		},
+	}
+}
+
+func (d *Deployment) buildShareCAVolume(podSpec *corev1.PodSpec, tcp *kamajiv1alpha1.TenantControlPlane) {
+	if index := int(usrShareCACertificates) + 1; len(podSpec.Volumes) < index {
+		podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{})
+	}
+
+	podSpec.Volumes[usrShareCACertificates] = corev1.Volume{
+		Name: "usr-share-ca-certificates",
+		VolumeSource: corev1.VolumeSource{
+			Secret: &corev1.SecretVolumeSource{
+				SecretName:  tcp.Status.Certificates.CA.SecretName,
+				DefaultMode: pointer.Int32Ptr(420),
+			},
+		},
+	}
+}
+
+func (d *Deployment) buildLocalShareCAVolume(podSpec *corev1.PodSpec, tcp *kamajiv1alpha1.TenantControlPlane) {
+	if index := int(usrLocalShareCACertificates) + 1; len(podSpec.Volumes) < index {
+		podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{})
+	}
+
+	podSpec.Volumes[usrLocalShareCACertificates] = corev1.Volume{
+		Name: "usr-local-share-ca-certificates",
+		VolumeSource: corev1.VolumeSource{
+			Secret: &corev1.SecretVolumeSource{
+				SecretName:  tcp.Status.Certificates.CA.SecretName,
+				DefaultMode: pointer.Int32Ptr(420),
+			},
+		},
+	}
+}
+
+func (d *Deployment) buildSchedulerVolume(podSpec *corev1.PodSpec, tcp *kamajiv1alpha1.TenantControlPlane) {
+	if index := int(schedulerKubeconfig) + 1; len(podSpec.Volumes) < index {
+		podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{})
+	}
+
+	podSpec.Volumes[schedulerKubeconfig] = corev1.Volume{
+		Name: "scheduler-kubeconfig",
+		VolumeSource: corev1.VolumeSource{
+			Secret: &corev1.SecretVolumeSource{
+				SecretName:  tcp.Status.KubeConfig.Scheduler.SecretName,
+				DefaultMode: pointer.Int32Ptr(420),
+			},
+		},
+	}
+}
+
+func (d *Deployment) buildControllerManagerVolume(podSpec *corev1.PodSpec, tcp *kamajiv1alpha1.TenantControlPlane) {
+	if index := int(controllerManagerKubeconfig) + 1; len(podSpec.Volumes) < index {
+		podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{})
+	}
+
+	podSpec.Volumes[controllerManagerKubeconfig] = corev1.Volume{
+		Name: "controller-manager-kubeconfig",
+		VolumeSource: corev1.VolumeSource{
+			Secret: &corev1.SecretVolumeSource{
+				SecretName:  tcp.Status.KubeConfig.ControllerManager.SecretName,
+				DefaultMode: pointer.Int32Ptr(420),
+			},
+		},
+	}
+}
+
+func (d *Deployment) BuildScheduler(podSpec *corev1.PodSpec, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) {
+	if index := int(schedulerIndex) + 1; len(podSpec.Containers) < index {
+		podSpec.Containers = append(podSpec.Containers, corev1.Container{})
+	}
+
+	args := map[string]string{}
+
+	if tenantControlPlane.Spec.ControlPlane.Deployment.ExtraArgs != nil {
+		args = utilities.ArgsFromSliceToMap(tenantControlPlane.Spec.ControlPlane.Deployment.ExtraArgs.Scheduler)
+	}
+
+	kubeconfig := "/etc/kubernetes/scheduler.conf"
+
+	args["--authentication-kubeconfig"] = kubeconfig
+	args["--authorization-kubeconfig"] = kubeconfig
+	args["--bind-address"] = "0.0.0.0"
+	args["--kubeconfig"] = kubeconfig
+	args["--leader-elect"] = "true" // nolint:goconst
+
+	podSpec.Containers[schedulerIndex].Name = "kube-scheduler"
+	podSpec.Containers[schedulerIndex].Image = fmt.Sprintf("k8s.gcr.io/kube-scheduler:%s", tenantControlPlane.Spec.Kubernetes.Version)
+	podSpec.Containers[schedulerIndex].Command = []string{"kube-scheduler"}
+	podSpec.Containers[schedulerIndex].Args = utilities.ArgsFromMapToSlice(args)
+	podSpec.Containers[schedulerIndex].VolumeMounts = []corev1.VolumeMount{
+		{
+			Name:      "scheduler-kubeconfig",
+			ReadOnly:  true,
+			MountPath: "/etc/kubernetes",
+		},
+	}
+	podSpec.Containers[schedulerIndex].LivenessProbe = &corev1.Probe{
+		ProbeHandler: corev1.ProbeHandler{
+			HTTPGet: &corev1.HTTPGetAction{
+				Path:   "/healthz",
+				Port:   intstr.FromInt(10259),
+				Scheme: corev1.URISchemeHTTPS,
+			},
+		},
+		InitialDelaySeconds: 0,
+		TimeoutSeconds:      1,
+		PeriodSeconds:       10,
+		SuccessThreshold:    1,
+		FailureThreshold:    3,
+	}
+	podSpec.Containers[schedulerIndex].StartupProbe = &corev1.Probe{
+		ProbeHandler: corev1.ProbeHandler{
+			HTTPGet: &corev1.HTTPGetAction{
+				Path:   "/healthz",
+				Port:   intstr.FromInt(10259),
+				Scheme: corev1.URISchemeHTTPS,
+			},
+		},
+		InitialDelaySeconds: 0,
+		TimeoutSeconds:      1,
+		PeriodSeconds:       10,
+		SuccessThreshold:    1,
+		FailureThreshold:    3,
+	}
+	podSpec.Containers[schedulerIndex].ImagePullPolicy = corev1.PullAlways
+	podSpec.Containers[schedulerIndex].Resources = corev1.ResourceRequirements{
+		Limits:   nil,
+		Requests: nil,
+	}
+
+	if componentsResources := tenantControlPlane.Spec.ControlPlane.Deployment.Resources; componentsResources != nil {
+		if resource := componentsResources.Scheduler; resource != nil {
+			podSpec.Containers[schedulerIndex].Resources = *resource
+		}
+	}
+}
+
+func (d *Deployment) buildControllerManager(podSpec *corev1.PodSpec, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) {
+	if index := int(controllerManagerIndex) + 1; len(podSpec.Containers) < index {
+		podSpec.Containers = append(podSpec.Containers, corev1.Container{})
+	}
+
+	args := map[string]string{}
+
+	if tenantControlPlane.Spec.ControlPlane.Deployment.ExtraArgs != nil {
+		args = utilities.ArgsFromSliceToMap(tenantControlPlane.Spec.ControlPlane.Deployment.ExtraArgs.ControllerManager)
+	}
+
+	kubeconfig := "/etc/kubernetes/controller-manager.conf"
+
+	args["--allocate-node-cidrs"] = "true"
+	args["--authentication-kubeconfig"] = kubeconfig
+	args["--authorization-kubeconfig"] = kubeconfig
+	args["--bind-address"] = "0.0.0.0"
+	args["--client-ca-file"] = path.Join(v1beta3.DefaultCertificatesDir, constants.CACertName)
+	args["--cluster-name"] = tenantControlPlane.GetName()
+	args["--cluster-signing-cert-file"] = path.Join(v1beta3.DefaultCertificatesDir, constants.CACertName)
+	args["--cluster-signing-key-file"] = path.Join(v1beta3.DefaultCertificatesDir, constants.CAKeyName)
+	args["--controllers"] = "*,bootstrapsigner,tokencleaner"
+	args["--kubeconfig"] = kubeconfig
+	args["--leader-elect"] = "true"
+	args["--service-cluster-ip-range"] = tenantControlPlane.Spec.NetworkProfile.ServiceCIDR
+	args["--cluster-cidr"] = tenantControlPlane.Spec.NetworkProfile.PodCIDR
+	args["--requestheader-client-ca-file"] = path.Join(v1beta3.DefaultCertificatesDir, constants.FrontProxyCACertName)
+	args["--root-ca-file"] = path.Join(v1beta3.DefaultCertificatesDir, constants.CACertName)
+	args["--service-account-private-key-file"] = path.Join(v1beta3.DefaultCertificatesDir, constants.ServiceAccountPrivateKeyName)
+	args["--use-service-account-credentials"] = "true"
+
+	podSpec.Containers[controllerManagerIndex].Name = "kube-controller-manager"
+	podSpec.Containers[controllerManagerIndex].Image = fmt.Sprintf("k8s.gcr.io/kube-controller-manager:%s", tenantControlPlane.Spec.Kubernetes.Version)
+	podSpec.Containers[controllerManagerIndex].Command = []string{"kube-controller-manager"}
+	podSpec.Containers[controllerManagerIndex].Args = utilities.ArgsFromMapToSlice(args)
+	podSpec.Containers[controllerManagerIndex].VolumeMounts = []corev1.VolumeMount{
+		{
+			Name:      "controller-manager-kubeconfig",
+			ReadOnly:  true,
+			MountPath: "/etc/kubernetes",
+		},
+		{
+			Name:      "etc-kubernetes-pki",
+			ReadOnly:  true,
+			MountPath: v1beta3.DefaultCertificatesDir,
+		},
+		{
+			Name:      "etc-ca-certificates",
+			ReadOnly:  true,
+			MountPath: "/etc/ca-certificates",
+		},
+		{
+			Name:      "etc-ssl-certs",
+			ReadOnly:  true,
+			MountPath: "/etc/ssl/certs",
+		},
+		{
+			Name:      "usr-share-ca-certificates",
+			ReadOnly:  true,
+			MountPath: "/usr/share/ca-certificates",
+		},
+		{
+			Name:      "usr-local-share-ca-certificates",
+			ReadOnly:  true,
+			MountPath: "/usr/local/share/ca-certificates",
+		},
+	}
+	podSpec.Containers[controllerManagerIndex].LivenessProbe = &corev1.Probe{
+		ProbeHandler: corev1.ProbeHandler{
+			HTTPGet: &corev1.HTTPGetAction{
+				Path:   "/healthz",
+				Port:   intstr.FromInt(10257),
+				Scheme: corev1.URISchemeHTTPS,
+			},
+		},
+		InitialDelaySeconds: 0,
+		TimeoutSeconds:      1,
+		PeriodSeconds:       10,
+		SuccessThreshold:    1,
+		FailureThreshold:    3,
+	}
+	podSpec.Containers[controllerManagerIndex].StartupProbe = &corev1.Probe{
+		ProbeHandler: corev1.ProbeHandler{
+			HTTPGet: &corev1.HTTPGetAction{
+				Path:   "/healthz",
+				Port:   intstr.FromInt(10257),
+				Scheme: corev1.URISchemeHTTPS,
+			},
+		},
+		InitialDelaySeconds: 0,
+		TimeoutSeconds:      1,
+		PeriodSeconds:       10,
+		SuccessThreshold:    1,
+		FailureThreshold:    3,
+	}
+	podSpec.Containers[controllerManagerIndex].ImagePullPolicy = corev1.PullAlways
+	podSpec.Containers[controllerManagerIndex].Resources = corev1.ResourceRequirements{
+		Limits:   nil,
+		Requests: nil,
+	}
+
+	if componentsResources := tenantControlPlane.Spec.ControlPlane.Deployment.Resources; componentsResources != nil {
+		if resource := componentsResources.ControllerManager; resource != nil {
+			podSpec.Containers[controllerManagerIndex].Resources = *resource
+		}
+	}
+}
+
+func (d *Deployment) buildKubeAPIServer(podSpec *corev1.PodSpec, tenantControlPlane *kamajiv1alpha1.TenantControlPlane, address string) {
+	if index := int(apiServerIndex) + 1; len(podSpec.Containers) < index {
+		podSpec.Containers = append(podSpec.Containers, corev1.Container{})
+	}
+
+	args := d.buildKubeAPIServerCommand(tenantControlPlane, address, utilities.ArgsFromSliceToMap(podSpec.Containers[apiServerIndex].Args))
+
+	podSpec.Containers[apiServerIndex].Name = "kube-apiserver"
+	podSpec.Containers[apiServerIndex].Args = utilities.ArgsFromMapToSlice(args)
+	podSpec.Containers[apiServerIndex].Image = fmt.Sprintf("k8s.gcr.io/kube-apiserver:%s", tenantControlPlane.Spec.Kubernetes.Version)
+	podSpec.Containers[apiServerIndex].Command = []string{"kube-apiserver"}
+	podSpec.Containers[apiServerIndex].LivenessProbe = &corev1.Probe{
+		ProbeHandler: corev1.ProbeHandler{
+			HTTPGet: &corev1.HTTPGetAction{
+				Path:   "/livez",
+				Port:   intstr.FromInt(int(tenantControlPlane.Spec.NetworkProfile.Port)),
+				Scheme: corev1.URISchemeHTTPS,
+			},
+		},
+		InitialDelaySeconds: 0,
+		TimeoutSeconds:      1,
+		PeriodSeconds:       10,
+		SuccessThreshold:    1,
+		FailureThreshold:    3,
+	}
+	podSpec.Containers[apiServerIndex].ReadinessProbe = &corev1.Probe{
+		ProbeHandler: corev1.ProbeHandler{
+			HTTPGet: &corev1.HTTPGetAction{
+				Path:   "/readyz",
+				Port:   intstr.FromInt(int(tenantControlPlane.Spec.NetworkProfile.Port)),
+				Scheme: corev1.URISchemeHTTPS,
+			},
+		},
+		InitialDelaySeconds: 0,
+		TimeoutSeconds:      1,
+		PeriodSeconds:       10,
+		SuccessThreshold:    1,
+		FailureThreshold:    3,
+	}
+	podSpec.Containers[apiServerIndex].StartupProbe = &corev1.Probe{
+		ProbeHandler: corev1.ProbeHandler{
+			HTTPGet: &corev1.HTTPGetAction{
+				Path:   "/livez",
+				Port:   intstr.FromInt(int(tenantControlPlane.Spec.NetworkProfile.Port)),
+				Scheme: corev1.URISchemeHTTPS,
+			},
+		},
+		InitialDelaySeconds: 0,
+		TimeoutSeconds:      1,
+		PeriodSeconds:       10,
+		SuccessThreshold:    1,
+		FailureThreshold:    3,
+	}
+	podSpec.Containers[apiServerIndex].ImagePullPolicy = corev1.PullAlways
+
+	if len(podSpec.Containers[apiServerIndex].VolumeMounts) < 5 {
+		podSpec.Containers[apiServerIndex].VolumeMounts = make([]corev1.VolumeMount, 5)
+	}
+	podSpec.Containers[apiServerIndex].VolumeMounts[0] = corev1.VolumeMount{
+		Name:      "etc-kubernetes-pki",
+		ReadOnly:  true,
+		MountPath: v1beta3.DefaultCertificatesDir,
+	}
+	podSpec.Containers[apiServerIndex].VolumeMounts[1] = corev1.VolumeMount{
+		Name:      "etc-ca-certificates",
+		ReadOnly:  true,
+		MountPath: "/etc/ca-certificates",
+	}
+	podSpec.Containers[apiServerIndex].VolumeMounts[2] = corev1.VolumeMount{
+		Name:      "etc-ssl-certs",
+		ReadOnly:  true,
+		MountPath: "/etc/ssl/certs",
+	}
+	podSpec.Containers[apiServerIndex].VolumeMounts[3] = corev1.VolumeMount{
+		Name:      "usr-share-ca-certificates",
+		ReadOnly:  true,
+		MountPath: "/usr/share/ca-certificates",
+	}
+	podSpec.Containers[apiServerIndex].VolumeMounts[4] = corev1.VolumeMount{
+		Name:      "usr-local-share-ca-certificates",
+		ReadOnly:  true,
+		MountPath: "/usr/local/share/ca-certificates",
+	}
+	podSpec.Containers[apiServerIndex].Resources = corev1.ResourceRequirements{
+		Limits:   nil,
+		Requests: nil,
+	}
+
+	if componentsResources := tenantControlPlane.Spec.ControlPlane.Deployment.Resources; componentsResources != nil {
+		if resource := componentsResources.APIServer; resource != nil {
+			podSpec.Containers[apiServerIndex].Resources = *resource
+		}
+	}
+}
+
+func (d *Deployment) buildKubeAPIServerCommand(tenantControlPlane *kamajiv1alpha1.TenantControlPlane, address string, current map[string]string) map[string]string {
+	var extraArgs map[string]string
+
+	if tenantControlPlane.Spec.ControlPlane.Deployment.ExtraArgs != nil {
+		extraArgs = utilities.ArgsFromSliceToMap(tenantControlPlane.Spec.ControlPlane.Deployment.ExtraArgs.APIServer)
+	}
+
+	desiredArgs := map[string]string{
+		"--allow-privileged":                   "true",
+		"--authorization-mode":                 "Node,RBAC",
+		"--advertise-address":                  address,
+		"--client-ca-file":                     path.Join(v1beta3.DefaultCertificatesDir, constants.CACertName),
+		"--enable-admission-plugins":           strings.Join(tenantControlPlane.Spec.Kubernetes.AdmissionControllers.ToSlice(), ","),
+		"--enable-bootstrap-token-auth":        "true",
+		"--etcd-servers":                       strings.Join(d.ETCDEndpoints, ","),
+		"--service-cluster-ip-range":           tenantControlPlane.Spec.NetworkProfile.ServiceCIDR,
+		"--kubelet-client-certificate":         path.Join(v1beta3.DefaultCertificatesDir, constants.APIServerKubeletClientCertName),
+		"--kubelet-client-key":                 path.Join(v1beta3.DefaultCertificatesDir, constants.APIServerKubeletClientKeyName),
+		"--kubelet-preferred-address-types":    "Hostname,InternalIP,ExternalIP",
+		"--proxy-client-cert-file":             path.Join(v1beta3.DefaultCertificatesDir, constants.FrontProxyClientCertName),
+		"--proxy-client-key-file":              path.Join(v1beta3.DefaultCertificatesDir, constants.FrontProxyClientKeyName),
+		"--requestheader-allowed-names":        "front-proxy-client",
+		"--requestheader-extra-headers-prefix": "X-Remote-Extra-",
+		"--requestheader-group-headers":        "X-Remote-Group",
+		"--requestheader-username-headers":     "X-Remote-User",
+		"--secure-port":                        fmt.Sprintf("%d", tenantControlPlane.Spec.NetworkProfile.Port),
+		"--service-account-issuer":             fmt.Sprintf("https://localhost:%d", tenantControlPlane.Spec.NetworkProfile.Port),
+		"--service-account-key-file":           path.Join(v1beta3.DefaultCertificatesDir, constants.ServiceAccountPublicKeyName),
+		"--service-account-signing-key-file":   path.Join(v1beta3.DefaultCertificatesDir, constants.ServiceAccountPrivateKeyName),
+		"--tls-cert-file":                      path.Join(v1beta3.DefaultCertificatesDir, constants.APIServerCertName),
+		"--tls-private-key-file":               path.Join(v1beta3.DefaultCertificatesDir, constants.APIServerKeyName),
+	}
+
+	if d.ETCDStorageType == types.ETCD {
+		desiredArgs["--etcd-compaction-interval"] = d.ETCDCompactionInterval
+		desiredArgs["--etcd-cafile"] = path.Join(v1beta3.DefaultCertificatesDir, constants.EtcdCACertName)
+		desiredArgs["--etcd-certfile"] = path.Join(v1beta3.DefaultCertificatesDir, constants.APIServerEtcdClientCertName)
+		desiredArgs["--etcd-keyfile"] = path.Join(v1beta3.DefaultCertificatesDir, constants.APIServerEtcdClientKeyName)
+		desiredArgs["--etcd-prefix"] = fmt.Sprintf("/%s", tenantControlPlane.GetName())
+	}
+	// Order matters, here: extraArgs could try to overwrite some arguments managed by Kamaji and that would be crucial.
+	// Adding as first element of the array of maps, we're sure that these overrides will be sanitized by our configuration.
+	return utilities.MergeMaps(extraArgs, current, desiredArgs)
+}
+
+func (d *Deployment) secretProjection(secretName, certKeyName, keyName string) *corev1.SecretProjection {
+	return &corev1.SecretProjection{
+		LocalObjectReference: corev1.LocalObjectReference{
+			Name: secretName,
+		},
+		Items: []corev1.KeyToPath{
+			{
+				Key:  certKeyName,
+				Path: certKeyName,
+			},
+			{
+				Key:  keyName,
+				Path: keyName,
+			},
+		},
+	}
+}
+
+func (d *Deployment) buildKineVolume(podSpec *corev1.PodSpec, tcp *kamajiv1alpha1.TenantControlPlane) {
+	// Kine is expecting an additional volume for its configuration, and it must be removed before proceeding with the
+	// customized storage that is idempotent
+	if found, index := utilities.HasNamedVolume(podSpec.Volumes, kineVolumeName); found {
+		var volumes []corev1.Volume
+
+		volumes = append(volumes, podSpec.Volumes[:index]...)
+		volumes = append(volumes, podSpec.Volumes[index+1:]...)
+
+		podSpec.Volumes = volumes
+	}
+
+	if d.ETCDStorageType == types.KineMySQL {
+		if index := int(kineConfig) + 1; len(podSpec.Volumes) < index {
+			podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{})
+		}
+
+		podSpec.Volumes[kineConfig].Name = kineVolumeName
+		podSpec.Volumes[kineConfig].VolumeSource = corev1.VolumeSource{
+			Secret: &corev1.SecretVolumeSource{
+				SecretName:  tcp.Status.Storage.KineMySQL.Certificate.SecretName,
+				DefaultMode: pointer.Int32Ptr(420),
+			},
+		}
+	}
+}
+
+func (d *Deployment) buildKine(podSpec *corev1.PodSpec, tcp *kamajiv1alpha1.TenantControlPlane) {
+	const kineContainerName = "kine"
+	// Kine is expecting an additional container, and it must be removed before proceeding with the additional one
+	// in order to make this function idempotent.
+	if found, index := utilities.HasNamedContainer(podSpec.Containers, kineContainerName); found {
+		var containers []corev1.Container
+
+		containers = append(containers, podSpec.Containers[:index]...)
+		containers = append(containers, podSpec.Containers[index+1:]...)
+
+		podSpec.Containers = containers
+	}
+
+	if d.ETCDStorageType == types.KineMySQL {
+		if index := int(kineIndex) + 1; len(podSpec.Containers) < index {
+			podSpec.Containers = append(podSpec.Containers, corev1.Container{})
+		}
+
+		args := map[string]string{}
+
+		if tcp.Spec.ControlPlane.Deployment.ExtraArgs != nil {
+			args = utilities.ArgsFromSliceToMap(tcp.Spec.ControlPlane.Deployment.ExtraArgs.Kine)
+		}
+
+		args["--endpoint"] = "mysql://$(MYSQL_USER):$(MYSQL_PASSWORD)@tcp($(MYSQL_HOST):$(MYSQL_PORT))/$(MYSQL_SCHEMA)"
+		args["--ca-file"] = "/kine/ca.crt"
+		args["--cert-file"] = "/kine/server.crt"
+		args["--key-file"] = "/kine/server.key"
+
+		podSpec.Containers[kineIndex].Name = kineContainerName
+		podSpec.Containers[kineIndex].Image = d.KineContainerImage
+		podSpec.Containers[kineIndex].Command = []string{"/bin/kine"}
+		podSpec.Containers[kineIndex].Args = utilities.ArgsFromMapToSlice(args)
+		podSpec.Containers[kineIndex].VolumeMounts = []corev1.VolumeMount{
+			{
+				Name:      kineVolumeName,
+				MountPath: "/kine",
+				ReadOnly:  true,
+			},
+		}
+		podSpec.Containers[kineIndex].Env = []corev1.EnvVar{
+			{
+				Name:  "GODEBUG",
+				Value: "x509ignoreCN=0",
+			},
+		}
+		podSpec.Containers[kineIndex].EnvFrom = []corev1.EnvFromSource{
+			{
+				SecretRef: &corev1.SecretEnvSource{
+					LocalObjectReference: corev1.LocalObjectReference{
+						Name: tcp.Status.Storage.KineMySQL.Config.SecretName,
+					},
+				},
+			},
+		}
+		podSpec.Containers[kineIndex].Ports = []corev1.ContainerPort{
+			{
+				ContainerPort: 2379,
+				Name:          "server",
+				Protocol:      corev1.ProtocolTCP,
+			},
+		}
+		podSpec.Containers[kineIndex].ImagePullPolicy = corev1.PullAlways
+	}
+}
+
+func (d *Deployment) SetSelector(deploymentSpec *appsv1.DeploymentSpec, tcp *kamajiv1alpha1.TenantControlPlane) {
+	deploymentSpec.Selector = &metav1.LabelSelector{
+		MatchLabels: map[string]string{
+			"kamaji.clastix.io/soot": tcp.GetName(),
+		},
+	}
+}
+
+func (d *Deployment) SetReplicas(deploymentSpec *appsv1.DeploymentSpec, tcp *kamajiv1alpha1.TenantControlPlane) {
+	deploymentSpec.Replicas = pointer.Int32(tcp.Spec.ControlPlane.Deployment.Replicas)
+}
+
+func (d *Deployment) SetTemplateLabels(template *corev1.PodTemplateSpec, labels map[string]string) {
+	template.SetLabels(labels)
+}
+
+func (d *Deployment) SetLabels(resource *appsv1.Deployment, labels map[string]string) {
+	resource.SetLabels(labels)
+}
+
+func (d *Deployment) SetAnnotations(resource *appsv1.Deployment, annotations map[string]string) {
+	resource.SetAnnotations(annotations)
+}
+
+// ResetKubeAPIServerFlags ensures that upon a change of the kube-apiserver extra flags the desired ones are properly
+// applied, also considering that the container could be lately patched by the konnectivity addon resources.
+func (d *Deployment) ResetKubeAPIServerFlags(resource *appsv1.Deployment, tcp *kamajiv1alpha1.TenantControlPlane) {
+	if tcp.Spec.ControlPlane.Deployment.ExtraArgs == nil {
+		return
+	}
+	// kube-apiserver container is not still there, we can skip the hashing
+	if found, _ := utilities.HasNamedContainer(resource.Spec.Template.Spec.Containers, "kube-apiserver"); !found {
+		return
+	}
+	// setting up annotation to avoid assignment to a nil one
+	if resource.GetAnnotations() == nil {
+		resource.SetAnnotations(map[string]string{})
+	}
+	// retrieving the current amount of extra flags, used as a sort of hash:
+	// in case of non-matching values, removing all the args in order to perform a full reconciliation from a clean start.
+	var count int
+
+	if v, ok := resource.GetAnnotations()[apiServerFlagsAnnotation]; ok {
+		var err error
+
+		if count, err = strconv.Atoi(v); err != nil {
+			return
+		}
+	}
+	// there's a mismatch in the count from the previous hash: let's reset and store the desired extra args count.
+	if count != len(tcp.Spec.ControlPlane.Deployment.ExtraArgs.APIServer) {
+		resource.Spec.Template.Spec.Containers[apiServerIndex].Args = []string{}
+	}
+
+	resource.GetAnnotations()[apiServerFlagsAnnotation] = fmt.Sprintf("%d", len(tcp.Spec.ControlPlane.Deployment.ExtraArgs.APIServer))
+}

internal/config/config.go

@@ -22,6 +22,7 @@ var (
 
 const (
 	envPrefix                        = "KAMAJI"
+	defaultETCDStorageType           = "etcd"
 	defaultETCDCASecretName          = "etcd-certs"
 	defaultETCDCASecretNamespace     = "kamaji-system"
 	defaultETCDEndpoints             = "etcd-server:2379"
@@ -29,6 +30,11 @@ const (
 	defaultETCDClientSecretName      = "root-client-certs"
 	defaultETCDClientSecretNamespace = "kamaji-system"
 	defaultTmpDirectory              = "/tmp/kamaji"
+	defaultKineMySQLSecretName       = "mysql-config"
+	defaultKineMySQLSecretNamespace  = "kamaji-system"
+	defaultKineMySQLHost             = "localhost"
+	defaultKineMySQLPort             = 3306
+	defaultKineImage                 = "rancher/kine:v0.9.2-amd64"
 )
 
 func InitConfig() (*viper.Viper, error) {
@@ -40,13 +46,19 @@ func InitConfig() (*viper.Viper, error) {
 	flag.String("health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
 	flag.Bool("leader-elect", false, "Enable leader election for controller manager. "+
 		"Enabling this will ensure there is only one active controller manager.")
+	flag.String("etcd-storage-type", defaultETCDStorageType, "Type of storage for ETCD (i.e etcd, kine-mysql, kine-postgres)")
 	flag.String("etcd-ca-secret-name", defaultETCDCASecretName, "Name of the secret which contains CA's certificate and private key.")
 	flag.String("etcd-ca-secret-namespace", defaultETCDCASecretNamespace, "Namespace of the secret which contains CA's certificate and private key.")
 	flag.String("etcd-client-secret-name", defaultETCDClientSecretName, "Name of the secret which contains ETCD client certificates")
 	flag.String("etcd-client-secret-namespace", defaultETCDClientSecretNamespace, "Name of the namespace where the secret which contains ETCD client certificates is")
-	flag.String("etcd-endpoints", defaultETCDEndpoints, "Comma-separated list with ETCD endpoints (i.e. etcd-0.etcd.kamaji-system.svc.cluster.local,etcd-1.etcd.kamaji-system.svc.cluster.local,etcd-2.etcd.kamaji-system.svc.cluster.local)")
+	flag.String("etcd-endpoints", defaultETCDEndpoints, "Comma-separated list with ETCD endpoints (i.e. https://etcd-0.etcd.kamaji-system.svc.cluster.local,https://etcd-1.etcd.kamaji-system.svc.cluster.local,https://etcd-2.etcd.kamaji-system.svc.cluster.local)")
 	flag.String("etcd-compaction-interval", defaultETCDCompactionInterval, "ETCD Compaction interval (i.e. \"5m0s\"). (default: \"0\" (disabled))")
 	flag.String("tmp-directory", defaultTmpDirectory, "Directory which will be used to work with temporary files.")
+	flag.String("kine-mysql-secret-name", defaultKineMySQLSecretName, "Name of the secret which contains MySQL (Kine) configuration.")
+	flag.String("kine-mysql-secret-namespace", defaultKineMySQLSecretNamespace, "Name of the namespace where the secret which contains MySQL (Kine) configuration.")
+	flag.String("kine-mysql-host", defaultKineMySQLHost, "Host where MySQL (Kine) is working")
+	flag.Int("kine-mysql-port", defaultKineMySQLPort, "Port where MySQL (Kine) is working")
+	flag.String("kine-image", defaultKineImage, "Container image along with tag to use for the Kine sidecar container (used only if etcd-storage-type is set to one of kine strategies)")
 
 	// Setup zap configuration
 	opts := zap.Options{
@@ -74,6 +86,9 @@ func InitConfig() (*viper.Viper, error) {
 	if err := config.BindEnv("leader-elect", fmt.Sprintf("%s_LEADER_ELECTION", envPrefix)); err != nil {
 		return nil, err
 	}
+	if err := config.BindEnv("etcd-storage-type", fmt.Sprintf("%s_ETCD_STORAGE_TYPE", envPrefix)); err != nil {
+		return nil, err
+	}
 	if err := config.BindEnv("etcd-ca-secret-name", fmt.Sprintf("%s_ETCD_CA_SECRET_NAME", envPrefix)); err != nil {
 		return nil, err
 	}
@@ -95,6 +110,21 @@ func InitConfig() (*viper.Viper, error) {
 	if err := config.BindEnv("tmp-directory", fmt.Sprintf("%s_TMP_DIRECTORY", envPrefix)); err != nil {
 		return nil, err
 	}
+	if err := config.BindEnv("kine-mysql-secret-name", fmt.Sprintf("%s_KINE_MYSQL_SECRET_NAME", envPrefix)); err != nil {
+		return nil, err
+	}
+	if err := config.BindEnv("kine-mysql-secret-namespace", fmt.Sprintf("%s_KINE_MYSQL_SECRET_NAMESPACE", envPrefix)); err != nil {
+		return nil, err
+	}
+	if err := config.BindEnv("kine-mysql-host", fmt.Sprintf("%s_KINE_MYSQL_HOST", envPrefix)); err != nil {
+		return nil, err
+	}
+	if err := config.BindEnv("kine-mysql-port", fmt.Sprintf("%s_KINE_MYSQL_PORT", envPrefix)); err != nil {
+		return nil, err
+	}
+	if err := config.BindEnv("kine-image", fmt.Sprintf("%s_KINE_IMAGE", envPrefix)); err != nil {
+		return nil, err
+	}
 
 	// Setup config file
 	if cfgFile != "" {

internal/crypto/crypto.go

@@ -13,6 +13,24 @@ import (
 	"time"
 )
 
+const (
+	certBitSize = 2048
+)
+
+func GetCertificateAndKeyPair(template *x509.Certificate, caCert []byte, caPrivKey []byte) (*bytes.Buffer, *bytes.Buffer, error) {
+	caCertBytes, err := GetCertificate(caCert)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	caPrivKeyBytes, err := GetPrivateKey(caPrivKey)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return GenerateCertificateKeyPairBytes(template, certBitSize, caCertBytes, caPrivKeyBytes)
+}
+
 func GetCertificate(cert []byte) (*x509.Certificate, error) {
 	pemContent, _ := pem.Decode(cert)
 	if pemContent == nil {

internal/errors/errors.go

@@ -0,0 +1,16 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package errors
+
+type NonExposedLoadBalancerError struct{}
+
+func (n NonExposedLoadBalancerError) Error() string {
+	return "cannot retrieve the TenantControlPlane address, Service resource is not yet exposed as LoadBalancer"
+}
+
+type MissingValidIPError struct{}
+
+func (m MissingValidIPError) Error() string {
+	return "the actual resource doesn't have yet a valid IP address"
+}

internal/errors/utils_controllers.go

@@ -0,0 +1,10 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package errors
+
+import "github.com/pkg/errors"
+
+func ShouldReconcileErrorBeIgnored(err error) bool {
+	return errors.As(err, &NonExposedLoadBalancerError{}) || errors.As(err, &MissingValidIPError{})
+}

internal/etcd/certs.go

@@ -17,17 +17,7 @@ import (
 func GetETCDCACertificateAndKeyPair(tenant string, caCert []byte, caPrivKey []byte) (*bytes.Buffer, *bytes.Buffer, error) {
 	template := getCertTemplate(tenant)
 
-	caCertBytes, err := crypto.GetCertificate(caCert)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	caPrivKeyBytes, err := crypto.GetPrivateKey(caPrivKey)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	return crypto.GenerateCertificateKeyPairBytes(template, certBitSize, caCertBytes, caPrivKeyBytes)
+	return crypto.GetCertificateAndKeyPair(template, caCert, caPrivKey)
 }
 
 func IsETCDCertificateAndKeyPairValid(cert []byte, privKey []byte) (bool, error) {

internal/etcd/constant.go

@@ -6,5 +6,4 @@ package etcd
 const (
 	certExpirationDelayYears = 10
 	certOrganization         = "system:masters"
-	certBitSize              = 2048
 )

internal/kubeadm/addon.go

@@ -4,12 +4,14 @@
 package kubeadm
 
 import (
+	"context"
 	"fmt"
 	"time"
 
 	"github.com/pkg/errors"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 	clientcmdapi "k8s.io/client-go/tools/clientcmd/api/v1"
@@ -20,13 +22,88 @@ import (
 	"k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/proxy"
 	"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
 	"k8s.io/utils/pointer"
+
+	"github.com/clastix/kamaji/internal/utilities"
 )
 
-func CoreDNSAddon(client kubernetes.Interface, config *Configuration) error {
+const (
+	kubeSystemNamespace = "kube-system"
+	kubeProxyName       = "kube-proxy"
+	coreDNSName         = "coredns"
+	kubeDNSName         = "kube-dns"
+)
+
+func AddCoreDNS(client kubernetes.Interface, config *Configuration) error {
 	return dns.EnsureDNSAddon(&config.InitConfiguration.ClusterConfiguration, client)
 }
 
-func KubeProxyAddon(client kubernetes.Interface, config *Configuration) error {
+func RemoveCoreDNSAddon(ctx context.Context, client kubernetes.Interface) error {
+	var result error
+
+	if err := removeCoreDNSService(ctx, client); err != nil {
+		if !k8serrors.IsNotFound(err) {
+			return err
+		}
+		result = err
+	}
+
+	if err := removeCoreDNSDeployment(ctx, client); err != nil {
+		if !k8serrors.IsNotFound(err) {
+			return err
+		}
+		result = err
+	}
+
+	if err := removeCoreDNSConfigMap(ctx, client); err != nil {
+		if !k8serrors.IsNotFound(err) {
+			return err
+		}
+		result = err
+	}
+
+	return result
+}
+
+func removeCoreDNSService(ctx context.Context, client kubernetes.Interface) error {
+	name, _ := getCoreDNSServiceName(ctx)
+	opts := metav1.DeleteOptions{}
+
+	return client.CoreV1().Services(kubeSystemNamespace).Delete(ctx, name, opts)
+}
+
+func removeCoreDNSDeployment(ctx context.Context, client kubernetes.Interface) error {
+	name, _ := getCoreDNSDeploymentName(ctx)
+	opts := metav1.DeleteOptions{}
+
+	return client.AppsV1().Deployments(kubeSystemNamespace).Delete(ctx, name, opts)
+}
+
+func removeCoreDNSConfigMap(ctx context.Context, client kubernetes.Interface) error {
+	name, _ := getCoreDNSConfigMapName(ctx)
+	opts := metav1.DeleteOptions{}
+
+	return client.CoreV1().ConfigMaps(kubeSystemNamespace).Delete(ctx, name, opts)
+}
+
+func getCoreDNSServiceName(ctx context.Context) (string, error) {
+	// TODO: Currently, DNS is installed using kubeadm phases, therefore we know the name.
+	// Implement a method for future approaches
+	return kubeDNSName, nil
+}
+
+func getCoreDNSDeploymentName(ctx context.Context) (string, error) {
+	// TODO: Currently, DNS is installed using kubeadm phases, therefore we know the name.
+	// Implement a method for future approaches
+	return coreDNSName, nil
+}
+
+func getCoreDNSConfigMapName(ctx context.Context) (string, error) {
+	// TODO: Currently, DNS is installed using kubeadm phases, therefore we know the name.
+	// Implement a method for future approaches
+	return coreDNSName, nil
+}
+
+func AddKubeProxy(client kubernetes.Interface, config *Configuration) error {
 	if err := proxy.CreateServiceAccount(client); err != nil {
 		return errors.Wrap(err, "error when creating kube-proxy service account")
 	}
@@ -46,6 +123,95 @@ func KubeProxyAddon(client kubernetes.Interface, config *Configuration) error {
 	return nil
 }
 
+func RemoveKubeProxy(ctx context.Context, client kubernetes.Interface) error {
+	var result error
+
+	if err := removeKubeProxyDaemonSet(ctx, client); err != nil {
+		if !k8serrors.IsNotFound(err) {
+			return err
+		}
+		result = err
+	}
+
+	if err := removeKubeProxyConfigMap(ctx, client); err != nil {
+		if !k8serrors.IsNotFound(err) {
+			return err
+		}
+		result = err
+	}
+
+	if err := removeKubeProxyRBAC(ctx, client); err != nil {
+		if !k8serrors.IsNotFound(err) {
+			return err
+		}
+		result = err
+	}
+
+	return result
+}
+
+func removeKubeProxyDaemonSet(ctx context.Context, client kubernetes.Interface) error {
+	name, _ := getKubeProxyDaemonSetName(ctx)
+	opts := metav1.DeleteOptions{}
+
+	return client.AppsV1().DaemonSets(kubeSystemNamespace).Delete(ctx, name, opts)
+}
+
+func removeKubeProxyConfigMap(ctx context.Context, client kubernetes.Interface) error {
+	name, _ := getKubeProxyConfigMapName(ctx)
+	opts := metav1.DeleteOptions{}
+
+	return client.CoreV1().ConfigMaps(kubeSystemNamespace).Delete(ctx, name, opts)
+}
+
+func removeKubeProxyRBAC(ctx context.Context, client kubernetes.Interface) error {
+	// TODO: Currently, kube-proxy is installed using kubeadm phases, therefore, name is the same.
+	name, _ := getKubeProxyRBACName(ctx)
+	opts := metav1.DeleteOptions{}
+	var result error
+
+	if err := client.RbacV1().RoleBindings(kubeSystemNamespace).Delete(ctx, name, opts); err != nil {
+		if !k8serrors.IsNotFound(err) {
+			return err
+		}
+		result = err
+	}
+
+	if err := client.RbacV1().Roles(kubeSystemNamespace).Delete(ctx, name, opts); err != nil {
+		if !k8serrors.IsNotFound(err) {
+			return err
+		}
+		result = err
+	}
+
+	if err := client.CoreV1().ServiceAccounts(kubeSystemNamespace).Delete(ctx, name, opts); err != nil {
+		if !k8serrors.IsNotFound(err) {
+			return err
+		}
+		result = err
+	}
+
+	return result
+}
+
+func getKubeProxyRBACName(ctx context.Context) (string, error) {
+	// TODO: Currently, kube-proxy is installed using kubeadm phases, therefore we know the name.
+	// Implement a method for future approaches
+	return kubeProxyName, nil
+}
+
+func getKubeProxyDaemonSetName(ctx context.Context) (string, error) {
+	// TODO: Currently, kube-proxy is installed using kubeadm phases, therefore we know the name.
+	// Implement a method for future approaches
+	return kubeProxyName, nil
+}
+
+func getKubeProxyConfigMapName(ctx context.Context) (string, error) {
+	// TODO: Currently, kube-proxy is installed using kubeadm phases, therefore we know the name.
+	// Implement a method for future approaches
+	return kubeProxyName, nil
+}
+
 func createKubeProxyConfigMap(client kubernetes.Interface, config *Configuration) error {
 	configConf, err := getKubeproxyConfigmapContent(config)
 	if err != nil {
@@ -248,7 +414,7 @@ func getKubeproxyConfigmapContent(config *Configuration) ([]byte, error) {
 		},
 	}
 
-	return EncondeToYaml(&kubeProxyConfiguration)
+	return utilities.EncondeToYaml(&kubeProxyConfiguration)
 }
 
 func getKubeproxyKubeconfigContent(config *Configuration) ([]byte, error) {
@@ -283,5 +449,5 @@ func getKubeproxyKubeconfigContent(config *Configuration) ([]byte, error) {
 		},
 	}
 
-	return EncondeToYaml(&kubeconfig)
+	return utilities.EncondeToYaml(&kubeconfig)
 }

internal/kubeadm/configuration.go

@@ -6,6 +6,7 @@ package kubeadm
 import (
 	"encoding/json"
 	"fmt"
+	"strings"
 	"time"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -14,6 +15,12 @@ import (
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 )
 
+const (
+	defaultCAFile   = "/etc/kubernetes/pki/etcd/ca.crt"
+	defaultCertFile = "/etc/kubernetes/pki/apiserver-etcd-client.crt"
+	defaultKeyFile  = "/etc/kubernetes/pki/apiserver-etcd-client.key"
+)
+
 func CreateKubeadmInitConfiguration(params Parameters) Configuration {
 	config := kubeadmapi.InitConfiguration{
 		ClusterConfiguration: getKubeadmClusterConfiguration(params),
@@ -40,7 +47,16 @@ func CreateKubeadmInitConfiguration(params Parameters) Configuration {
 	return Configuration{InitConfiguration: config}
 }
 
+func isHTTPS(url string) bool {
+	return strings.HasPrefix(url, "https")
+}
+
 func getKubeadmClusterConfiguration(params Parameters) kubeadmapi.ClusterConfiguration {
+	caFile, certFile, keyFile := "", "", ""
+	if isHTTPS(params.ETCDs[0]) {
+		caFile, certFile, keyFile = defaultCAFile, defaultCertFile, defaultKeyFile
+	}
+
 	return kubeadmapi.ClusterConfiguration{
 		KubernetesVersion: params.TenantControlPlaneVersion,
 		ClusterName:       params.TenantControlPlaneName,
@@ -57,22 +73,21 @@ func getKubeadmClusterConfiguration(params Parameters) kubeadmapi.ClusterConfigu
 		ControlPlaneEndpoint: params.TenantControlPlaneEndpoint,
 		Etcd: kubeadmapi.Etcd{
 			External: &kubeadmapi.ExternalEtcd{
-				Endpoints: formatETCDEndpoints(params.ETCDs),
-				CAFile:    "/etc/kubernetes/pki/etcd/ca.crt",
-				CertFile:  "/etc/kubernetes/pki/apiserver-etcd-client.crt",
-				KeyFile:   "/etc/kubernetes/pki/apiserver-etcd-client.key",
+				Endpoints: params.ETCDs,
+				CAFile:    caFile,
+				CertFile:  certFile,
+				KeyFile:   keyFile,
 			},
 		},
 		APIServer: kubeadmapi.APIServer{
-			CertSANs: []string{
+			CertSANs: append([]string{
 				"127.0.0.1",
 				"localhost",
-				fmt.Sprintf("%s.%s", params.TenantControlPlaneName, params.TenantControlPlaneDomain),
 				params.TenantControlPlaneName,
 				fmt.Sprintf("%s.%s.svc", params.TenantControlPlaneName, params.TenantControlPlaneNamespace),
 				fmt.Sprintf("%s.%s.svc.cluster.local", params.TenantControlPlaneName, params.TenantControlPlaneNamespace),
 				params.TenantControlPlaneAddress,
-			},
+			}, params.TenantControlPlaneCertSANs...),
 			ControlPlaneComponent: kubeadmapi.ControlPlaneComponent{
 				ExtraArgs: map[string]string{
 					"etcd-compaction-interval": params.ETCDCompactionInterval,
@@ -131,12 +146,3 @@ func getJSONStringFromStruct(i interface{}) (string, error) {
 
 	return string(b), nil
 }
-
-func formatETCDEndpoints(etcds []string) []string {
-	formatedETCDs := make([]string, 0, len(etcds))
-	for _, etcd := range etcds {
-		formatedETCDs = append(formatedETCDs, fmt.Sprintf("https://%s/", etcd))
-	}
-
-	return formatedETCDs
-}

internal/kubeadm/types.go

@@ -4,9 +4,11 @@
 package kubeadm
 
 import (
+	json "github.com/json-iterator/go"
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
 
 	kubeconfigutil "github.com/clastix/kamaji/internal/kubeconfig"
+	"github.com/clastix/kamaji/internal/utilities"
 )
 
 type Configuration struct {
@@ -15,13 +17,27 @@ type Configuration struct {
 	Parameters        Parameters
 }
 
+func (c *Configuration) Checksum() string {
+	initConfiguration, _ := utilities.EncondeToYaml(&c.InitConfiguration)
+	kubeconfig, _ := json.Marshal(c.Kubeconfig)
+	parameters, _ := json.Marshal(c.Parameters)
+
+	data := map[string]string{
+		"InitConfiguration": string(initConfiguration),
+		"Kubeconfig":        string(kubeconfig),
+		"Parameters":        string(parameters),
+	}
+
+	return utilities.CalculateConfigMapChecksum(data)
+}
+
 type Parameters struct {
 	TenantControlPlaneName         string
 	TenantControlPlaneNamespace    string
 	TenantControlPlaneEndpoint     string
 	TenantControlPlaneAddress      string
+	TenantControlPlaneCertSANs     []string
 	TenantControlPlanePort         int32
-	TenantControlPlaneDomain       string
 	TenantControlPlanePodCIDR      string
 	TenantControlPlaneServiceCIDR  string
 	TenantDNSServiceIPs            []string

internal/kubeadm/uploadconfig.go

@@ -18,6 +18,8 @@ import (
 	"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
 	"k8s.io/kubernetes/pkg/apis/rbac"
 	"k8s.io/utils/pointer"
+
+	"github.com/clastix/kamaji/internal/utilities"
 )
 
 func UploadKubeadmConfig(client kubernetes.Interface, config *Configuration) error {
@@ -115,7 +117,7 @@ func getKubeletConfigmapContent(kubeletConfiguration KubeletConfiguration) ([]by
 		VolumeStatsAggPeriod:             zeroDuration,
 	}
 
-	return EncondeToYaml(&kc)
+	return utilities.EncondeToYaml(&kc)
 }
 
 func createConfigMapRBACRules(client kubernetes.Interface, kubernetesVersion string) error {

internal/kubeadm/utils.go

@@ -1,20 +0,0 @@
-// Copyright 2022 Clastix Labs
-// SPDX-License-Identifier: Apache-2.0
-
-package kubeadm
-
-import (
-	"bytes"
-
-	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/apimachinery/pkg/runtime/serializer/json"
-)
-
-func EncondeToYaml(o runtime.Object) ([]byte, error) {
-	scheme := runtime.NewScheme()
-	encoder := json.NewYAMLSerializer(json.SimpleMetaFactory{}, scheme, scheme)
-	buf := bytes.NewBuffer([]byte{})
-	err := encoder.Encode(o, buf)
-
-	return buf.Bytes(), err
-}

internal/resources/api_server_certificate.go

@@ -37,7 +37,7 @@ func (r *APIServerCertificate) ShouldCleanup(plane *kamajiv1alpha1.TenantControl
 	return false
 }
 
-func (r *APIServerCertificate) CleanUp(ctx context.Context) (bool, error) {
+func (r *APIServerCertificate) CleanUp(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) {
 	return false, nil
 }
 
@@ -64,8 +64,8 @@ func (r *APIServerCertificate) GetTmpDirectory() string {
 	return r.TmpDirectory
 }
 
-func (r *APIServerCertificate) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) {
-	return controllerutil.CreateOrUpdate(ctx, r.Client, r.resource, r.mutate(ctx, tenantControlPlane))
+func (r *APIServerCertificate) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (res controllerutil.OperationResult, err error) {
+	return utilities.CreateOrUpdateWithConflict(ctx, r.Client, r.resource, r.mutate(ctx, tenantControlPlane))
 }
 
 func (r *APIServerCertificate) GetName() string {
@@ -96,7 +96,7 @@ func (r *APIServerCertificate) mutate(ctx context.Context, tenantControlPlane *k
 			}
 		}
 
-		config, _, err := getKubeadmConfiguration(ctx, r, tenantControlPlane)
+		config, err := getStoredKubeadmConfiguration(ctx, r, tenantControlPlane)
 		if err != nil {
 			return err
 		}

internal/resources/api_server_kubelet_client_certificate.go

@@ -37,7 +37,7 @@ func (r *APIServerKubeletClientCertificate) ShouldCleanup(plane *kamajiv1alpha1.
 	return false
 }
 
-func (r *APIServerKubeletClientCertificate) CleanUp(ctx context.Context) (bool, error) {
+func (r *APIServerKubeletClientCertificate) CleanUp(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) {
 	return false, nil
 }
 
@@ -64,8 +64,8 @@ func (r *APIServerKubeletClientCertificate) GetTmpDirectory() string {
 	return r.TmpDirectory
 }
 
-func (r *APIServerKubeletClientCertificate) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) {
-	return controllerutil.CreateOrUpdate(ctx, r.Client, r.resource, r.mutate(ctx, tenantControlPlane))
+func (r *APIServerKubeletClientCertificate) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (res controllerutil.OperationResult, err error) {
+	return utilities.CreateOrUpdateWithConflict(ctx, r.Client, r.resource, r.mutate(ctx, tenantControlPlane))
 }
 
 func (r *APIServerKubeletClientCertificate) GetName() string {
@@ -96,7 +96,7 @@ func (r *APIServerKubeletClientCertificate) mutate(ctx context.Context, tenantCo
 			}
 		}
 
-		config, _, err := getKubeadmConfiguration(ctx, r, tenantControlPlane)
+		config, err := getStoredKubeadmConfiguration(ctx, r, tenantControlPlane)
 		if err != nil {
 			return err
 		}

internal/resources/ca_certificate.go

@@ -29,14 +29,15 @@ type CACertificate struct {
 }
 
 func (r *CACertificate) ShouldStatusBeUpdated(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
-	return tenantControlPlane.Status.Certificates.CA.SecretName != r.resource.GetName()
+	return tenantControlPlane.Status.Certificates.CA.SecretName != r.resource.GetName() ||
+		tenantControlPlane.Status.Certificates.CA.ResourceVersion != r.resource.ResourceVersion
 }
 
 func (r *CACertificate) ShouldCleanup(plane *kamajiv1alpha1.TenantControlPlane) bool {
 	return false
 }
 
-func (r *CACertificate) CleanUp(ctx context.Context) (bool, error) {
+func (r *CACertificate) CleanUp(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) {
 	return false, nil
 }
 
@@ -64,7 +65,7 @@ func (r *CACertificate) GetTmpDirectory() string {
 }
 
 func (r *CACertificate) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) {
-	return controllerutil.CreateOrUpdate(ctx, r.Client, r.resource, r.mutate(ctx, tenantControlPlane))
+	return utilities.CreateOrUpdateWithConflict(ctx, r.Client, r.resource, r.mutate(ctx, tenantControlPlane))
 }
 
 func (r *CACertificate) GetName() string {
@@ -74,6 +75,7 @@ func (r *CACertificate) GetName() string {
 func (r *CACertificate) UpdateTenantControlPlaneStatus(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
 	tenantControlPlane.Status.Certificates.CA.LastUpdate = metav1.Now()
 	tenantControlPlane.Status.Certificates.CA.SecretName = r.resource.GetName()
+	tenantControlPlane.Status.Certificates.CA.ResourceVersion = r.resource.ResourceVersion
 
 	return nil
 }
@@ -91,7 +93,7 @@ func (r *CACertificate) mutate(ctx context.Context, tenantControlPlane *kamajiv1
 			return nil
 		}
 
-		config, _, err := getKubeadmConfiguration(ctx, r, tenantControlPlane)
+		config, err := getStoredKubeadmConfiguration(ctx, r, tenantControlPlane)
 		if err != nil {
 			return err
 		}

internal/resources/constants.go

@@ -1,8 +0,0 @@
-// Copyright 2022 Clastix Labs
-// SPDX-License-Identifier: Apache-2.0
-
-package resources
-
-const (
-	defaultIngressPort = 443
-)

internal/resources/etcd_ca_certificates.go

@@ -43,7 +43,7 @@ func (r *ETCDCACertificatesResource) ShouldCleanup(plane *kamajiv1alpha1.TenantC
 	return false
 }
 
-func (r *ETCDCACertificatesResource) CleanUp(ctx context.Context) (bool, error) {
+func (r *ETCDCACertificatesResource) CleanUp(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) {
 	return false, nil
 }
 
@@ -59,7 +59,7 @@ func (r *ETCDCACertificatesResource) Define(ctx context.Context, tenantControlPl
 }
 
 func (r *ETCDCACertificatesResource) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) {
-	return controllerutil.CreateOrUpdate(ctx, r.Client, r.resource, r.mutate(ctx, tenantControlPlane))
+	return utilities.CreateOrUpdateWithConflict(ctx, r.Client, r.resource, r.mutate(ctx, tenantControlPlane))
 }
 
 func (r *ETCDCACertificatesResource) GetName() string {

internal/resources/etcd_certificates.go

@@ -40,7 +40,7 @@ func (r *ETCDCertificatesResource) ShouldCleanup(plane *kamajiv1alpha1.TenantCon
 	return false
 }
 
-func (r *ETCDCertificatesResource) CleanUp(ctx context.Context) (bool, error) {
+func (r *ETCDCertificatesResource) CleanUp(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) {
 	return false, nil
 }
 
@@ -56,7 +56,7 @@ func (r *ETCDCertificatesResource) Define(ctx context.Context, tenantControlPlan
 }
 
 func (r *ETCDCertificatesResource) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) {
-	return controllerutil.CreateOrUpdate(ctx, r.Client, r.resource, r.mutate(ctx, tenantControlPlane))
+	return utilities.CreateOrUpdateWithConflict(ctx, r.Client, r.resource, r.mutate(ctx, tenantControlPlane))
 }
 
 func (r *ETCDCertificatesResource) GetName() string {

internal/resources/etcd_setup.go

@@ -9,7 +9,6 @@ import (
 	"github.com/go-logr/logr"
 	etcdclient "go.etcd.io/etcd/client/v3"
 	corev1 "k8s.io/api/core/v1"
-	"k8s.io/apimachinery/pkg/runtime"
 	k8stypes "k8s.io/apimachinery/pkg/types"
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -23,15 +22,14 @@ const (
 	caKeyName = kubeadmconstants.CACertName
 )
 
-type resource struct {
+type etcdSetupResource struct {
 	role etcd.Role
 	user etcd.User
 }
 
 type ETCDSetupResource struct {
-	resource              *resource
+	resource              *etcdSetupResource
 	Client                client.Client
-	Scheme                *runtime.Scheme
 	Log                   logr.Logger
 	Name                  string
 	Endpoints             []string
@@ -52,12 +50,12 @@ func (r *ETCDSetupResource) ShouldCleanup(plane *kamajiv1alpha1.TenantControlPla
 	return false
 }
 
-func (r *ETCDSetupResource) CleanUp(ctx context.Context) (bool, error) {
+func (r *ETCDSetupResource) CleanUp(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) {
 	return false, nil
 }
 
 func (r *ETCDSetupResource) Define(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
-	r.resource = &resource{
+	r.resource = &etcdSetupResource{
 		role: etcd.Role{Name: tenantControlPlane.Name, Exists: false},
 		user: etcd.User{Name: tenantControlPlane.Name, Exists: false},
 	}
@@ -116,6 +114,7 @@ func (r *ETCDSetupResource) reconcile(ctx context.Context) (controllerutil.Opera
 	if err != nil {
 		return reconcilationResult, err
 	}
+	defer client.Close()
 
 	operationResult, err = r.reconcileUser(ctx, client)
 	if err != nil {

internal/resources/front-proxy-client-certificate.go

@@ -37,7 +37,7 @@ func (r *FrontProxyClientCertificate) ShouldCleanup(plane *kamajiv1alpha1.Tenant
 	return false
 }
 
-func (r *FrontProxyClientCertificate) CleanUp(ctx context.Context) (bool, error) {
+func (r *FrontProxyClientCertificate) CleanUp(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) {
 	return false, nil
 }
 
@@ -65,7 +65,7 @@ func (r *FrontProxyClientCertificate) GetTmpDirectory() string {
 }
 
 func (r *FrontProxyClientCertificate) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) {
-	return controllerutil.CreateOrUpdate(ctx, r.Client, r.resource, r.mutate(ctx, tenantControlPlane))
+	return utilities.CreateOrUpdateWithConflict(ctx, r.Client, r.resource, r.mutate(ctx, tenantControlPlane))
 }
 
 func (r *FrontProxyClientCertificate) GetName() string {
@@ -96,7 +96,7 @@ func (r *FrontProxyClientCertificate) mutate(ctx context.Context, tenantControlP
 			}
 		}
 
-		config, _, err := getKubeadmConfiguration(ctx, r, tenantControlPlane)
+		config, err := getStoredKubeadmConfiguration(ctx, r, tenantControlPlane)
 		if err != nil {
 			return err
 		}

internal/resources/front_proxy_ca_certificate.go

@@ -36,7 +36,7 @@ func (r *FrontProxyCACertificate) ShouldCleanup(plane *kamajiv1alpha1.TenantCont
 	return false
 }
 
-func (r *FrontProxyCACertificate) CleanUp(ctx context.Context) (bool, error) {
+func (r *FrontProxyCACertificate) CleanUp(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) {
 	return false, nil
 }
 
@@ -66,7 +66,7 @@ func (r *FrontProxyCACertificate) GetTmpDirectory() string {
 }
 
 func (r *FrontProxyCACertificate) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) {
-	return controllerutil.CreateOrUpdate(ctx, r.Client, r.resource, r.mutate(ctx, tenantControlPlane))
+	return utilities.CreateOrUpdateWithConflict(ctx, r.Client, r.resource, r.mutate(ctx, tenantControlPlane))
 }
 
 func (r *FrontProxyCACertificate) GetName() string {
@@ -93,7 +93,7 @@ func (r *FrontProxyCACertificate) mutate(ctx context.Context, tenantControlPlane
 			return nil
 		}
 
-		config, _, err := getKubeadmConfiguration(ctx, r, tenantControlPlane)
+		config, err := getStoredKubeadmConfiguration(ctx, r, tenantControlPlane)
 		if err != nil {
 			return err
 		}

internal/resources/k8s_deployment_resource.go

@@ -5,59 +5,50 @@ package resources
 
 import (
 	"context"
-	"crypto/md5"
-	"fmt"
-	"path"
-	"sort"
-	"strings"
 
 	"github.com/pkg/errors"
 	appsv1 "k8s.io/api/apps/v1"
-	corev1 "k8s.io/api/core/v1"
-	quantity "k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/types"
-	"k8s.io/apimachinery/pkg/util/intstr"
-	"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3"
-	"k8s.io/kubernetes/cmd/kubeadm/app/constants"
-	"k8s.io/utils/pointer"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 
 	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+	builder "github.com/clastix/kamaji/internal/builders/controlplane"
+	"github.com/clastix/kamaji/internal/types"
 	"github.com/clastix/kamaji/internal/utilities"
 )
 
 type KubernetesDeploymentResource struct {
 	resource               *appsv1.Deployment
 	Client                 client.Client
+	ETCDStorageType        types.ETCDStorageType
 	ETCDEndpoints          []string
 	ETCDCompactionInterval string
 	Name                   string
+	KineContainerImage     string
 }
 
 func (r *KubernetesDeploymentResource) isStatusEqual(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
 	return r.resource.Status.String() == tenantControlPlane.Status.Kubernetes.Deployment.DeploymentStatus.String()
 }
 
-func (r *KubernetesDeploymentResource) ShouldStatusBeUpdated(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
+func (r *KubernetesDeploymentResource) ShouldStatusBeUpdated(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
 	return !r.isStatusEqual(tenantControlPlane) || tenantControlPlane.Spec.Kubernetes.Version != tenantControlPlane.Status.Kubernetes.Version.Version
 }
 
-func (r *KubernetesDeploymentResource) ShouldCleanup(plane *kamajiv1alpha1.TenantControlPlane) bool {
+func (r *KubernetesDeploymentResource) ShouldCleanup(*kamajiv1alpha1.TenantControlPlane) bool {
 	return false
 }
 
-func (r *KubernetesDeploymentResource) CleanUp(ctx context.Context) (bool, error) {
+func (r *KubernetesDeploymentResource) CleanUp(context.Context, *kamajiv1alpha1.TenantControlPlane) (bool, error) {
 	return false, nil
 }
 
-func (r *KubernetesDeploymentResource) Define(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+func (r *KubernetesDeploymentResource) Define(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
 	r.resource = &appsv1.Deployment{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      tenantControlPlane.GetName(),
 			Namespace: tenantControlPlane.GetNamespace(),
-			Labels:    utilities.CommonLabels(tenantControlPlane.GetName()),
 		},
 	}
 
@@ -66,497 +57,43 @@ func (r *KubernetesDeploymentResource) Define(ctx context.Context, tenantControl
 	return nil
 }
 
-// secretHashValue function returns the md5 value for the given secret:
-// this will trigger a new rollout in case of value change.
-func (r *KubernetesDeploymentResource) secretHashValue(ctx context.Context, namespace, name string) (string, error) {
-	secret := &corev1.Secret{}
+func (r *KubernetesDeploymentResource) mutate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) controllerutil.MutateFn {
+	return func() error {
+		address, _, err := tenantControlPlane.AssignedControlPlaneAddress()
+		if err != nil {
+			return errors.Wrap(err, "cannot create TenantControlPlane Deployment")
+		}
 
-	if err := r.Client.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, secret); err != nil {
-		return "", errors.Wrap(err, "cannot retrieve *corev1.Secret for resource version retrieval")
+		d := builder.Deployment{
+			Address:                address,
+			ETCDEndpoints:          r.ETCDEndpoints,
+			ETCDCompactionInterval: r.ETCDCompactionInterval,
+			ETCDStorageType:        r.ETCDStorageType,
+			KineContainerImage:     r.KineContainerImage,
+		}
+		d.SetLabels(r.resource, utilities.MergeMaps(utilities.CommonLabels(tenantControlPlane.GetName()), tenantControlPlane.Spec.ControlPlane.Deployment.AdditionalMetadata.Labels))
+		d.SetAnnotations(r.resource, utilities.MergeMaps(r.resource.Annotations, tenantControlPlane.Spec.ControlPlane.Deployment.AdditionalMetadata.Annotations))
+		d.SetTemplateLabels(&r.resource.Spec.Template, r.deploymentTemplateLabels(ctx, tenantControlPlane))
+		d.SetStrategy(&r.resource.Spec)
+		d.SetSelector(&r.resource.Spec, tenantControlPlane)
+		d.SetReplicas(&r.resource.Spec, tenantControlPlane)
+		d.ResetKubeAPIServerFlags(r.resource, tenantControlPlane)
+		d.SetContainers(&r.resource.Spec.Template.Spec, tenantControlPlane, address)
+		d.SetVolumes(&r.resource.Spec.Template.Spec, tenantControlPlane)
+
+		return controllerutil.SetControllerReference(tenantControlPlane, r.resource, r.Client.Scheme())
 	}
-	// Go access map values in random way, it means we have to sort them
-	keys := make([]string, 0, len(secret.Data))
-
-	for k := range secret.Data {
-		keys = append(keys, k)
-	}
-
-	sort.Strings(keys)
-	// Generating MD5 of Secret values, sorted by key
-	h := md5.New()
-
-	for _, key := range keys {
-		h.Write(secret.Data[key])
-	}
-
-	return fmt.Sprintf("%x", h.Sum(nil)), nil
 }
 
 func (r *KubernetesDeploymentResource) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) {
-	maxSurge := intstr.FromString("100%")
-
-	maxUnavailable := intstr.FromInt(0)
-
-	etcdEndpoints := make([]string, len(r.ETCDEndpoints))
-	for i, v := range r.ETCDEndpoints {
-		etcdEndpoints[i] = fmt.Sprintf("https://%s", v)
-	}
-
-	address, err := tenantControlPlane.GetAddress(ctx, r.Client)
-	if err != nil {
-		return controllerutil.OperationResultNone, errors.Wrap(err, "cannot create TenantControlPlane Deployment")
-	}
-
-	return controllerutil.CreateOrUpdate(ctx, r.Client, r.resource, func() error {
-		labels := utilities.MergeMaps(r.resource.GetLabels(), tenantControlPlane.Spec.ControlPlane.Deployment.AdditionalMetadata.Labels)
-		r.resource.SetLabels(labels)
-
-		annotations := utilities.MergeMaps(r.resource.GetAnnotations(), tenantControlPlane.Spec.ControlPlane.Deployment.AdditionalMetadata.Annotations)
-		r.resource.SetAnnotations(annotations)
-
-		r.resource.Spec.Replicas = pointer.Int32(tenantControlPlane.Spec.ControlPlane.Deployment.Replicas)
-		r.resource.Spec.Selector = &metav1.LabelSelector{
-			MatchLabels: map[string]string{
-				"kamaji.clastix.io/soot": tenantControlPlane.GetName(),
-			},
-		}
-		r.resource.Spec.Template.ObjectMeta = metav1.ObjectMeta{
-			Labels: map[string]string{
-				"kamaji.clastix.io/soot": tenantControlPlane.GetName(),
-				"component.kamaji.clastix.io/api-server-certificate": func() (hash string) {
-					hash, _ = r.secretHashValue(ctx, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.Certificates.APIServer.SecretName)
-
-					return
-				}(),
-				"component.kamaji.clastix.io/api-server-kubelet-client-certificate": func() (hash string) {
-					hash, _ = r.secretHashValue(ctx, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.Certificates.APIServerKubeletClient.SecretName)
-
-					return
-				}(),
-				"component.kamaji.clastix.io/ca": func() (hash string) {
-					hash, _ = r.secretHashValue(ctx, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.Certificates.CA.SecretName)
-
-					return
-				}(),
-				"component.kamaji.clastix.io/controller-manager-kubeconfig": func() (hash string) {
-					hash, _ = r.secretHashValue(ctx, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.KubeConfig.ControllerManager.SecretName)
-
-					return
-				}(),
-				"component.kamaji.clastix.io/etcd-ca-certificates": func() (hash string) {
-					hash, _ = r.secretHashValue(ctx, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.Certificates.ETCD.CA.SecretName)
-
-					return
-				}(),
-				"component.kamaji.clastix.io/etcd-certificates": func() (hash string) {
-					hash, _ = r.secretHashValue(ctx, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.Certificates.ETCD.APIServer.SecretName)
-
-					return
-				}(),
-				"component.kamaji.clastix.io/front-proxy-ca-certificate": func() (hash string) {
-					hash, _ = r.secretHashValue(ctx, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.Certificates.FrontProxyCA.SecretName)
-
-					return
-				}(),
-				"component.kamaji.clastix.io/front-proxy-client-certificate": func() (hash string) {
-					hash, _ = r.secretHashValue(ctx, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.Certificates.FrontProxyClient.SecretName)
-
-					return
-				}(),
-				"component.kamaji.clastix.io/service-account": func() (hash string) {
-					hash, _ = r.secretHashValue(ctx, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.Certificates.SA.SecretName)
-
-					return
-				}(),
-				"component.kamaji.clastix.io/scheduler-kubeconfig": func() (hash string) {
-					hash, _ = r.secretHashValue(ctx, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.KubeConfig.Scheduler.SecretName)
-
-					return
-				}(),
-			},
-		}
-		r.resource.Spec.Template.Spec.Volumes = []corev1.Volume{
-			{
-				Name: "etc-kubernetes-pki",
-				VolumeSource: corev1.VolumeSource{
-					Projected: &corev1.ProjectedVolumeSource{
-						Sources: []corev1.VolumeProjection{
-							{
-								Secret: secretProjection(tenantControlPlane.Status.Certificates.APIServer.SecretName, constants.APIServerCertName, constants.APIServerKeyName),
-							},
-							{
-								Secret: secretProjection(tenantControlPlane.Status.Certificates.CA.SecretName, constants.CACertName, constants.CAKeyName),
-							},
-							{
-								Secret: secretProjection(tenantControlPlane.Status.Certificates.APIServerKubeletClient.SecretName, constants.APIServerKubeletClientCertName, constants.APIServerKubeletClientKeyName),
-							},
-							{
-								Secret: secretProjection(tenantControlPlane.Status.Certificates.FrontProxyCA.SecretName, constants.FrontProxyCACertName, constants.FrontProxyCAKeyName),
-							},
-							{
-								Secret: secretProjection(tenantControlPlane.Status.Certificates.FrontProxyClient.SecretName, constants.FrontProxyClientCertName, constants.FrontProxyClientKeyName),
-							},
-							{
-								Secret: secretProjection(tenantControlPlane.Status.Certificates.SA.SecretName, constants.ServiceAccountPublicKeyName, constants.ServiceAccountPrivateKeyName),
-							},
-							{
-								Secret: secretProjection(tenantControlPlane.Status.Certificates.ETCD.APIServer.SecretName, constants.APIServerEtcdClientCertName, constants.APIServerEtcdClientKeyName),
-							},
-							{
-								Secret: &corev1.SecretProjection{
-									LocalObjectReference: corev1.LocalObjectReference{
-										Name: tenantControlPlane.Status.Certificates.ETCD.CA.SecretName,
-									},
-									Items: []corev1.KeyToPath{
-										{
-											Key:  constants.CACertName,
-											Path: constants.EtcdCACertName,
-										},
-									},
-								},
-							},
-						},
-						DefaultMode: pointer.Int32Ptr(420),
-					},
-				},
-			},
-			{
-				Name: "etc-ca-certificates",
-				VolumeSource: corev1.VolumeSource{
-					Secret: &corev1.SecretVolumeSource{
-						SecretName:  tenantControlPlane.Status.Certificates.CA.SecretName,
-						DefaultMode: pointer.Int32Ptr(420),
-					},
-				},
-			},
-			{
-				Name: "etc-ssl-certs",
-				VolumeSource: corev1.VolumeSource{
-					Secret: &corev1.SecretVolumeSource{
-						SecretName:  tenantControlPlane.Status.Certificates.CA.SecretName,
-						DefaultMode: pointer.Int32Ptr(420),
-					},
-				},
-			},
-			{
-				Name: "usr-share-ca-certificates",
-				VolumeSource: corev1.VolumeSource{
-					Secret: &corev1.SecretVolumeSource{
-						SecretName:  tenantControlPlane.Status.Certificates.CA.SecretName,
-						DefaultMode: pointer.Int32Ptr(420),
-					},
-				},
-			},
-			{
-				Name: "usr-local-share-ca-certificates",
-				VolumeSource: corev1.VolumeSource{
-					Secret: &corev1.SecretVolumeSource{
-						SecretName:  tenantControlPlane.Status.Certificates.CA.SecretName,
-						DefaultMode: pointer.Int32Ptr(420),
-					},
-				},
-			},
-			{
-				Name: "scheduler-kubeconfig",
-				VolumeSource: corev1.VolumeSource{
-					Secret: &corev1.SecretVolumeSource{
-						SecretName:  tenantControlPlane.Status.KubeConfig.Scheduler.SecretName,
-						DefaultMode: pointer.Int32Ptr(420),
-					},
-				},
-			},
-			{
-				Name: "controller-manager-kubeconfig",
-				VolumeSource: corev1.VolumeSource{
-					Secret: &corev1.SecretVolumeSource{
-						SecretName:  tenantControlPlane.Status.KubeConfig.ControllerManager.SecretName,
-						DefaultMode: pointer.Int32Ptr(420),
-					},
-				},
-			},
-		}
-
-		r.resource.Spec.Template.Spec.Containers = []corev1.Container{
-			{
-				Name:  "kube-apiserver",
-				Image: fmt.Sprintf("k8s.gcr.io/kube-apiserver:%s", tenantControlPlane.Spec.Kubernetes.Version),
-				Command: []string{
-					"kube-apiserver",
-					"--allow-privileged=true",
-					"--authorization-mode=Node,RBAC",
-					fmt.Sprintf("--advertise-address=%s", address),
-					fmt.Sprintf("--client-ca-file=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.CACertName)),
-					fmt.Sprintf("--enable-admission-plugins=%s", strings.Join(tenantControlPlane.Spec.Kubernetes.AdmissionControllers.ToSlice(), ",")),
-					"--enable-bootstrap-token-auth=true",
-					fmt.Sprintf("--etcd-compaction-interval=%s", r.ETCDCompactionInterval),
-					fmt.Sprintf("--etcd-cafile=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.EtcdCACertName)),
-					fmt.Sprintf("--etcd-certfile=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.APIServerEtcdClientCertName)),
-					fmt.Sprintf("--etcd-keyfile=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.APIServerEtcdClientKeyName)),
-					fmt.Sprintf("--etcd-servers=%s", strings.Join(etcdEndpoints, ",")),
-					fmt.Sprintf("--etcd-prefix=/%s", tenantControlPlane.GetName()),
-					fmt.Sprintf("--service-cluster-ip-range=%s", tenantControlPlane.Spec.NetworkProfile.ServiceCIDR),
-					fmt.Sprintf("--kubelet-client-certificate=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.APIServerKubeletClientCertName)),
-					fmt.Sprintf("--kubelet-client-key=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.APIServerKubeletClientKeyName)),
-					"--kubelet-preferred-address-types=Hostname,InternalIP,ExternalIP",
-					fmt.Sprintf("--proxy-client-cert-file=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.FrontProxyClientCertName)),
-					fmt.Sprintf("--proxy-client-key-file=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.FrontProxyClientKeyName)),
-					"--requestheader-allowed-names=front-proxy-client",
-					"--requestheader-extra-headers-prefix=X-Remote-Extra-",
-					"--requestheader-group-headers=X-Remote-Group",
-					"--requestheader-username-headers=X-Remote-User",
-					fmt.Sprintf("--secure-port=%d", tenantControlPlane.Spec.NetworkProfile.Port),
-					fmt.Sprintf("--service-account-issuer=https://localhost:%d", tenantControlPlane.Spec.NetworkProfile.Port),
-					fmt.Sprintf("--service-account-key-file=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.ServiceAccountPublicKeyName)),
-					fmt.Sprintf("--service-account-signing-key-file=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.ServiceAccountPrivateKeyName)),
-					fmt.Sprintf("--tls-cert-file=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.APIServerCertName)),
-					fmt.Sprintf("--tls-private-key-file=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.APIServerKeyName)),
-				},
-				Resources: corev1.ResourceRequirements{
-					Limits: nil,
-					Requests: corev1.ResourceList{
-						corev1.ResourceCPU: quantity.MustParse("250m"),
-					},
-				},
-				LivenessProbe: &corev1.Probe{
-					ProbeHandler: corev1.ProbeHandler{
-						HTTPGet: &corev1.HTTPGetAction{
-							Path:   "/livez",
-							Port:   intstr.FromInt(int(tenantControlPlane.Spec.NetworkProfile.Port)),
-							Scheme: corev1.URISchemeHTTPS,
-						},
-					},
-					InitialDelaySeconds: 0,
-					TimeoutSeconds:      1,
-					PeriodSeconds:       10,
-					SuccessThreshold:    1,
-					FailureThreshold:    3,
-				},
-				ReadinessProbe: &corev1.Probe{
-					ProbeHandler: corev1.ProbeHandler{
-						HTTPGet: &corev1.HTTPGetAction{
-							Path:   "/readyz",
-							Port:   intstr.FromInt(int(tenantControlPlane.Spec.NetworkProfile.Port)),
-							Scheme: corev1.URISchemeHTTPS,
-						},
-					},
-					InitialDelaySeconds: 0,
-					TimeoutSeconds:      1,
-					PeriodSeconds:       10,
-					SuccessThreshold:    1,
-					FailureThreshold:    3,
-				},
-				StartupProbe: &corev1.Probe{
-					ProbeHandler: corev1.ProbeHandler{
-						HTTPGet: &corev1.HTTPGetAction{
-							Path:   "/livez",
-							Port:   intstr.FromInt(int(tenantControlPlane.Spec.NetworkProfile.Port)),
-							Scheme: corev1.URISchemeHTTPS,
-						},
-					},
-					InitialDelaySeconds: 0,
-					TimeoutSeconds:      1,
-					PeriodSeconds:       10,
-					SuccessThreshold:    1,
-					FailureThreshold:    3,
-				},
-				TerminationMessagePath:   "/dev/termination-log",
-				TerminationMessagePolicy: "File",
-				ImagePullPolicy:          corev1.PullAlways,
-				VolumeMounts: []corev1.VolumeMount{
-					{
-						Name:      "etc-kubernetes-pki",
-						ReadOnly:  true,
-						MountPath: v1beta3.DefaultCertificatesDir,
-					},
-					{
-						Name:      "etc-ca-certificates",
-						ReadOnly:  true,
-						MountPath: "/etc/ca-certificates",
-					},
-					{
-						Name:      "etc-ssl-certs",
-						ReadOnly:  true,
-						MountPath: "/etc/ssl/certs",
-					},
-					{
-						Name:      "usr-share-ca-certificates",
-						ReadOnly:  true,
-						MountPath: "/usr/share/ca-certificates",
-					},
-					{
-						Name:      "usr-local-share-ca-certificates",
-						ReadOnly:  true,
-						MountPath: "/usr/local/share/ca-certificates",
-					},
-				},
-			},
-			{
-				Name:  "kube-scheduler",
-				Image: fmt.Sprintf("k8s.gcr.io/kube-scheduler:%s", tenantControlPlane.Spec.Kubernetes.Version),
-				Command: []string{
-					"kube-scheduler",
-					"--authentication-kubeconfig=/etc/kubernetes/scheduler.conf",
-					"--authorization-kubeconfig=/etc/kubernetes/scheduler.conf",
-					"--bind-address=0.0.0.0",
-					"--kubeconfig=/etc/kubernetes/scheduler.conf",
-					"--leader-elect=true",
-				},
-				Resources: corev1.ResourceRequirements{
-					Limits: nil,
-					Requests: corev1.ResourceList{
-						corev1.ResourceCPU: quantity.MustParse("100m"),
-					},
-				},
-				VolumeMounts: []corev1.VolumeMount{
-					{
-						Name:      "scheduler-kubeconfig",
-						ReadOnly:  true,
-						MountPath: "/etc/kubernetes",
-					},
-				},
-				LivenessProbe: &corev1.Probe{
-					ProbeHandler: corev1.ProbeHandler{
-						HTTPGet: &corev1.HTTPGetAction{
-							Path:   "/healthz",
-							Port:   intstr.FromInt(10259),
-							Scheme: corev1.URISchemeHTTPS,
-						},
-					},
-					InitialDelaySeconds: 0,
-					TimeoutSeconds:      1,
-					PeriodSeconds:       10,
-					SuccessThreshold:    1,
-					FailureThreshold:    3,
-				},
-				StartupProbe: &corev1.Probe{
-					ProbeHandler: corev1.ProbeHandler{
-						HTTPGet: &corev1.HTTPGetAction{
-							Path:   "/healthz",
-							Port:   intstr.FromInt(10259),
-							Scheme: corev1.URISchemeHTTPS,
-						},
-					},
-					InitialDelaySeconds: 0,
-					TimeoutSeconds:      1,
-					PeriodSeconds:       10,
-					SuccessThreshold:    1,
-					FailureThreshold:    3,
-				},
-				TerminationMessagePath:   "/dev/termination-log",
-				TerminationMessagePolicy: "File",
-				ImagePullPolicy:          corev1.PullIfNotPresent,
-			},
-			{
-				Name:  "kube-controller-manager",
-				Image: fmt.Sprintf("k8s.gcr.io/kube-controller-manager:%s", tenantControlPlane.Spec.Kubernetes.Version),
-				Command: []string{
-					"kube-controller-manager",
-					"--allocate-node-cidrs=true",
-					"--authentication-kubeconfig=/etc/kubernetes/controller-manager.conf",
-					"--authorization-kubeconfig=/etc/kubernetes/controller-manager.conf",
-					"--bind-address=0.0.0.0",
-					fmt.Sprintf("--client-ca-file=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.CACertName)),
-					fmt.Sprintf("--cluster-name=%s", tenantControlPlane.GetName()),
-					fmt.Sprintf("--cluster-signing-cert-file=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.CACertName)),
-					fmt.Sprintf("--cluster-signing-key-file=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.CAKeyName)),
-					"--controllers=*,bootstrapsigner,tokencleaner",
-					"--kubeconfig=/etc/kubernetes/controller-manager.conf",
-					"--leader-elect=true",
-					fmt.Sprintf("--service-cluster-ip-range=%s", tenantControlPlane.Spec.NetworkProfile.ServiceCIDR),
-					fmt.Sprintf("--cluster-cidr=%s", tenantControlPlane.Spec.NetworkProfile.PodCIDR),
-					fmt.Sprintf("--requestheader-client-ca-file=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.FrontProxyCACertName)),
-					fmt.Sprintf("--root-ca-file=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.CACertName)),
-					fmt.Sprintf("--service-account-private-key-file=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.ServiceAccountPrivateKeyName)),
-					"--use-service-account-credentials=true",
-				},
-				Resources: corev1.ResourceRequirements{
-					Limits: nil,
-					Requests: corev1.ResourceList{
-						corev1.ResourceCPU: quantity.MustParse("200m"),
-					},
-				},
-				VolumeMounts: []corev1.VolumeMount{
-					{
-						Name:      "controller-manager-kubeconfig",
-						ReadOnly:  true,
-						MountPath: "/etc/kubernetes",
-					},
-					{
-						Name:      "etc-kubernetes-pki",
-						ReadOnly:  true,
-						MountPath: v1beta3.DefaultCertificatesDir,
-					},
-					{
-						Name:      "etc-ca-certificates",
-						ReadOnly:  true,
-						MountPath: "/etc/ca-certificates",
-					},
-					{
-						Name:      "etc-ssl-certs",
-						ReadOnly:  true,
-						MountPath: "/etc/ssl/certs",
-					},
-					{
-						Name:      "usr-share-ca-certificates",
-						ReadOnly:  true,
-						MountPath: "/usr/share/ca-certificates",
-					},
-					{
-						Name:      "usr-local-share-ca-certificates",
-						ReadOnly:  true,
-						MountPath: "/usr/local/share/ca-certificates",
-					},
-				},
-				LivenessProbe: &corev1.Probe{
-					ProbeHandler: corev1.ProbeHandler{
-						HTTPGet: &corev1.HTTPGetAction{
-							Path:   "/healthz",
-							Port:   intstr.FromInt(10257),
-							Scheme: corev1.URISchemeHTTPS,
-						},
-					},
-					InitialDelaySeconds: 0,
-					TimeoutSeconds:      1,
-					PeriodSeconds:       10,
-					SuccessThreshold:    1,
-					FailureThreshold:    3,
-				},
-				StartupProbe: &corev1.Probe{
-					ProbeHandler: corev1.ProbeHandler{
-						HTTPGet: &corev1.HTTPGetAction{
-							Path:   "/healthz",
-							Port:   intstr.FromInt(10257),
-							Scheme: corev1.URISchemeHTTPS,
-						},
-					},
-					InitialDelaySeconds: 0,
-					TimeoutSeconds:      1,
-					PeriodSeconds:       10,
-					SuccessThreshold:    1,
-					FailureThreshold:    3,
-				},
-				TerminationMessagePath:   "/dev/termination-log",
-				TerminationMessagePolicy: "File",
-				ImagePullPolicy:          corev1.PullIfNotPresent,
-			},
-		}
-		r.resource.Spec.Strategy = appsv1.DeploymentStrategy{
-			Type: appsv1.RollingUpdateDeploymentStrategyType,
-			RollingUpdate: &appsv1.RollingUpdateDeployment{
-				MaxUnavailable: &maxUnavailable,
-				MaxSurge:       &maxSurge,
-			},
-		}
-
-		return controllerutil.SetControllerReference(tenantControlPlane, r.resource, r.Client.Scheme())
-	})
+	return utilities.CreateOrUpdateWithConflict(ctx, r.Client, r.resource, r.mutate(ctx, tenantControlPlane))
 }
 
 func (r *KubernetesDeploymentResource) GetName() string {
 	return r.Name
 }
 
-func (r *KubernetesDeploymentResource) UpdateTenantControlPlaneStatus(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+func (r *KubernetesDeploymentResource) UpdateTenantControlPlaneStatus(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
 	switch {
 	case !r.isProgressingUpgrade():
 		tenantControlPlane.Status.Kubernetes.Version.Status = &kamajiv1alpha1.VersionReady
@@ -573,11 +110,39 @@ func (r *KubernetesDeploymentResource) UpdateTenantControlPlaneStatus(ctx contex
 		DeploymentStatus: r.resource.Status,
 		Name:             r.resource.GetName(),
 		Namespace:        r.resource.GetNamespace(),
+		LastUpdate:       metav1.Now(),
 	}
 
 	return nil
 }
 
+func (r *KubernetesDeploymentResource) deploymentTemplateLabels(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (labels map[string]string) {
+	hash := func(ctx context.Context, namespace, secretName string) (hash string) {
+		hash, _ = utilities.SecretHashValue(ctx, r.Client, namespace, secretName)
+
+		return
+	}
+
+	labels = map[string]string{
+		"kamaji.clastix.io/soot":                                            tenantControlPlane.GetName(),
+		"component.kamaji.clastix.io/api-server-certificate":                hash(ctx, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.Certificates.APIServer.SecretName),
+		"component.kamaji.clastix.io/api-server-kubelet-client-certificate": hash(ctx, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.Certificates.APIServerKubeletClient.SecretName),
+		"component.kamaji.clastix.io/ca":                                    hash(ctx, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.Certificates.CA.SecretName),
+		"component.kamaji.clastix.io/controller-manager-kubeconfig":         hash(ctx, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.KubeConfig.ControllerManager.SecretName),
+		"component.kamaji.clastix.io/front-proxy-ca-certificate":            hash(ctx, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.Certificates.FrontProxyCA.SecretName),
+		"component.kamaji.clastix.io/front-proxy-client-certificate":        hash(ctx, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.Certificates.FrontProxyClient.SecretName),
+		"component.kamaji.clastix.io/service-account":                       hash(ctx, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.Certificates.SA.SecretName),
+		"component.kamaji.clastix.io/scheduler-kubeconfig":                  hash(ctx, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.KubeConfig.Scheduler.SecretName),
+	}
+
+	if r.ETCDStorageType == types.ETCD {
+		labels["component.kamaji.clastix.io/etcd-ca-certificates"] = hash(ctx, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.Certificates.ETCD.CA.SecretName)
+		labels["component.kamaji.clastix.io/etcd-certificates"] = hash(ctx, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.Certificates.ETCD.APIServer.SecretName)
+	}
+
+	return labels
+}
+
 func (r *KubernetesDeploymentResource) isProgressingUpgrade() bool {
 	if r.resource.ObjectMeta.GetGeneration() != r.resource.Status.ObservedGeneration {
 		return true

internal/resources/k8s_ingress_resource.go

@@ -33,7 +33,7 @@ func (r *KubernetesIngressResource) ShouldCleanup(tenantControlPlane *kamajiv1al
 	return !tenantControlPlane.Spec.ControlPlane.Ingress.Enabled
 }
 
-func (r *KubernetesIngressResource) CleanUp(ctx context.Context) (bool, error) {
+func (r *KubernetesIngressResource) CleanUp(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) {
 	if err := r.Client.Delete(ctx, r.resource); err != nil {
 		if !k8serrors.IsNotFound(err) {
 			return false, err
@@ -73,8 +73,8 @@ func (r *KubernetesIngressResource) Define(ctx context.Context, tenantControlPla
 	return nil
 }
 
-func (r *KubernetesIngressResource) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) {
-	return controllerutil.CreateOrUpdate(ctx, r.Client, r.resource, func() error {
+func (r *KubernetesIngressResource) mutate(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) controllerutil.MutateFn {
+	return func() error {
 		labels := utilities.MergeMaps(r.resource.GetLabels(), tenantControlPlane.Spec.ControlPlane.Ingress.AdditionalMetadata.Labels)
 		r.resource.SetLabels(labels)
 
@@ -119,17 +119,23 @@ func (r *KubernetesIngressResource) CreateOrUpdate(ctx context.Context, tenantCo
 		}
 
 		rule.HTTP.Paths[0] = path
-		rule.Host = tenantControlPlane.Spec.ControlPlane.Ingress.Hostname
-		if rule.Host == "" {
-			rule.Host = fmt.Sprintf("%s.%s.%s", tenantControlPlane.GetName(), tenantControlPlane.GetNamespace(), tenantControlPlane.Spec.NetworkProfile.Domain)
+
+		if len(tenantControlPlane.Spec.ControlPlane.Ingress.Hostname) == 0 {
+			return fmt.Errorf("missing hostname to expose the Tenant Control Plane using an Ingress resource")
 		}
 
+		rule.Host = tenantControlPlane.Spec.ControlPlane.Ingress.Hostname
+
 		r.resource.Spec.Rules = []networkingv1.IngressRule{
 			rule,
 		}
 
 		return controllerutil.SetControllerReference(tenantControlPlane, r.resource, r.Client.Scheme())
-	})
+	}
+}
+
+func (r *KubernetesIngressResource) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) {
+	return utilities.CreateOrUpdateWithConflict(ctx, r.Client, r.resource, r.mutate(tenantControlPlane))
 }
 
 func (r *KubernetesIngressResource) GetName() string {

internal/resources/k8s_service_resource.go

@@ -5,6 +5,7 @@ package resources
 
 import (
 	"context"
+	"fmt"
 
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -34,7 +35,7 @@ func (r *KubernetesServiceResource) ShouldCleanup(plane *kamajiv1alpha1.TenantCo
 	return false
 }
 
-func (r *KubernetesServiceResource) CleanUp(ctx context.Context) (bool, error) {
+func (r *KubernetesServiceResource) CleanUp(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) {
 	return false, nil
 }
 
@@ -44,6 +45,13 @@ func (r *KubernetesServiceResource) UpdateTenantControlPlaneStatus(ctx context.C
 	tenantControlPlane.Status.Kubernetes.Service.Namespace = r.resource.GetNamespace()
 	tenantControlPlane.Status.Kubernetes.Service.Port = r.resource.Spec.Ports[0].Port
 
+	address, err := tenantControlPlane.DeclaredControlPlaneAddress(ctx, r.Client)
+	if err != nil {
+		return err
+	}
+
+	tenantControlPlane.Status.ControlPlaneEndpoint = fmt.Sprintf("%s:%d", address, tenantControlPlane.Spec.NetworkProfile.Port)
+
 	return nil
 }
 
@@ -52,7 +60,6 @@ func (r *KubernetesServiceResource) Define(ctx context.Context, tenantControlPla
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      tenantControlPlane.GetName(),
 			Namespace: tenantControlPlane.GetNamespace(),
-			Labels:    utilities.CommonLabels(tenantControlPlane.GetName()),
 		},
 	}
 
@@ -62,30 +69,34 @@ func (r *KubernetesServiceResource) Define(ctx context.Context, tenantControlPla
 }
 
 func (r *KubernetesServiceResource) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) {
+	return utilities.CreateOrUpdateWithConflict(ctx, r.Client, r.resource, r.mutate(ctx, tenantControlPlane))
+}
+
+func (r *KubernetesServiceResource) mutate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) controllerutil.MutateFn {
 	// We don't need to check error here: in case of dynamic external IP, the Service must be created in advance.
 	// After that, the specific cloud controller-manager will provide an IP that will be then used.
-	address, _ := tenantControlPlane.GetAddress(ctx, r.Client)
+	address, _ := tenantControlPlane.DeclaredControlPlaneAddress(ctx, r.Client)
 
-	return controllerutil.CreateOrUpdate(ctx, r.Client, r.resource, func() error {
-		var servicePort corev1.ServicePort
-		if len(r.resource.Spec.Ports) > 0 {
-			servicePort = r.resource.Spec.Ports[0]
-		}
-		servicePort.Protocol = corev1.ProtocolTCP
-		servicePort.Port = tenantControlPlane.Spec.NetworkProfile.Port
-		servicePort.TargetPort = intstr.FromInt(int(tenantControlPlane.Spec.NetworkProfile.Port))
-
-		r.resource.Spec.Ports = []corev1.ServicePort{servicePort}
-		r.resource.Spec.Selector = map[string]string{
-			"kamaji.clastix.io/soot": tenantControlPlane.GetName(),
-		}
-
-		labels := utilities.MergeMaps(r.resource.GetLabels(), tenantControlPlane.Spec.ControlPlane.Service.AdditionalMetadata.Labels)
+	return func() error {
+		labels := utilities.MergeMaps(utilities.CommonLabels(tenantControlPlane.GetName()), tenantControlPlane.Spec.ControlPlane.Service.AdditionalMetadata.Labels)
 		r.resource.SetLabels(labels)
 
 		annotations := utilities.MergeMaps(r.resource.GetAnnotations(), tenantControlPlane.Spec.ControlPlane.Service.AdditionalMetadata.Annotations)
 		r.resource.SetAnnotations(annotations)
 
+		r.resource.Spec.Selector = map[string]string{
+			"kamaji.clastix.io/soot": tenantControlPlane.GetName(),
+		}
+
+		if len(r.resource.Spec.Ports) == 0 {
+			r.resource.Spec.Ports = make([]corev1.ServicePort, 1)
+		}
+
+		r.resource.Spec.Ports[0].Name = "kube-apiserver"
+		r.resource.Spec.Ports[0].Protocol = corev1.ProtocolTCP
+		r.resource.Spec.Ports[0].Port = tenantControlPlane.Spec.NetworkProfile.Port
+		r.resource.Spec.Ports[0].TargetPort = intstr.FromInt(int(tenantControlPlane.Spec.NetworkProfile.Port))
+
 		switch tenantControlPlane.Spec.ControlPlane.Service.ServiceType {
 		case kamajiv1alpha1.ServiceTypeLoadBalancer:
 			r.resource.Spec.Type = corev1.ServiceTypeLoadBalancer
@@ -109,7 +120,7 @@ func (r *KubernetesServiceResource) CreateOrUpdate(ctx context.Context, tenantCo
 		}
 
 		return controllerutil.SetControllerReference(tenantControlPlane, r.resource, r.Client.Scheme())
-	})
+	}
 }
 
 func (r *KubernetesServiceResource) GetName() string {

internal/resources/konnectivity/agent.go

@@ -0,0 +1,201 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package konnectivity
+
+import (
+	"context"
+	"fmt"
+
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/utils/pointer"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+	"github.com/clastix/kamaji/internal/utilities"
+)
+
+const (
+	agentNamespace = "kube-system"
+)
+
+type Agent struct {
+	resource     *appsv1.DaemonSet
+	Client       client.Client
+	Name         string
+	tenantClient client.Client
+}
+
+func (r *Agent) ShouldStatusBeUpdated(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
+	return tenantControlPlane.Status.Addons.Konnectivity.Agent.Name != r.resource.GetName() ||
+		tenantControlPlane.Status.Addons.Konnectivity.Agent.Namespace != r.resource.GetNamespace() ||
+		tenantControlPlane.Status.Addons.Konnectivity.Agent.RV != r.resource.ObjectMeta.ResourceVersion
+}
+
+func (r *Agent) ShouldCleanup(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
+	return tenantControlPlane.Spec.Addons.Konnectivity == nil
+}
+
+func (r *Agent) CleanUp(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) {
+	if err := r.tenantClient.Delete(ctx, r.resource); err != nil {
+		if !k8serrors.IsNotFound(err) {
+			return false, err
+		}
+
+		return false, nil
+	}
+
+	return true, nil
+}
+
+func (r *Agent) Define(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+	r.resource = &appsv1.DaemonSet{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      AgentName,
+			Namespace: agentNamespace,
+		},
+	}
+
+	client, err := utilities.GetTenantClient(ctx, r.Client, tenantControlPlane)
+	if err != nil {
+		return err
+	}
+
+	r.tenantClient = client
+
+	return nil
+}
+
+func (r *Agent) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) {
+	return controllerutil.CreateOrUpdate(ctx, r.tenantClient, r.resource, r.mutate(ctx, tenantControlPlane))
+}
+
+func (r *Agent) GetName() string {
+	return r.Name
+}
+
+func (r *Agent) UpdateTenantControlPlaneStatus(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+	if tenantControlPlane.Spec.Addons.Konnectivity != nil {
+		tenantControlPlane.Status.Addons.Konnectivity.Agent = kamajiv1alpha1.ExternalKubernetesObjectStatus{
+			Name:       r.resource.GetName(),
+			Namespace:  r.resource.GetNamespace(),
+			RV:         r.resource.ObjectMeta.ResourceVersion,
+			LastUpdate: metav1.Now(),
+		}
+		tenantControlPlane.Status.Addons.Konnectivity.Enabled = true
+
+		return nil
+	}
+
+	tenantControlPlane.Status.Addons.Konnectivity.Enabled = false
+	tenantControlPlane.Status.Addons.Konnectivity.Agent = kamajiv1alpha1.ExternalKubernetesObjectStatus{}
+
+	return nil
+}
+
+func (r *Agent) mutate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) controllerutil.MutateFn {
+	return func() (err error) {
+		address, _, err := tenantControlPlane.AssignedControlPlaneAddress()
+		if err != nil {
+			return err
+		}
+
+		r.resource.SetLabels(utilities.MergeMaps(
+			utilities.KamajiLabels(),
+			map[string]string{
+				"k8s-app":                         AgentName,
+				"addonmanager.kubernetes.io/mode": "Reconcile",
+			},
+		))
+
+		if r.resource.Spec.Selector == nil {
+			r.resource.Spec.Selector = &metav1.LabelSelector{}
+		}
+		r.resource.Spec.Selector.MatchLabels = map[string]string{
+			"k8s-app": AgentName,
+		}
+
+		r.resource.Spec.Template.SetLabels(utilities.MergeMaps(
+			r.resource.Spec.Template.GetLabels(),
+			map[string]string{
+				"k8s-app": AgentName,
+			},
+		))
+
+		r.resource.Spec.Template.Spec.PriorityClassName = "system-cluster-critical"
+		r.resource.Spec.Template.Spec.Tolerations = []corev1.Toleration{
+			{
+				Key:      "CriticalAddonsOnly",
+				Operator: "Exists",
+			},
+		}
+		r.resource.Spec.Template.Spec.NodeSelector = map[string]string{
+			"kubernetes.io/os": "linux",
+		}
+		r.resource.Spec.Template.Spec.ServiceAccountName = AgentName
+		r.resource.Spec.Template.Spec.Volumes = []corev1.Volume{
+			{
+				Name: agentTokenName,
+				VolumeSource: corev1.VolumeSource{
+					Projected: &corev1.ProjectedVolumeSource{
+						Sources: []corev1.VolumeProjection{
+							{
+								ServiceAccountToken: &corev1.ServiceAccountTokenProjection{
+									Path:              agentTokenName,
+									Audience:          tenantControlPlane.Status.Addons.Konnectivity.ClusterRoleBinding.Name,
+									ExpirationSeconds: pointer.Int64(3600),
+								},
+							},
+						},
+						DefaultMode: pointer.Int32Ptr(420),
+					},
+				},
+			},
+		}
+
+		if len(r.resource.Spec.Template.Spec.Containers) != 1 {
+			r.resource.Spec.Template.Spec.Containers = make([]corev1.Container, 1)
+		}
+
+		r.resource.Spec.Template.Spec.Containers[0].Image = fmt.Sprintf("%s:%s", tenantControlPlane.Spec.Addons.Konnectivity.AgentImage, tenantControlPlane.Spec.Addons.Konnectivity.Version)
+		r.resource.Spec.Template.Spec.Containers[0].Name = AgentName
+		r.resource.Spec.Template.Spec.Containers[0].Command = []string{"/proxy-agent"}
+		r.resource.Spec.Template.Spec.Containers[0].Args = []string{
+			"-v=8",
+			"--logtostderr=true",
+			"--ca-cert=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt",
+			fmt.Sprintf("--proxy-server-host=%s", address),
+			fmt.Sprintf("--proxy-server-port=%d", tenantControlPlane.Spec.Addons.Konnectivity.ProxyPort),
+			"--admin-server-port=8133",
+			"--health-server-port=8134",
+			"--service-account-token-path=/var/run/secrets/tokens/konnectivity-agent-token",
+		}
+		r.resource.Spec.Template.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{
+			{
+				MountPath: "/var/run/secrets/tokens",
+				Name:      agentTokenName,
+			},
+		}
+		r.resource.Spec.Template.Spec.Containers[0].LivenessProbe = &corev1.Probe{
+			ProbeHandler: corev1.ProbeHandler{
+				HTTPGet: &corev1.HTTPGetAction{
+					Path:   "/healthz",
+					Port:   intstr.FromInt(8134),
+					Scheme: corev1.URISchemeHTTP,
+				},
+			},
+			InitialDelaySeconds: 15,
+			TimeoutSeconds:      15,
+			PeriodSeconds:       10,
+			SuccessThreshold:    1,
+			FailureThreshold:    3,
+		}
+
+		return nil
+	}
+}

internal/resources/konnectivity/certificate_resource.go

@@ -0,0 +1,180 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package konnectivity
+
+import (
+	"bytes"
+	"context"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"fmt"
+	"math/big"
+	"math/rand"
+	"time"
+
+	"github.com/go-logr/logr"
+	corev1 "k8s.io/api/core/v1"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	k8stypes "k8s.io/apimachinery/pkg/types"
+	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+	"github.com/clastix/kamaji/internal/crypto"
+	"github.com/clastix/kamaji/internal/kubeadm"
+	"github.com/clastix/kamaji/internal/utilities"
+)
+
+type CertificateResource struct {
+	resource *corev1.Secret
+	Client   client.Client
+	Log      logr.Logger
+	Name     string
+}
+
+func (r *CertificateResource) ShouldStatusBeUpdated(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
+	return tenantControlPlane.Status.Addons.Konnectivity.Certificate.SecretName != r.resource.GetName() ||
+		tenantControlPlane.Status.Addons.Konnectivity.Certificate.ResourceVersion != r.resource.ResourceVersion
+}
+
+func (r *CertificateResource) ShouldCleanup(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
+	return tenantControlPlane.Spec.Addons.Konnectivity == nil
+}
+
+func (r *CertificateResource) CleanUp(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) {
+	if err := r.Client.Delete(ctx, r.resource); err != nil {
+		if !k8serrors.IsNotFound(err) {
+			return false, err
+		}
+
+		return false, nil
+	}
+
+	return true, nil
+}
+
+func (r *CertificateResource) Define(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+	r.resource = &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      r.getPrefixedName(tenantControlPlane),
+			Namespace: tenantControlPlane.GetNamespace(),
+		},
+	}
+
+	return nil
+}
+
+func (r *CertificateResource) getPrefixedName(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) string {
+	return utilities.AddTenantPrefix(r.Name, tenantControlPlane)
+}
+
+func (r *CertificateResource) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) {
+	return controllerutil.CreateOrUpdate(ctx, r.Client, r.resource, r.mutate(ctx, tenantControlPlane))
+}
+
+func (r *CertificateResource) GetName() string {
+	return r.Name
+}
+
+func (r *CertificateResource) UpdateTenantControlPlaneStatus(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+	if tenantControlPlane.Spec.Addons.Konnectivity != nil {
+		tenantControlPlane.Status.Addons.Konnectivity.Certificate.LastUpdate = metav1.Now()
+		tenantControlPlane.Status.Addons.Konnectivity.Certificate.SecretName = r.resource.GetName()
+		tenantControlPlane.Status.Addons.Konnectivity.Certificate.ResourceVersion = r.resource.ResourceVersion
+		tenantControlPlane.Status.Addons.Konnectivity.Enabled = true
+
+		return nil
+	}
+
+	tenantControlPlane.Status.Addons.Konnectivity.Certificate = kamajiv1alpha1.CertificatePrivateKeyPairStatus{}
+	tenantControlPlane.Status.Addons.Konnectivity.Enabled = false
+
+	return nil
+}
+
+func (r *CertificateResource) mutate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) controllerutil.MutateFn {
+	return func() error {
+		latestCARV := tenantControlPlane.Status.Certificates.CA.ResourceVersion
+		actualCARV := r.resource.GetLabels()["latest-ca-rv"]
+		if latestCARV == actualCARV {
+			isValid, err := isCertificateAndKeyPairValid(
+				r.resource.Data[corev1.TLSCertKey],
+				r.resource.Data[corev1.TLSPrivateKeyKey],
+			)
+			if err != nil {
+				r.Log.Info(fmt.Sprintf("%s certificate-private_key pair is not valid: %s", konnectivityCertAndKeyBaseName, err.Error()))
+			}
+			if isValid {
+				return nil
+			}
+		}
+
+		namespacedName := k8stypes.NamespacedName{Namespace: tenantControlPlane.GetNamespace(), Name: tenantControlPlane.Status.Certificates.CA.SecretName}
+		secretCA := &corev1.Secret{}
+		if err := r.Client.Get(ctx, namespacedName, secretCA); err != nil {
+			return err
+		}
+
+		ca := kubeadm.CertificatePrivateKeyPair{
+			Name:        kubeadmconstants.CACertAndKeyBaseName,
+			Certificate: secretCA.Data[kubeadmconstants.CACertName],
+			PrivateKey:  secretCA.Data[kubeadmconstants.CAKeyName],
+		}
+		cert, privKey, err := getCertificateAndKeyPair(ca.Certificate, ca.PrivateKey)
+		if err != nil {
+			return err
+		}
+
+		r.resource.Type = corev1.SecretTypeTLS
+		r.resource.Data = map[string][]byte{
+			corev1.TLSCertKey:       cert.Bytes(),
+			corev1.TLSPrivateKeyKey: privKey.Bytes(),
+		}
+		r.resource.SetLabels(utilities.MergeMaps(
+			utilities.KamajiLabels(),
+			map[string]string{
+				"latest-ca-rv":                latestCARV,
+				"kamaji.clastix.io/name":      tenantControlPlane.GetName(),
+				"kamaji.clastix.io/component": r.GetName(),
+			},
+		))
+
+		return ctrl.SetControllerReference(tenantControlPlane, r.resource, r.Client.Scheme())
+	}
+}
+
+func getCertificateAndKeyPair(caCert []byte, caPrivKey []byte) (*bytes.Buffer, *bytes.Buffer, error) {
+	template := getCertTemplate()
+
+	return crypto.GetCertificateAndKeyPair(template, caCert, caPrivKey)
+}
+
+func isCertificateAndKeyPairValid(cert []byte, privKey []byte) (bool, error) {
+	return crypto.IsValidCertificateKeyPairBytes(cert, privKey)
+}
+
+func getCertTemplate() *x509.Certificate {
+	serialNumber := big.NewInt(rand.Int63())
+
+	return &x509.Certificate{
+		PublicKeyAlgorithm: x509.RSA,
+		SerialNumber:       serialNumber,
+		Subject: pkix.Name{
+			CommonName:   CertCommonName,
+			Organization: []string{certOrganization},
+		},
+		NotBefore:    time.Now(),
+		NotAfter:     time.Now().AddDate(certExpirationDelayYears, 0, 0),
+		SubjectKeyId: []byte{1, 2, 3, 4, 6},
+		ExtKeyUsage: []x509.ExtKeyUsage{
+			x509.ExtKeyUsageClientAuth,
+			x509.ExtKeyUsageServerAuth,
+			x509.ExtKeyUsageCodeSigning,
+		},
+		KeyUsage: x509.KeyUsageDigitalSignature,
+	}
+}

internal/resources/konnectivity/cluster_role_binding_resource.go

@@ -0,0 +1,115 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package konnectivity
+
+import (
+	"context"
+
+	rbacv1 "k8s.io/api/rbac/v1"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+	"github.com/clastix/kamaji/internal/utilities"
+)
+
+type ClusterRoleBindingResource struct {
+	resource     *rbacv1.ClusterRoleBinding
+	Client       client.Client
+	Name         string
+	tenantClient client.Client
+}
+
+func (r *ClusterRoleBindingResource) ShouldStatusBeUpdated(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
+	return tenantControlPlane.Status.Addons.Konnectivity.ClusterRoleBinding.Name != r.resource.GetName() ||
+		tenantControlPlane.Status.Addons.Konnectivity.ClusterRoleBinding.RV != r.resource.ObjectMeta.ResourceVersion
+}
+
+func (r *ClusterRoleBindingResource) ShouldCleanup(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
+	return tenantControlPlane.Spec.Addons.Konnectivity == nil
+}
+
+func (r *ClusterRoleBindingResource) CleanUp(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) {
+	if err := r.tenantClient.Delete(ctx, r.resource); err != nil {
+		if !k8serrors.IsNotFound(err) {
+			return false, err
+		}
+
+		return false, nil
+	}
+
+	return true, nil
+}
+
+func (r *ClusterRoleBindingResource) Define(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+	r.resource = &rbacv1.ClusterRoleBinding{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: CertCommonName,
+		},
+	}
+
+	client, err := utilities.GetTenantClient(ctx, r.Client, tenantControlPlane)
+	if err != nil {
+		return err
+	}
+
+	r.tenantClient = client
+
+	return nil
+}
+
+func (r *ClusterRoleBindingResource) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) {
+	return controllerutil.CreateOrUpdate(ctx, r.tenantClient, r.resource, r.mutate(ctx, tenantControlPlane))
+}
+
+func (r *ClusterRoleBindingResource) GetName() string {
+	return r.Name
+}
+
+func (r *ClusterRoleBindingResource) UpdateTenantControlPlaneStatus(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+	if tenantControlPlane.Spec.Addons.Konnectivity != nil {
+		tenantControlPlane.Status.Addons.Konnectivity.Enabled = true
+		tenantControlPlane.Status.Addons.Konnectivity.ClusterRoleBinding = kamajiv1alpha1.ExternalKubernetesObjectStatus{
+			Name: r.resource.GetName(),
+			RV:   r.resource.ObjectMeta.ResourceVersion,
+		}
+
+		return nil
+	}
+
+	tenantControlPlane.Status.Addons.Konnectivity.ClusterRoleBinding = kamajiv1alpha1.ExternalKubernetesObjectStatus{}
+	tenantControlPlane.Status.Addons.Konnectivity.Enabled = false
+
+	return nil
+}
+
+func (r *ClusterRoleBindingResource) mutate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) controllerutil.MutateFn {
+	return func() error {
+		r.resource.SetLabels(utilities.MergeMaps(
+			utilities.KamajiLabels(),
+			map[string]string{
+				"kubernetes.io/cluster-service":   "true",
+				"addonmanager.kubernetes.io/mode": "Reconcile",
+			},
+		))
+
+		r.resource.RoleRef = rbacv1.RoleRef{
+			APIGroup: rbacv1.GroupName,
+			Kind:     "ClusterRole",
+			Name:     roleAuthDelegator,
+		}
+
+		r.resource.Subjects = []rbacv1.Subject{
+			{
+				APIGroup: rbacv1.GroupName,
+				Kind:     rbacv1.UserKind,
+				Name:     CertCommonName,
+			},
+		}
+
+		return nil
+	}
+}

internal/resources/konnectivity/constants.go

@@ -0,0 +1,22 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package konnectivity
+
+const (
+	AgentName      = "konnectivity-agent"
+	CertCommonName = "system:konnectivity-server"
+
+	agentTokenName                  = "konnectivity-agent-token"
+	apiServerAPIVersion             = "apiserver.k8s.io/v1beta1"
+	certExpirationDelayYears        = 10
+	certOrganization                = "system:master"
+	defaultClusterName              = "kubernetes"
+	defaultUDSName                  = "/run/konnectivity/konnectivity-server.socket"
+	egressSelectorConfigurationKind = "EgressSelectorConfiguration"
+	egressSelectorConfigurationName = "cluster"
+	konnectivityCertAndKeyBaseName  = "konnectivity"
+	konnectivityKubeconfigFileName  = "konnectivity-server.conf"
+	kubeconfigAPIVersion            = "v1"
+	roleAuthDelegator               = "system:auth-delegator"
+)

internal/resources/konnectivity/deployment_resource.go

@@ -0,0 +1,327 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package konnectivity
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/pkg/errors"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/utils/pointer"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+	"sigs.k8s.io/controller-runtime/pkg/log"
+
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+	"github.com/clastix/kamaji/internal/types"
+	"github.com/clastix/kamaji/internal/utilities"
+)
+
+const (
+	konnectivityEgressSelectorConfigurationPath = "/etc/kubernetes/konnectivity/configurations/egress-selector-configuration.yaml"
+	konnectivityServerName                      = "konnectivity-server"
+	konnectivityServerPath                      = "/run/konnectivity"
+
+	egressSelectorConfigurationVolume  = "egress-selector-configuration"
+	konnectivityUDSVolume              = "konnectivity-uds"
+	konnectivityServerKubeconfigVolume = "konnectivity-server-kubeconfig"
+)
+
+type KubernetesDeploymentResource struct {
+	resource               *appsv1.Deployment
+	Client                 client.Client
+	ETCDStorageType        types.ETCDStorageType
+	ETCDEndpoints          []string
+	ETCDCompactionInterval string
+	Name                   string
+}
+
+func (r *KubernetesDeploymentResource) isStatusEqual(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
+	return r.resource.Status.String() == tenantControlPlane.Status.Kubernetes.Deployment.DeploymentStatus.String()
+}
+
+func (r *KubernetesDeploymentResource) ShouldStatusBeUpdated(context.Context, *kamajiv1alpha1.TenantControlPlane) bool {
+	return false
+}
+
+func (r *KubernetesDeploymentResource) ShouldCleanup(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
+	return tenantControlPlane.Spec.Addons.Konnectivity == nil
+}
+
+func (r *KubernetesDeploymentResource) CleanUp(ctx context.Context, _ *kamajiv1alpha1.TenantControlPlane) (bool, error) {
+	logger := log.FromContext(ctx)
+
+	logger.Info("performing clean-up from Deployment of Konnectivity")
+
+	res, err := utilities.CreateOrUpdateWithConflict(ctx, r.Client, r.resource, func() error {
+		if found, index := utilities.HasNamedContainer(r.resource.Spec.Template.Spec.Containers, konnectivityServerName); found {
+			logger.Info("removing Konnectivity container")
+
+			var containers []corev1.Container
+
+			containers = append(containers, r.resource.Spec.Template.Spec.Containers[:index]...)
+			containers = append(containers, r.resource.Spec.Template.Spec.Containers[index+1:]...)
+
+			r.resource.Spec.Template.Spec.Containers = containers
+		}
+
+		if found, index := utilities.HasNamedContainer(r.resource.Spec.Template.Spec.Containers, "kube-apiserver"); found {
+			argsMap := utilities.ArgsFromSliceToMap(r.resource.Spec.Template.Spec.Containers[index].Args)
+
+			if utilities.ArgsRemoveFlag(argsMap, "--egress-selector-config-file") {
+				logger.Info("removing egress selector configuration file from kube-apiserver container")
+
+				r.resource.Spec.Template.Spec.Containers[index].Args = utilities.ArgsFromMapToSlice(argsMap)
+			}
+
+			for _, volumeName := range []string{konnectivityUDSVolume, egressSelectorConfigurationVolume} {
+				if volumeFound, volumeIndex := utilities.HasNamedVolume(r.resource.Spec.Template.Spec.Volumes, egressSelectorConfigurationVolume); volumeFound {
+					logger.Info("removing Konnectivity volume " + volumeName)
+
+					var volumes []corev1.Volume
+
+					volumes = append(volumes, r.resource.Spec.Template.Spec.Volumes[:volumeIndex]...)
+					volumes = append(volumes, r.resource.Spec.Template.Spec.Volumes[volumeIndex+1:]...)
+
+					r.resource.Spec.Template.Spec.Volumes = volumes
+				}
+			}
+		}
+
+		return nil
+	})
+
+	return res == controllerutil.OperationResultUpdated, err
+}
+
+func (r *KubernetesDeploymentResource) Define(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+	r.resource = &appsv1.Deployment{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      tenantControlPlane.GetName(),
+			Namespace: tenantControlPlane.GetNamespace(),
+		},
+	}
+
+	return nil
+}
+
+func (r *KubernetesDeploymentResource) syncContainer(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+	found, index := utilities.HasNamedContainer(r.resource.Spec.Template.Spec.Containers, konnectivityServerName)
+	if !found {
+		r.resource.Spec.Template.Spec.Containers = append(r.resource.Spec.Template.Spec.Containers, corev1.Container{})
+		index = len(r.resource.Spec.Template.Spec.Containers) - 1
+	}
+
+	r.resource.Spec.Template.Spec.Containers[index].Name = konnectivityServerName
+	r.resource.Spec.Template.Spec.Containers[index].Image = fmt.Sprintf("%s:%s", tenantControlPlane.Spec.Addons.Konnectivity.ServerImage, tenantControlPlane.Spec.Addons.Konnectivity.Version)
+	r.resource.Spec.Template.Spec.Containers[index].Command = []string{"/proxy-server"}
+	r.resource.Spec.Template.Spec.Containers[index].Args = []string{
+		"-v=8",
+		"--logtostderr=true",
+		fmt.Sprintf("--uds-name=%s/konnectivity-server.socket", konnectivityServerPath),
+		"--cluster-cert=/etc/kubernetes/pki/apiserver.crt",
+		"--cluster-key=/etc/kubernetes/pki/apiserver.key",
+		"--mode=grpc",
+		"--server-port=0",
+		fmt.Sprintf("--agent-port=%d", tenantControlPlane.Spec.Addons.Konnectivity.ProxyPort),
+		"--admin-port=8133",
+		"--health-port=8134",
+		"--agent-namespace=kube-system",
+		fmt.Sprintf("--agent-service-account=%s", AgentName),
+		"--kubeconfig=/etc/kubernetes/konnectivity-server.conf",
+		fmt.Sprintf("--authentication-audience=%s", CertCommonName),
+		fmt.Sprintf("--server-count=%d", tenantControlPlane.Spec.ControlPlane.Deployment.Replicas),
+	}
+	r.resource.Spec.Template.Spec.Containers[index].LivenessProbe = &corev1.Probe{
+		InitialDelaySeconds: 30,
+		TimeoutSeconds:      60,
+		PeriodSeconds:       10,
+		SuccessThreshold:    1,
+		FailureThreshold:    3,
+		ProbeHandler: corev1.ProbeHandler{
+			HTTPGet: &corev1.HTTPGetAction{
+				Path:   "/healthz",
+				Port:   intstr.FromInt(8134),
+				Scheme: corev1.URISchemeHTTP,
+			},
+		},
+	}
+	r.resource.Spec.Template.Spec.Containers[index].Ports = []corev1.ContainerPort{
+		{
+			Name:          "agentport",
+			ContainerPort: tenantControlPlane.Spec.Addons.Konnectivity.ProxyPort,
+			Protocol:      corev1.ProtocolTCP,
+		},
+		{
+			Name:          "adminport",
+			ContainerPort: 8133,
+			Protocol:      corev1.ProtocolTCP,
+		},
+		{
+			Name:          "healthport",
+			ContainerPort: 8134,
+			Protocol:      corev1.ProtocolTCP,
+		},
+	}
+	r.resource.Spec.Template.Spec.Containers[index].VolumeMounts = []corev1.VolumeMount{
+		{
+			Name:      "etc-kubernetes-pki",
+			MountPath: "/etc/kubernetes/pki",
+			ReadOnly:  true,
+		},
+		{
+			Name:      "konnectivity-server-kubeconfig",
+			MountPath: "/etc/kubernetes/konnectivity-server.conf",
+			SubPath:   "konnectivity-server.conf",
+			ReadOnly:  true,
+		},
+		{
+			Name:      "konnectivity-uds",
+			MountPath: konnectivityServerPath,
+			ReadOnly:  false,
+		},
+	}
+	r.resource.Spec.Template.Spec.Containers[index].ImagePullPolicy = corev1.PullAlways
+	r.resource.Spec.Template.Spec.Containers[index].Resources = corev1.ResourceRequirements{
+		Limits:   nil,
+		Requests: nil,
+	}
+
+	if resources := tenantControlPlane.Spec.Addons.Konnectivity.Resources; resources != nil {
+		r.resource.Spec.Template.Spec.Containers[index].Resources = *resources
+	}
+
+	return nil
+}
+
+func (r *KubernetesDeploymentResource) mutate(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) controllerutil.MutateFn {
+	return func() (err error) {
+		// If konnectivity is disabled, no operation is required:
+		// removal of the container will be performed by clean-up.
+		if tenantControlPlane.Spec.Addons.Konnectivity == nil {
+			return nil
+		}
+
+		if len(r.resource.Spec.Template.Spec.Containers) == 0 {
+			return fmt.Errorf("the Deployment resource is not ready to be mangled for Konnectivity server enrichment")
+		}
+
+		if err = r.syncContainer(tenantControlPlane); err != nil {
+			return errors.Wrap(err, "cannot sync konnectivity-server container")
+		}
+		if err = r.patchKubeAPIServerContainer(); err != nil {
+			return errors.Wrap(err, "cannot sync patch kube-apiserver container")
+		}
+		if err = r.syncVolumes(tenantControlPlane); err != nil {
+			return errors.Wrap(err, "cannot patch required konnectivity volumes")
+		}
+
+		return nil
+	}
+}
+
+func (r *KubernetesDeploymentResource) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) {
+	return utilities.CreateOrUpdateWithConflict(ctx, r.Client, r.resource, r.mutate(ctx, tenantControlPlane))
+}
+
+func (r *KubernetesDeploymentResource) GetName() string {
+	return r.Name
+}
+
+func (r *KubernetesDeploymentResource) UpdateTenantControlPlaneStatus(context.Context, *kamajiv1alpha1.TenantControlPlane) error {
+	return nil
+}
+
+func (r *KubernetesDeploymentResource) patchKubeAPIServerContainer() error {
+	// Patching VolumesMounts
+	found, index := false, 0
+
+	found, index = utilities.HasNamedContainer(r.resource.Spec.Template.Spec.Containers, "kube-apiserver")
+	if !found {
+		return fmt.Errorf("missing kube-apiserver container, cannot patch arguments")
+	}
+	// Adding the egress selector config file flag
+	args := utilities.ArgsFromSliceToMap(r.resource.Spec.Template.Spec.Containers[index].Args)
+
+	if utilities.ArgsAddFlagValue(args, "--egress-selector-config-file", konnectivityEgressSelectorConfigurationPath) {
+		// LOG
+	}
+
+	r.resource.Spec.Template.Spec.Containers[index].Args = utilities.ArgsFromMapToSlice(args)
+
+	vFound, vIndex := false, 0
+	// Patching the volume mounts
+	if vFound, vIndex = utilities.HasNamedVolumeMount(r.resource.Spec.Template.Spec.Containers[index].VolumeMounts, konnectivityUDSVolume); !vFound {
+		r.resource.Spec.Template.Spec.Containers[index].VolumeMounts = append(r.resource.Spec.Template.Spec.Containers[index].VolumeMounts, corev1.VolumeMount{})
+		vIndex = len(r.resource.Spec.Template.Spec.Containers[index].VolumeMounts) - 1
+	}
+
+	r.resource.Spec.Template.Spec.Containers[index].VolumeMounts[vIndex].Name = konnectivityUDSVolume
+	r.resource.Spec.Template.Spec.Containers[index].VolumeMounts[vIndex].ReadOnly = false
+	r.resource.Spec.Template.Spec.Containers[index].VolumeMounts[vIndex].MountPath = konnectivityServerPath
+
+	if vFound, vIndex = utilities.HasNamedVolumeMount(r.resource.Spec.Template.Spec.Containers[index].VolumeMounts, egressSelectorConfigurationVolume); !vFound {
+		r.resource.Spec.Template.Spec.Containers[index].VolumeMounts = append(r.resource.Spec.Template.Spec.Containers[index].VolumeMounts, corev1.VolumeMount{})
+		vIndex = len(r.resource.Spec.Template.Spec.Containers[index].VolumeMounts) - 1
+	}
+
+	r.resource.Spec.Template.Spec.Containers[index].VolumeMounts[vIndex].Name = egressSelectorConfigurationVolume
+	r.resource.Spec.Template.Spec.Containers[index].VolumeMounts[vIndex].ReadOnly = false
+	r.resource.Spec.Template.Spec.Containers[index].VolumeMounts[vIndex].MountPath = "/etc/kubernetes/konnectivity/configurations"
+
+	return nil
+}
+
+func (r *KubernetesDeploymentResource) syncVolumes(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+	found, index := false, 0
+	// Defining volumes for the UDS socket
+	found, index = utilities.HasNamedVolume(r.resource.Spec.Template.Spec.Volumes, konnectivityUDSVolume)
+	if !found {
+		r.resource.Spec.Template.Spec.Volumes = append(r.resource.Spec.Template.Spec.Volumes, corev1.Volume{})
+		index = len(r.resource.Spec.Template.Spec.Volumes) - 1
+	}
+
+	r.resource.Spec.Template.Spec.Volumes[index].Name = konnectivityUDSVolume
+	r.resource.Spec.Template.Spec.Volumes[index].VolumeSource = corev1.VolumeSource{
+		EmptyDir: &corev1.EmptyDirVolumeSource{
+			Medium: "Memory",
+		},
+	}
+	// Defining volumes for the egress selector configuration
+	found, index = utilities.HasNamedVolume(r.resource.Spec.Template.Spec.Volumes, egressSelectorConfigurationVolume)
+	if !found {
+		r.resource.Spec.Template.Spec.Volumes = append(r.resource.Spec.Template.Spec.Volumes, corev1.Volume{})
+		index = len(r.resource.Spec.Template.Spec.Volumes) - 1
+	}
+
+	r.resource.Spec.Template.Spec.Volumes[index].Name = egressSelectorConfigurationVolume
+	r.resource.Spec.Template.Spec.Volumes[index].VolumeSource = corev1.VolumeSource{
+		ConfigMap: &corev1.ConfigMapVolumeSource{
+			LocalObjectReference: corev1.LocalObjectReference{
+				Name: tenantControlPlane.Status.Addons.Konnectivity.EgressSelectorConfiguration,
+			},
+			DefaultMode: pointer.Int32Ptr(420),
+		},
+	}
+	// Defining volume for the Konnectivity kubeconfig
+	found, index = utilities.HasNamedVolume(r.resource.Spec.Template.Spec.Volumes, konnectivityServerKubeconfigVolume)
+	if !found {
+		r.resource.Spec.Template.Spec.Volumes = append(r.resource.Spec.Template.Spec.Volumes, corev1.Volume{})
+		index = len(r.resource.Spec.Template.Spec.Volumes) - 1
+	}
+
+	r.resource.Spec.Template.Spec.Volumes[index].Name = konnectivityServerKubeconfigVolume
+	r.resource.Spec.Template.Spec.Volumes[index].VolumeSource = corev1.VolumeSource{
+		Secret: &corev1.SecretVolumeSource{
+			SecretName:  tenantControlPlane.Status.Addons.Konnectivity.Kubeconfig.SecretName,
+			DefaultMode: pointer.Int32Ptr(420),
+		},
+	}
+
+	return nil
+}

internal/resources/konnectivity/egress_selector_configuration_resource.go

@@ -0,0 +1,118 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+package konnectivity
+
+import (
+	"context"
+
+	corev1 "k8s.io/api/core/v1"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	apiserverv1alpha1 "k8s.io/apiserver/pkg/apis/apiserver/v1alpha1"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+	"github.com/clastix/kamaji/internal/utilities"
+)
+
+type EgressSelectorConfigurationResource struct {
+	resource *corev1.ConfigMap
+	Client   client.Client
+	Name     string
+}
+
+func (r *EgressSelectorConfigurationResource) Define(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+	r.resource = &corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      r.getPrefixedName(tenantControlPlane),
+			Namespace: tenantControlPlane.GetNamespace(),
+		},
+	}
+
+	return nil
+}
+
+func (r *EgressSelectorConfigurationResource) ShouldCleanup(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
+	return tenantControlPlane.Spec.Addons.Konnectivity == nil
+}
+
+func (r *EgressSelectorConfigurationResource) CleanUp(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) {
+	if err := r.Client.Delete(ctx, r.resource); err != nil {
+		if !k8serrors.IsNotFound(err) {
+			return false, err
+		}
+
+		return false, nil
+	}
+
+	return true, nil
+}
+
+func (r *EgressSelectorConfigurationResource) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) {
+	return controllerutil.CreateOrUpdate(ctx, r.Client, r.resource, r.mutate(ctx, tenantControlPlane))
+}
+
+func (r *EgressSelectorConfigurationResource) GetName() string {
+	return r.Name
+}
+
+func (r *EgressSelectorConfigurationResource) ShouldStatusBeUpdated(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
+	return tenantControlPlane.Status.Addons.Konnectivity.EgressSelectorConfiguration != r.resource.GetName()
+}
+
+func (r *EgressSelectorConfigurationResource) UpdateTenantControlPlaneStatus(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+	if tenantControlPlane.Spec.Addons.Konnectivity != nil {
+		tenantControlPlane.Status.Addons.Konnectivity.Enabled = true
+		tenantControlPlane.Status.Addons.Konnectivity.EgressSelectorConfiguration = r.resource.GetName()
+
+		return nil
+	}
+
+	tenantControlPlane.Status.Addons.Konnectivity.Enabled = true
+	tenantControlPlane.Status.Addons.Konnectivity.EgressSelectorConfiguration = ""
+
+	return nil
+}
+
+func (r *EgressSelectorConfigurationResource) mutate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) func() error {
+	return func() error {
+		r.resource.SetLabels(utilities.MergeMaps(r.resource.GetLabels(), utilities.KamajiLabels()))
+
+		configuration := &apiserverv1alpha1.EgressSelectorConfiguration{
+			TypeMeta: metav1.TypeMeta{
+				Kind:       egressSelectorConfigurationKind,
+				APIVersion: apiServerAPIVersion,
+			},
+			EgressSelections: []apiserverv1alpha1.EgressSelection{
+				{
+					Name: egressSelectorConfigurationName,
+					Connection: apiserverv1alpha1.Connection{
+						ProxyProtocol: apiserverv1alpha1.ProtocolGRPC,
+						Transport: &apiserverv1alpha1.Transport{
+							UDS: &apiserverv1alpha1.UDSTransport{
+								UDSName: defaultUDSName,
+							},
+						},
+					},
+				},
+			},
+		}
+
+		yamlConfiguration, err := utilities.EncondeToYaml(configuration)
+		if err != nil {
+			return err
+		}
+
+		r.resource.Data = map[string]string{
+			"egress-selector-configuration.yaml": string(yamlConfiguration),
+		}
+
+		return ctrl.SetControllerReference(tenantControlPlane, r.resource, r.Client.Scheme())
+	}
+}
+
+func (r *EgressSelectorConfigurationResource) getPrefixedName(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) string {
+	return utilities.AddTenantPrefix(r.Name, tenantControlPlane)
+}