docs: releasing v0.4.2

Signed-off-by: Dario Tranchitella <dario@tranchitella.eu>

.github/workflows/ci.yaml

@@ -9,29 +9,29 @@ on:
 jobs:
   golangci:
     name: lint
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-go@v3
         with:
-          go-version: '1.18'
+          go-version: '1.21'
           check-latest: true
       - name: Run golangci-lint
         uses: golangci/golangci-lint-action@v3.2.0
         with:
-          version: v1.49.0
+          version: v1.54.2
           only-new-issues: false
           args: --timeout 5m --config .golangci.yml
   diff:
     name: diff
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v2
         with:
           fetch-depth: 0
       - uses: actions/setup-go@v3
         with:
-          go-version: '1.18'
+          go-version: '1.21'
           check-latest: true
       - run: make yaml-installation-file
       - name: Checking if YAML installer file is not aligned
@@ -40,3 +40,8 @@ jobs:
         run: test -z "$(git ls-files --others --exclude-standard 2> /dev/null)"
       - name: Checking if source code is not formatted
         run: test -z "$(git diff 2> /dev/null)"
+      - run: make apidoc
+      - name: Checking if generated API documentation files are not aligned
+        run: if [[ $(git diff | wc -l) -gt 0 ]]; then echo ">>> Untracked generated files have not been committed" && git --no-pager diff && exit 1; fi
+      - name: Checking if generated API documentation generated untracked files
+        run: test -z "$(git ls-files --others --exclude-standard 2> /dev/null)"

.github/workflows/docker-ci.yml

@@ -7,11 +7,30 @@ on:
 
 jobs:
   docker-ci:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     steps:
 
       - name: Checkout
         uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: Generate build-args
+        id: build-args
+        run: |
+          # Declare vars for internal use
+          VERSION=$(make get_version)
+          GIT_HEAD_COMMIT=$(git rev-parse --short HEAD)
+          GIT_TAG_COMMIT=$(git rev-parse --short $VERSION)
+          GIT_MODIFIED_1=$(git diff $GIT_HEAD_COMMIT $GIT_TAG_COMMIT --quiet && echo "" || echo ".dev")
+          GIT_MODIFIED_2=$(git diff --quiet && echo "" || echo ".dirty")
+          # Export to GH_ENV
+          echo "GIT_LAST_TAG=$VERSION" >> $GITHUB_ENV
+          echo "GIT_HEAD_COMMIT=$GIT_HEAD_COMMIT" >> $GITHUB_ENV
+          echo "GIT_TAG_COMMIT=$GIT_TAG_COMMIT" >> $GITHUB_ENV
+          echo "GIT_MODIFIED=$(echo "$GIT_MODIFIED_1""$GIT_MODIFIED_2")" >> $GITHUB_ENV
+          echo "GIT_REPO=$(git config --get remote.origin.url)" >> $GITHUB_ENV
+          echo "BUILD_DATE=$(git log -1 --format="%at" | xargs -I{} date -d @{} +%Y-%m-%dT%H:%M:%S)" >> $GITHUB_ENV
 
       - name: Docker meta
         id: meta
@@ -68,7 +87,13 @@ jobs:
           platforms: linux/amd64,linux/arm64,linux/arm
           push: true
           tags: ${{ steps.meta.outputs.tags }}
-          build-args:
+          build-args: |
+            GIT_LAST_TAG=${{ env.GIT_LAST_TAG }}
+            GIT_HEAD_COMMIT=${{ env.GIT_HEAD_COMMIT }}
+            GIT_TAG_COMMIT=${{ env.GIT_TAG_COMMIT }}
+            GIT_MODIFIED=${{ env.GIT_MODIFIED }}
+            GIT_REPO=${{ env.GIT_REPO }}
+            BUILD_DATE=${{ env.BUILD_DATE }}
 
       - name: Image digest
         run: echo ${{ steps.build-release.outputs.digest }}

.github/workflows/e2e.yaml

@@ -13,6 +13,7 @@ on:
       - 'main.go'
       - 'Makefile'
       - 'internal/**'
+      - 'cmd/**'
   pull_request:
     branches: [ "*" ]
     paths:
@@ -25,18 +26,19 @@ on:
       - 'main.go'
       - 'Makefile'
       - 'internal/**'
+      - 'cmd/**'
 
 jobs:
   kind:
     name: Kubernetes
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v2
         with:
           fetch-depth: 0
       - uses: actions/setup-go@v3
         with:
-          go-version: '1.18'
+          go-version: '1.21'
           check-latest: true
       - run: |
           sudo apt-get update

.github/workflows/helm.yaml

@@ -10,7 +10,7 @@ on:
 jobs:
   diff:
     name: diff
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v2
         with:
@@ -19,7 +19,7 @@ jobs:
       - name: Checking if Helm docs is not aligned
         run: if [[ $(git diff | wc -l) -gt 0 ]]; then echo ">>> Untracked changes have not been committed" && git --no-pager diff && exit 1; fi
   lint:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v2
       - uses: azure/setup-helm@v1
@@ -29,7 +29,7 @@ jobs:
         run: helm lint ./charts/kamaji
   release:
     if: startsWith(github.ref, 'refs/tags/helm-v')
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v2
       - name: Publish Helm chart

.gitignore

@@ -24,9 +24,13 @@ bin
 *~
 .vscode
 
+# Tilt files.
+.tiltbuild
+
 **/*.kubeconfig
 **/*.crt
 **/*.key
 **/*.pem
 **/*.csr
+**/server-csr.json
 .DS_Store

.golangci.yml

@@ -11,6 +11,7 @@ linters-settings:
 
 linters:
   disable:
+    - depguard
     - wrapcheck
     - gomnd
     - scopelint

ADOPTERS.md

@@ -0,0 +1,21 @@
+# Adopters
+
+This is a list of companies that have adopted Kamaji.
+Feel free to open a Pull-Request to get yours listed.
+
+### Adopter list (alphabetically)
+
+| Type | Name | Since | Website | Use-Case |
+|:-|:-|:-|:-|:-|
+| End-user | KINX | 2024 | [link](https://kinx.net/?lang=en) | KINX is an Internet infrastructure service provider and will use kamaji for its new [Managed Kubernetes Service](https://kinx.net/service/cloud/kubernetes/intro/?lang=en). |
+| End-user | sevensphere | 2023 | [link](https://www.sevensphere.io) | Sevensphere provides consulting services for end-user companies / cloud providers and uses Kamaji for designing cloud/on-premises Kubernetes-as-a-Service platform. |
+| Vendor | Ænix | 2023 | [link](https://aenix.io/) | Ænix provides consulting services for cloud providers and uses Kamaji for running Kubernetes-as-a-Service in free PaaS platform [Cozystack](https://cozystack.io). |
+| Vendor | Netsons | 2023 | [link](https://www.netsons.com) | Netsons is an Italian hosting and cloud provider and uses Kamaji in its [Managed Kubernetes](https://www.netsons.com/kubernetes) offering. |
+
+### Adopter Types
+
+**End-user**: The organization runs Kamaji in production in some way.
+
+**Integration**: The organization has a product that integrates with Kamaji, but does not contain Kamaji.
+
+**Vendor**: The organization packages Kamaji in their product and sells it as part of their product.

Dockerfile

@@ -1,13 +1,5 @@
 # Build the manager binary
-FROM golang:1.18 as builder
-
-ARG TARGETARCH
-ARG GIT_HEAD_COMMIT
-ARG GIT_TAG_COMMIT
-ARG GIT_LAST_TAG
-ARG GIT_MODIFIED
-ARG GIT_REPO
-ARG BUILD_DATE
+FROM golang:1.21 as builder
 
 WORKDIR /workspace
 # Copy the Go Modules manifests
@@ -19,22 +11,30 @@ RUN go mod download
 
 # Copy the go source
 COPY main.go main.go
+COPY cmd/ cmd/
 COPY api/ api/
 COPY controllers/ controllers/
 COPY internal/ internal/
 COPY indexers/ indexers/
 
 # Build
+ARG TARGETARCH
+ARG GIT_HEAD_COMMIT
+ARG GIT_TAG_COMMIT
+ARG GIT_LAST_TAG
+ARG GIT_MODIFIED
+ARG GIT_REPO
+ARG BUILD_DATE
+
 RUN CGO_ENABLED=0 GOOS=linux GOARCH=$TARGETARCH go build \
     -ldflags "-X github.com/clastix/kamaji/internal.GitRepo=$GIT_REPO -X github.com/clastix/kamaji/internal.GitTag=$GIT_LAST_TAG -X github.com/clastix/kamaji/internal.GitCommit=$GIT_HEAD_COMMIT -X github.com/clastix/kamaji/internal.GitDirty=$GIT_MODIFIED -X github.com/clastix/kamaji/internal.BuildTime=$BUILD_DATE" \
-    -a -o manager main.go
+    -a -o kamaji main.go
 
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details
 FROM gcr.io/distroless/static:nonroot
 WORKDIR /
-COPY --from=builder /workspace/manager .
-COPY ./kamaji.yaml .
+COPY --from=builder /workspace/kamaji .
 USER 65532:65532
 
-ENTRYPOINT ["/manager"]
+ENTRYPOINT ["/kamaji"]

Makefile

@@ -3,7 +3,7 @@
 # To re-generate a bundle for another specific version without changing the standard setup, you can:
 # - use the VERSION as arg of the bundle target (e.g make bundle VERSION=0.0.2)
 # - use environment variables to overwrite this value (e.g export VERSION=0.0.2)
-VERSION ?= 0.1.1
+VERSION ?= 0.4.2
 
 # CHANNELS define the bundle channels used in the bundle.
 # Add a new line here if you would like to change its default config. (E.g CHANNELS = "candidate,fast,stable")
@@ -77,7 +77,7 @@ helm: ## Download helm locally if necessary.
 
 GINKGO = $(shell pwd)/bin/ginkgo
 ginkgo: ## Download ginkgo locally if necessary.
-	$(call go-install-tool,$(GINKGO),github.com/onsi/ginkgo/ginkgo@v1.16.5)
+	$(call go-install-tool,$(GINKGO),github.com/onsi/ginkgo/v2/ginkgo@v2.6.0)
 
 KIND = $(shell pwd)/bin/kind
 kind: ## Download kind locally if necessary.
@@ -85,11 +85,11 @@ kind: ## Download kind locally if necessary.
 
 CONTROLLER_GEN = $(shell pwd)/bin/controller-gen
 controller-gen: ## Download controller-gen locally if necessary.
-	$(call go-install-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.9.2)
+	$(call go-install-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.11.4)
 
 GOLANGCI_LINT = $(shell pwd)/bin/golangci-lint
 golangci-lint: ## Download golangci-lint locally if necessary.
-	$(call go-install-tool,$(GOLANGCI_LINT),github.com/golangci/golangci-lint/cmd/golangci-lint@v1.49.0)
+	$(call go-install-tool,$(GOLANGCI_LINT),github.com/golangci/golangci-lint/cmd/golangci-lint@v1.54.2)
 
 KUSTOMIZE = $(shell pwd)/bin/kustomize
 kustomize: ## Download kustomize locally if necessary.
@@ -103,8 +103,6 @@ apidocs-gen: ## Download crdoc locally if necessary.
 
 manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects.
 	$(CONTROLLER_GEN) rbac:roleName=manager-role crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases
-	cp config/crd/bases/kamaji.clastix.io_tenantcontrolplanes.yaml charts/kamaji/crds/tenantcontrolplane.yaml
-	cp config/crd/bases/kamaji.clastix.io_datastores.yaml charts/kamaji/crds/datastore.yaml
 
 generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations.
 	$(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..."
@@ -115,17 +113,51 @@ golint: golangci-lint ## Linting the code according to the styling guide.
 test:
 	go test ./... -coverprofile cover.out
 
+_datastore-mysql:
+	$(MAKE) NAME=$(NAME) -C deploy/kine/mysql mariadb
+	kubectl apply -f $(shell pwd)/config/samples/kamaji_v1alpha1_datastore_mysql_$(NAME).yaml
+
+datastore-mysql:
+	$(MAKE) NAME=bronze _datastore-mysql
+	$(MAKE) NAME=silver _datastore-mysql
+	$(MAKE) NAME=gold _datastore-mysql
+
+_datastore-postgres:
+	$(MAKE) NAME=$(NAME) NAMESPACE=postgres-system -C deploy/kine/postgresql postgresql
+	kubectl apply -f $(shell pwd)/config/samples/kamaji_v1alpha1_datastore_postgresql_$(NAME).yaml
+
+datastore-postgres:
+	$(MAKE) NAME=bronze _datastore-postgres
+	$(MAKE) NAME=silver _datastore-postgres
+	$(MAKE) NAME=gold _datastore-postgres
+
+_datastore-etcd:
+	$(HELM) upgrade --install etcd-$(NAME) clastix/kamaji-etcd --create-namespace -n etcd-system --set datastore.enabled=true
+
+datastore-etcd: helm
+	$(HELM) repo add clastix https://clastix.github.io/charts
+	$(HELM) repo update
+	$(MAKE) NAME=bronze _datastore-etcd
+	$(MAKE) NAME=silver _datastore-etcd
+	$(MAKE) NAME=gold _datastore-etcd
+
+datastores: datastore-mysql datastore-etcd datastore-postgres ## Install all Kamaji DataStores with multiple drivers, and different tiers.
+
 ##@ Build
 
 # Get information about git current status
 GIT_HEAD_COMMIT ?= $$(git rev-parse --short HEAD)
-GIT_TAG_COMMIT  ?= $$(git rev-parse --short $(VERSION))
+GIT_TAG_COMMIT  ?= $$(git rev-parse --short v$(VERSION))
 GIT_MODIFIED_1  ?= $$(git diff $(GIT_HEAD_COMMIT) $(GIT_TAG_COMMIT) --quiet && echo "" || echo ".dev")
 GIT_MODIFIED_2  ?= $$(git diff --quiet && echo "" || echo ".dirty")
 GIT_MODIFIED    ?= $$(echo "$(GIT_MODIFIED_1)$(GIT_MODIFIED_2)")
 GIT_REPO        ?= $$(git config --get remote.origin.url)
 BUILD_DATE      ?= $$(git log -1 --format="%at" | xargs -I{} date -d @{} +%Y-%m-%dT%H:%M:%S)
 
+.PHONY: get_version
+get_version:
+	@echo -n v$(VERSION)
+
 build: generate fmt vet ## Build manager binary.
 	go build -o bin/manager main.go
 
@@ -137,7 +169,7 @@ docker-build: ## Build docker image with the manager.
 		--build-arg GIT_TAG_COMMIT=$(GIT_TAG_COMMIT) \
 		--build-arg GIT_MODIFIED=$(GIT_MODIFIED) \
 		--build-arg GIT_REPO=$(GIT_REPO) \
-		--build-arg GIT_LAST_TAG=$(VERSION) \
+		--build-arg GIT_LAST_TAG=v$(VERSION) \
 		--build-arg BUILD_DATE=$(BUILD_DATE)
 
 docker-push: ## Push docker image with the manager.
@@ -145,6 +177,15 @@ docker-push: ## Push docker image with the manager.
 
 ##@ Deployment
 
+metallb:
+	kubectl apply -f https://raw.githubusercontent.com/metallb/metallb/v0.13.7/config/manifests/metallb-native.yaml
+	kubectl apply -f https://kind.sigs.k8s.io/examples/loadbalancer/metallb-config.yaml
+	echo ""
+	docker network inspect -f '{{.IPAM.Config}}' kind
+
+cert-manager:
+	kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.10.1/cert-manager.yaml
+
 dev: generate manifests uninstall install rbac ## Full installation for development purposes
 	go fmt ./...
 
@@ -253,8 +294,9 @@ env:
 ##@ e2e
 
 .PHONY: e2e
-e2e: env load helm ginkgo ## Create a KinD cluster, install Kamaji on it and run the test suite.
+e2e: env load helm ginkgo cert-manager ## Create a KinD cluster, install Kamaji on it and run the test suite.
 	$(HELM) upgrade --debug --install kamaji ./charts/kamaji --create-namespace --namespace kamaji-system --set "image.pullPolicy=Never"
+	$(MAKE) datastores
 	$(GINKGO) -v ./e2e
 
 ##@ Document

PROJECT

@@ -18,7 +18,6 @@ resources:
   version: v1alpha1
 - api:
     crdVersion: v1
-    namespaced: false
   domain: clastix.io
   group: kamaji
   kind: DataStore

README.md

@@ -3,62 +3,158 @@
 <p align="left">
   <img src="https://img.shields.io/github/license/clastix/kamaji"/>
   <img src="https://img.shields.io/github/go-mod/go-version/clastix/kamaji"/>
-  <a href="https://github.com/clastix/kamaji/releases">
-    <img src="https://img.shields.io/github/v/release/clastix/kamaji"/>
-  </a>
+  <a href="https://github.com/clastix/kamaji/releases"><img src="https://img.shields.io/github/v/release/clastix/kamaji"/></a>
+  <img src="https://goreportcard.com/badge/github.com/clastix/kamaji">
+  <a href="https://kubernetes.slack.com/archives/C03GLTTMWNN"><img alt="#kamaji on Kubernetes Slack" src="https://img.shields.io/badge/slack-@kubernetes/kamaji-blue.svg?logo=slack"/></a>
 </p>
 
-**Kamaji** deploys and operates **Kubernetes** at scale with a fraction of the operational burden.
+![Logo](assets/logo-black.png#gh-light-mode-only)
+![Logo](assets/logo-white.png#gh-dark-mode-only)
 
-<p align="center" style="padding: 6px 6px">
-  <img src="assets/kamaji-logo.png" />
-</p>
+### 🤔 What is Kamaji?
 
-## Why we are building it?
-Global hyper-scalers are leading the Managed Kubernetes space, while other cloud providers, as well as large corporations, are struggling to offer the same experience to their DevOps teams because of the lack of the right tools. Also, current Kubernetes solutions are mainly designed with an enterprise-first approach and they are too costly when deployed at scale.
+**Kamaji** is a **Kubernetes Control Plane Manager** leveraging on the concept of [**Hosted Control Plane**](https://clastix.io/post/the-raise-of-hosted-control-plane-in-kubernetes/).
 
-**Kamaji** aims to solve these pains by leveraging multi-tenancy and simplifying how to run multiple control planes on the same infrastructure with a fraction of the operational burden.
+Kamaji's approach is based on running the Kubernetes Control Plane components in Pods instead of dedicated machines.
+This allows operating Kubernetes clusters at scale, with a fraction of the operational burden.
+Thanks to this approach, running multiple Control Planes can be cheaper and easier to deploy and operate.
 
-## How it works
-Kamaji turns any Kubernetes cluster into an _“admin cluster”_ to orchestrate other Kubernetes clusters called _“tenant clusters”_. What makes Kamaji special is that Control Planes of _“tenant clusters”_ are just regular pods running in the _“admin cluster”_ instead of dedicated Virtual Machines. This solution makes running control planes at scale cheaper and easier to deploy and operate.
+_Kamaji is like a fleet of Site Reliability Engineers with expertise codified into its logic, working 24/7 to keep up and running your Control Planes._
 
-![Architecture](docs/content/images/kamaji-light.png#gh-light-mode-only)
-![Architecture](docs/content/images/kamaji-dark.png#gh-dark-mode-only)
+<img src="docs/content/images/architecture.png"  width="600" style="display: block; margin: 0 auto">
 
-## Getting started
+### 📖 How it works
 
-Please refer to the [Getting Started guide](https://kamaji.clastix.io/getting-started/) to deploy a minimal setup of Kamaji on KinD.
+Kamaji is extending the Kubernetes API capabilities thanks to [Custom Resource Definitions](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/#customresourcedefinitions).
 
-## Features
+By installing Kamaji, two pairs of new APIs will be available:
 
-- **Self Service Kubernetes:** leave users the freedom to self-provision their Kubernetes clusters according to the assigned boundaries.
-- **Multi-cluster Management:** centrally manage multiple tenant clusters from a single admin cluster. Happy SREs. 
-- **Cheaper Control Planes:** place multiple tenant control planes on a single node, instead of having three nodes for a single control plane.
-- **Stronger Multi-Tenancy:** leave tenants to access the control plane with admin permissions while keeping the tenant isolated at the infrastructure level.
-- **Kubernetes Inception:** use Kubernetes to manage Kubernetes by re-using all the Kubernetes goodies you already know and love.
-- **Full APIs compliant:** tenant clusters are fully CNCF compliant built with upstream Kubernetes binaries. A user does not see differences between a Kamaji provisioned cluster and a dedicated cluster.
+- `TenantControlPlane`, the instance definition of your desired Kubernetes Control Plane
+- `Datastore`, the backing store used by one (or more) `TenantControlPlane`
 
-## Roadmap
+The `TenantControlPlane` (short-named as `tcp`) objects are Namespace-scoped and allows configuring every aspect of your desired Control Plane.
+Besides the Kubernetes configuration values, you can specify the Pod options such as limit, request, tolerations, node selector, etc.,
+as well as how these should be exposed (e.g.: using a `ClusterIP`, a `LoadBalancer`, or a `NodePort`).
 
-- [ ] Benchmarking and stress-test
-- [x] Support for dynamic address allocation on native Load Balancer
+The `TenantControlPlane` is the stateless definition of the Control Plane allowing to set up the required components for a full-fledged Kubernetest cluster.
+The state is managed by the `Datastore` API, a cluster-scoped resource which can hold the data of one or more Kubernetes clusters.
+
+> For further information about the API specifications and all the available options,
+> refer to the official [API reference](https://kamaji.clastix.io/reference/api/#tenantcontrolplane).
+
+### ⭐️ Main features
+
+- **Fast provisioning time**: depending on the infrastructure, Tenant Control Planes are up and ready to serve traffic in **16 seconds**.
+- **Streamlined update**: the rollout to a new Kubernetes version for a given Tenant Control Plane takes just **10 seconds**, with a Blue/Green deployment to avoid serving mixed Kubernetes versions.
+- **Resource optimization**: thanks to the Datastore decoupling, there's no need of odd number instances (e.g.: RAFT consensus) by allowing to save up to 60% of HW resources.
+- **Scale from zero to the moon**: scale down the instance when there's no usage, or automatically scale to support the traffic spikes reusing the Kubernetes patterns.
+- **Declarative approach, constant reconciliation**: thanks to the Operator pattern, drift detection happens in real-time, maintaining the desired state.
+- **Automated certificates management**: Kamaji leverages on `kubeadm` and the certificates are automatically created and rotated for you.
+- **Managing core addons**: Kamaji allows configuring automatically `kube-proxy`, `CoreDNS`, and `konnectivity`, with automatic remediation in case of user errors (e.g.: deleting the `CoreDNS` deployment).
+- **Auto Healing**: the `TenantControlPlane` objects in the management cluster are tracked by Kamaji, in case of deletion of those, everything is created in an idempotent way.
+- **Datastore multi-tenancy**: optionally, Kamaji allows running multiple Control Planes on the same _Datastore_ instance leveraging on the multi-tenancy of each driver, decreasing operations and optimizing costs.
+- **Overcoming `etcd` limitations**: optionally, Kamaji allows using a different _Datastore_ thanks to [`kine`](https://github.com/k3s-io/kine) by supporting `MySQL` or `PostgreSQL` as an alternative.
+- **Simplifying mixed-networks setup**: thanks to [`Konnectivity`](https://kubernetes.io/docs/tasks/extend-kubernetes/setup-konnectivity/),
+  the Tenant Control Plane is connected to the worker nodes hosted in a different network, overcoming the no-NAT availability when dealing with nodes with a non routable IP address
+  (e.g.: worker nodes in a different infrastructure).
+
+### 🚀 Use cases
+
+- [**Creating a private Managed Kubernetes Service**](https://clastix.io/post/netsons-builds-a-managed-kubernetes-service-with-kamaji-and-open-stack/)
+- [**Building a Platform as a Service**](https://aenix.io/cozystack/)
+- [**Overcoming public Managed Kubernetes Services**](https://clastix.io/post/overcoming-eks-limitations-with-kamaji-on-aws/) such as EKS
+- [**Hybrid infrastructures**](https://clastix.io/post/bridging-the-gap-hybrid-kubernetes-clusters-with-remote-control-planes/):
+  host the Control Plane on the Cloud and worker nodes on prem or vice-versa, according to your needs.
+- [**Kubernetes at the edge**](https://clastix.io/post/edgevolution-unleashing-the-power-of-kubernetes-clusters-for-a-revolutionary-edge-computing-experience/):
+  take full advantage of the _Kubernetes API Server as a service_ paradigm.
+- **Kubernetes Control Plane as a Service:** centrally manage multiple Kubernetes clusters from a single management point (_Multi-Cluster management_). 
+- **High-density Control Plane:** place multiple control planes on the same infrastructure, instead of having dedicated machines for each control plane.
+- **Strong Multi-tenancy:** leave users to access the control plane with admin permissions while keeping them isolated at the infrastructure level.
+- **Kubernetes Inception:** use Kubernetes to manage Kubernetes with automation, high-availability, fault tolerance, and autoscaling out of the box. 
+- **Bring Your Own Device:** keep the control plane isolated from data plane. Worker nodes can join and run consistently from everywhere: cloud, edge, and data-center.
+- **Full CNCF compliant:** all clusters are built with upstream Kubernetes binaries, resulting in full CNCF compliant Kubernetes clusters.
+
+> 🤔 You'd like to do the same but don't know how?
+> 💡 [CLASTIX](https://clastix.io/) can help you with your needs!
+
+### 🧑‍💻‍ Production grade
+
+Kamaji is empowering several businesses, and it counts public adopters.
+Check out the [adopters](./ADOPTERS.md) file to learn more.
+
+> 🤗 If you're using Kamaji, share your love by opening a PR!
+
+### 🍦 Vanilla Kubernetes clusters
+
+Kamaji is **not** yet-another-Kubernetes distribution: you have full freedom on the technology stack to provide to end users.
+Kamaji is a perfect fit for Platform Engineering, hiding the complexity of the Control Plane management to developers and DevOps engineers.
+
+The provided Kubernetes Control Planes are [CNCF compliant clusters](https://kamaji.clastix.io/reference/conformance/).
+
+<img src="https://raw.githubusercontent.com/cncf/artwork/master/projects/kubernetes/certified-kubernetes/versionless/color/certified-kubernetes-color.png" style="display: block; width: 75px; margin: 0 auto">
+
+### 🐢 Cluster API support
+
+Kamaji is **not** a [Cluster API](https://cluster-api.sigs.k8s.io/) replacement, rather, it plays very well with it.
+
+Since Kamaji is just focusing on the Control Plane a [Kamaji's Cluster API Control Plane provider](https://github.com/clastix/cluster-api-control-plane-provider-kamaji) has been developed.
+
+### 🛣️ Roadmap
+
+- [x] Dynamic address on Load Balancer
 - [x] Zero Downtime Tenant Control Plane upgrade
-- [x] `konnectivity` integration
-- [ ] Provisioning of Tenant Control Plane through Cluster APIs
+- [x] [Join worker nodes from anywhere thanks to Konnectivity](https://kamaji.clastix.io/concepts/#konnectivity)
+- [x] [Alternative datastore MySQL and PostgreSQL](https://kamaji.clastix.io/guides/alternative-datastore/)
+- [x] [Pool of multiple datastores](https://kamaji.clastix.io/concepts/#datastores)
+- [x] [Seamless migration between datastores](https://kamaji.clastix.io/guides/datastore-migration/)
+- [ ] Automatic assignment to a datastore
+- [ ] Autoscaling of Tenant Control Plane
+- [x] [Provisioning through Cluster APIs](https://github.com/clastix/cluster-api-control-plane-provider-kamaji)
 - [ ] Terraform provider
-- [ ] Custom Prometheus metrics for monitoring and alerting
-- [x] `kine` integration for MySQL as datastore
-- [x] `kine` integration for PostgreSQL as datastore
-- [x] Pool of multiple datastores
-- [ ] Automatic assigning of Tenant Control Plane to a datastore
-- [ ] Autoscaling of Tenant Control Plane pods
+- [ ] Custom Prometheus metrics
 
+### 🎥 Multimedia
 
-## Documentation
-Please, check the project's [documentation](https://kamaji.clastix.io/) for getting started with Kamaji.
+- Playlist ▶️ [Tutorials and How-Tos by Dario Tranchitella, CLASTIX](https://www.youtube.com/playlist?list=PLjiUjoV4Ws_3pNsUpTXI-KKk731nD2MQY)
+- YouTube ▶️ [Metal³ provisioning with Kamaji Hosted Control Planes by Huy Mai, Ericsson](https://youtu.be/u9sbURj6jXY?t=10536)
+- YouTube ▶️ [Hands-on introduction to Kamaji](https://www.youtube.com/watch?v=HhevxwQWQ88)
+- YouTube ▶️ [Scaling Kubernetes up to 1,000 Control Planes](https://www.youtube.com/watch?v=W_HXRXJh96U)
+- YouTube ▶️ [Equinix, Kamaji, and Cluster API](https://www.youtube.com/watch?v=TLBTqROj_wA)
+- YouTube ▶️ [Rancher & Kamaji: solving multitenancy challenges in the Kubernetes world](https://www.youtube.com/watch?v=VXHNrMmlF8U)
+- YouTube ▶️ [Enabling Self-Service Kubernetes clusters with Kamaji and Paralus](https://www.youtube.com/watch?v=JWA2LwZazM0)
 
-## Contributions
-Kamaji is Open Source with Apache 2 license and any contribution is welcome.
+### 🏷️ Versioning
 
-## Community
-Join the [Kubernetes Slack Workspace](https://slack.k8s.io/) and the [`#kamaji`](https://kubernetes.slack.com/archives/C03GLTTMWNN) channel to meet end-users and contributors.
+Versioning adheres to the [Semantic Versioning](http://semver.org/) principles.
+A full list of the available releases is available in the GitHub repository's [**Release** section](https://github.com/clastix/kamaji/releases).
+
+### 📄 Documentation
+
+Further documentation can be found on the official [Kamaji documentation website](https://kamaji.clastix.io/).
+
+### 🤝 Contributions
+
+Contributions are highly appreciated and very welcomed!
+
+In case of bugs, please, check if the issue has been already opened by checking the [GitHub Issues](https://github.com/clastix/kamaji/issues) section.
+In case it isn't, you can open a new one: a detailed report will help us to replicate it, assess it, and work on a fix.
+
+You can express your intention in working on the fix on your own.
+The commit messages are checked according to the described [semantics](https://github.com/projectcapsule/capsule/blob/main/CONTRIBUTING.md#semantics).
+Commits are used to generate the changelog, and their author will be referenced in it.
+
+In case of **✨ Feature Requests** please use the [Discussion's Feature Request section](https://github.com/clastix/kamaji/discussions/categories/feature-requests).
+
+### 📝 License
+
+The Kamaji Cluster API Control Plane provider is licensed under Apache 2.0.
+The code is provided as-is with no warranties.
+
+### 🛟 Commercial Support
+
+![CLASTIX](https://avatars.githubusercontent.com/u/39170129?s=50&v=4) [CLASTIX](https://clastix.io/) is the commercial company behind Kamaji and the Cluster API Control Plane provider.
+
+If you're looking to run Kamaji in production and would like to learn more, **CLASTIX** can help by offering [Open Source support plans](https://clastix.io/support),
+as well as providing a comprehensive Enterprise Platform named [CLASTIX Enterprise Platform](https://clastix.cloud/), built on top of the Kamaji and [Capsule](https://capsule.clastix.io/) project (now donated to CNCF as a Sandbox project).
+
+Feel free to get in touch with the provided [Contact form](https://clastix.io/contact).

api/v1alpha1/datastore_funcs.go

@@ -30,7 +30,7 @@ func (in *ContentRef) GetContent(ctx context.Context, client client.Client) ([]b
 		return nil, err
 	}
 
-	v, ok := secret.Data[secretRef.KeyPath]
+	v, ok := secret.Data[string(secretRef.KeyPath)]
 	if !ok {
 		return nil, fmt.Errorf("secret %s does not have key %s", namespacedName.String(), secretRef.KeyPath)
 	}

api/v1alpha1/datastore_types.go

@@ -8,7 +8,9 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
-type Driver string //+kubebuilder:validation:Enum=etcd;MySQL;PostgreSQL
+// +kubebuilder:validation:Enum=etcd;MySQL;PostgreSQL
+
+type Driver string
 
 var (
 	EtcdDriver           Driver = "etcd"
@@ -16,13 +18,17 @@ var (
 	KinePostgreSQLDriver Driver = "PostgreSQL"
 )
 
+// +kubebuilder:validation:MinItems=1
+
+type Endpoints []string
+
 // DataStoreSpec defines the desired state of DataStore.
 type DataStoreSpec struct {
 	// The driver to use to connect to the shared datastore.
 	Driver Driver `json:"driver"`
 	// List of the endpoints to connect to the shared datastore.
 	// No need for protocol, just bare IP/FQDN and port.
-	Endpoints []string `json:"endpoints"` //+kubebuilder:validation:MinLength=1
+	Endpoints Endpoints `json:"endpoints"`
 	// In case of authentication enabled for the given data store, specifies the username and password pair.
 	// This value is optional.
 	BasicAuth *BasicAuth `json:"basicAuth,omitempty"`
@@ -62,11 +68,14 @@ type ContentRef struct {
 	SecretRef *SecretReference `json:"secretReference,omitempty"`
 }
 
+// +kubebuilder:validation:MinLength=1
+type secretReferKeyPath string
+
 type SecretReference struct {
 	corev1.SecretReference `json:",inline"`
 	// Name of the key for the given Secret reference where the content is stored.
 	// This value is mandatory.
-	KeyPath string `json:"keyPath"`
+	KeyPath secretReferKeyPath `json:"keyPath"`
 }
 
 // DataStoreStatus defines the observed state of DataStore.

api/v1alpha1/indexer_datastore_usedsecret.go

@@ -0,0 +1,68 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package v1alpha1
+
+import (
+	"context"
+	"fmt"
+
+	controllerruntime "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+const (
+	DatastoreUsedSecretNamespacedNameKey = "secretRef"
+)
+
+type DatastoreUsedSecret struct{}
+
+func (d *DatastoreUsedSecret) SetupWithManager(ctx context.Context, mgr controllerruntime.Manager) error {
+	return mgr.GetFieldIndexer().IndexField(ctx, d.Object(), d.Field(), d.ExtractValue())
+}
+
+func (d *DatastoreUsedSecret) Object() client.Object {
+	return &DataStore{}
+}
+
+func (d *DatastoreUsedSecret) Field() string {
+	return DatastoreUsedSecretNamespacedNameKey
+}
+
+func (d *DatastoreUsedSecret) ExtractValue() client.IndexerFunc {
+	return func(object client.Object) (res []string) {
+		ds := object.(*DataStore) //nolint:forcetypeassert
+
+		if ds.Spec.BasicAuth != nil {
+			if ds.Spec.BasicAuth.Username.SecretRef != nil {
+				res = append(res, d.namespacedName(*ds.Spec.BasicAuth.Username.SecretRef))
+			}
+
+			if ds.Spec.BasicAuth.Password.SecretRef != nil {
+				res = append(res, d.namespacedName(*ds.Spec.BasicAuth.Password.SecretRef))
+			}
+		}
+
+		if ds.Spec.TLSConfig.CertificateAuthority.Certificate.SecretRef != nil {
+			res = append(res, d.namespacedName(*ds.Spec.TLSConfig.CertificateAuthority.Certificate.SecretRef))
+		}
+
+		if ds.Spec.TLSConfig.CertificateAuthority.PrivateKey != nil && ds.Spec.TLSConfig.CertificateAuthority.PrivateKey.SecretRef != nil {
+			res = append(res, d.namespacedName(*ds.Spec.TLSConfig.CertificateAuthority.PrivateKey.SecretRef))
+		}
+
+		if ds.Spec.TLSConfig.ClientCertificate.Certificate.SecretRef != nil {
+			res = append(res, d.namespacedName(*ds.Spec.TLSConfig.ClientCertificate.Certificate.SecretRef))
+		}
+
+		if ds.Spec.TLSConfig.ClientCertificate.PrivateKey.SecretRef != nil {
+			res = append(res, d.namespacedName(*ds.Spec.TLSConfig.ClientCertificate.PrivateKey.SecretRef))
+		}
+
+		return res
+	}
+}
+
+func (d *DatastoreUsedSecret) namespacedName(ref SecretReference) string {
+	return fmt.Sprintf("%s/%s", ref.Namespace, ref.Name)
+}

api/v1alpha1/indexer_tenantcontrolplane_useddatastore.go

@@ -1,15 +1,13 @@
 // Copyright 2022 Clastix Labs
 // SPDX-License-Identifier: Apache-2.0
 
-package indexers
+package v1alpha1
 
 import (
 	"context"
 
 	controllerruntime "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
-
-	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
 )
 
 const (
@@ -19,7 +17,7 @@ const (
 type TenantControlPlaneStatusDataStore struct{}
 
 func (t *TenantControlPlaneStatusDataStore) Object() client.Object {
-	return &kamajiv1alpha1.TenantControlPlane{}
+	return &TenantControlPlane{}
 }
 
 func (t *TenantControlPlaneStatusDataStore) Field() string {
@@ -28,8 +26,7 @@ func (t *TenantControlPlaneStatusDataStore) Field() string {
 
 func (t *TenantControlPlaneStatusDataStore) ExtractValue() client.IndexerFunc {
 	return func(object client.Object) []string {
-		//nolint:forcetypeassert
-		tcp := object.(*kamajiv1alpha1.TenantControlPlane)
+		tcp := object.(*TenantControlPlane) //nolint:forcetypeassert
 
 		return []string{tcp.Status.Storage.DataStoreName}
 	}

api/v1alpha1/tenantcontrolplane_addons_funcs.go

@@ -1,17 +0,0 @@
-// Copyright 2022 Clastix Labs
-// SPDX-License-Identifier: Apache-2.0
-
-package v1alpha1
-
-import (
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-)
-
-func (in AddonStatus) GetChecksum() string {
-	return in.Checksum
-}
-
-func (in *AddonStatus) SetChecksum(checksum string) {
-	in.LastUpdate = metav1.Now()
-	in.Checksum = checksum
-}

api/v1alpha1/tenantcontrolplane_registrysettings.go

@@ -0,0 +1,18 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package v1alpha1
+
+type RegistrySettings struct {
+	// +kubebuilder:default="registry.k8s.io"
+	Registry string `json:"registry,omitempty"`
+	// The tag to append to all the Control Plane container images.
+	// Optional.
+	TagSuffix string `json:"tagSuffix,omitempty"`
+	// +kubebuilder:default="kube-apiserver"
+	APIServerImage string `json:"apiServerImage,omitempty"`
+	// +kubebuilder:default="kube-controller-manager"
+	ControllerManagerImage string `json:"controllerManagerImage,omitempty"`
+	// +kubebuilder:default="kube-scheduler"
+	SchedulerImage string `json:"schedulerImage,omitempty"`
+}

api/v1alpha1/tenantcontrolplane_registrysettings_funcs.go

@@ -0,0 +1,30 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package v1alpha1
+
+import (
+	"fmt"
+)
+
+func (r *RegistrySettings) buildContainerImage(name, tag string) string {
+	image := fmt.Sprintf("%s/%s:%s", r.Registry, name, tag)
+
+	if len(r.TagSuffix) > 0 {
+		image += r.TagSuffix
+	}
+
+	return image
+}
+
+func (r *RegistrySettings) KubeAPIServerImage(version string) string {
+	return r.buildContainerImage(r.APIServerImage, version)
+}
+
+func (r *RegistrySettings) KubeSchedulerImage(version string) string {
+	return r.buildContainerImage(r.SchedulerImage, version)
+}
+
+func (r *RegistrySettings) KubeControllerManagerImage(version string) string {
+	return r.buildContainerImage(r.ControllerManagerImage, version)
+}

api/v1alpha1/tenantcontrolplane_status.go

@@ -112,15 +112,12 @@ type KubeadmPhaseStatus struct {
 
 // KubeadmPhasesStatus contains the status of the different kubeadm phases action.
 type KubeadmPhasesStatus struct {
-	UploadConfigKubeadm KubeadmPhaseStatus `json:"uploadConfigKubeadm"`
-	UploadConfigKubelet KubeadmPhaseStatus `json:"uploadConfigKubelet"`
-	BootstrapToken      KubeadmPhaseStatus `json:"bootstrapToken"`
+	BootstrapToken KubeadmPhaseStatus `json:"bootstrapToken"`
 }
 
 type ExternalKubernetesObjectStatus struct {
 	Name      string `json:"name,omitempty"`
 	Namespace string `json:"namespace,omitempty"`
-	Checksum  string `json:"checksum,omitempty"`
 	// Last time when k8s object was updated
 	LastUpdate metav1.Time `json:"lastUpdate,omitempty"`
 }
@@ -145,7 +142,6 @@ type KonnectivityConfigMap struct {
 // AddonStatus defines the observed state of an Addon.
 type AddonStatus struct {
 	Enabled    bool        `json:"enabled"`
-	Checksum   string      `json:"checksum,omitempty"`
 	LastUpdate metav1.Time `json:"lastUpdate,omitempty"`
 }
 
@@ -187,12 +183,14 @@ type KubernetesStatus struct {
 	Ingress    *KubernetesIngressStatus   `json:"ingress,omitempty"`
 }
 
-// +kubebuilder:validation:Enum=Provisioning;Upgrading;Ready;NotReady
+// +kubebuilder:validation:Enum=Provisioning;CertificateAuthorityRotating;Upgrading;Migrating;Ready;NotReady
 type KubernetesVersionStatus string
 
 var (
 	VersionProvisioning KubernetesVersionStatus = "Provisioning"
+	VersionCARotating   KubernetesVersionStatus = "CertificateAuthorityRotating"
 	VersionUpgrading    KubernetesVersionStatus = "Upgrading"
+	VersionMigrating    KubernetesVersionStatus = "Migrating"
 	VersionReady        KubernetesVersionStatus = "Ready"
 	VersionNotReady     KubernetesVersionStatus = "NotReady"
 )

api/v1alpha1/tenantcontrolplane_types.go

@@ -4,6 +4,7 @@
 package v1alpha1
 
 import (
+	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
@@ -32,7 +33,23 @@ type NetworkProfileSpec struct {
 	DNSServiceIPs []string `json:"dnsServiceIPs,omitempty"`
 }
 
+// +kubebuilder:validation:Enum=Hostname;InternalIP;ExternalIP;InternalDNS;ExternalDNS
+type KubeletPreferredAddressType string
+
+const (
+	NodeHostName    KubeletPreferredAddressType = "Hostname"
+	NodeInternalIP  KubeletPreferredAddressType = "InternalIP"
+	NodeExternalIP  KubeletPreferredAddressType = "ExternalIP"
+	NodeInternalDNS KubeletPreferredAddressType = "InternalDNS"
+	NodeExternalDNS KubeletPreferredAddressType = "ExternalDNS"
+)
+
 type KubeletSpec struct {
+	// Ordered list of the preferred NodeAddressTypes to use for kubelet connections.
+	// Default to Hostname, InternalIP, ExternalIP.
+	// +kubebuilder:default={"Hostname","InternalIP","ExternalIP"}
+	// +kubebuilder:validation:MinItems=1
+	PreferredAddressTypes []KubeletPreferredAddressType `json:"preferredAddressTypes,omitempty"`
 	// CGroupFS defines the  cgroup driver for Kubelet
 	// https://kubernetes.io/docs/tasks/administer-cluster/kubeadm/configure-cgroup-driver/
 	CGroupFS CGroupDriver `json:"cgroupfs,omitempty"`
@@ -80,15 +97,32 @@ type ControlPlaneComponentsResources struct {
 	APIServer         *corev1.ResourceRequirements `json:"apiServer,omitempty"`
 	ControllerManager *corev1.ResourceRequirements `json:"controllerManager,omitempty"`
 	Scheduler         *corev1.ResourceRequirements `json:"scheduler,omitempty"`
+	// Define the kine container resources.
+	// Available only if Kamaji is running using Kine as backing storage.
+	Kine *corev1.ResourceRequirements `json:"kine,omitempty"`
 }
 
 type DeploymentSpec struct {
+	// RegistrySettings allows to override the default images for the given Tenant Control Plane instance.
+	// It could be used to point to a different container registry rather than the public one.
+	// +kubebuilder:default={registry:"registry.k8s.io",apiServerImage:"kube-apiserver",controllerManagerImage:"kube-controller-manager",schedulerImage:"kube-scheduler"}
+	RegistrySettings RegistrySettings `json:"registrySettings,omitempty"`
 	// +kubebuilder:default=2
-	Replicas int32 `json:"replicas,omitempty"`
+	Replicas *int32 `json:"replicas,omitempty"`
 	// NodeSelector is a selector which must be true for the pod to fit on a node.
 	// Selector which must match a node's labels for the pod to be scheduled on that node.
 	// More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/
 	NodeSelector map[string]string `json:"nodeSelector,omitempty"`
+	// RuntimeClassName refers to a RuntimeClass object in the node.k8s.io group, which should be used
+	// to run the Tenant Control Plane pod. If no RuntimeClass resource matches the named class, the pod will not be run.
+	// If unset or empty, the "legacy" RuntimeClass will be used, which is an implicit class with an
+	// empty definition that uses the default runtime handler.
+	// More info: https://git.k8s.io/enhancements/keps/sig-node/585-runtime-class
+	RuntimeClassName string `json:"runtimeClassName,omitempty"`
+	// Strategy describes how to replace existing pods with new ones for the given Tenant Control Plane.
+	// Default value is set to Rolling Update, with a blue/green strategy.
+	// +kubebuilder:default={type:"RollingUpdate",rollingUpdate:{maxUnavailable:0,maxSurge:"100%"}}
+	Strategy appsv1.DeploymentStrategy `json:"strategy,omitempty"`
 	// If specified, the Tenant Control Plane pod's tolerations.
 	// More info: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/
 	Tolerations []corev1.Toleration `json:"tolerations,omitempty"`
@@ -104,9 +138,27 @@ type DeploymentSpec struct {
 	// (kube-apiserver, controller-manager, and scheduler).
 	Resources *ControlPlaneComponentsResources `json:"resources,omitempty"`
 	// ExtraArgs allows adding additional arguments to the Control Plane components,
-	// such as kube-apiserver, controller-manager, and scheduler.
+	// such as kube-apiserver, controller-manager, and scheduler. WARNING - This option
+	// can override existing parameters and cause components to misbehave in unxpected ways.
+	// Only modify if you know what you are doing.
 	ExtraArgs          *ControlPlaneExtraArgs `json:"extraArgs,omitempty"`
 	AdditionalMetadata AdditionalMetadata     `json:"additionalMetadata,omitempty"`
+	// AdditionalInitContainers allows adding additional init containers to the Control Plane deployment.
+	AdditionalInitContainers []corev1.Container `json:"additionalInitContainers,omitempty"`
+	// AdditionalContainers allows adding additional containers to the Control Plane deployment.
+	AdditionalContainers []corev1.Container `json:"additionalContainers,omitempty"`
+	// AdditionalVolumes allows to add additional volumes to the Control Plane deployment.
+	AdditionalVolumes []corev1.Volume `json:"additionalVolumes,omitempty"`
+	// AdditionalVolumeMounts allows to mount an additional volume into each component of the Control Plane
+	// (kube-apiserver, controller-manager, and scheduler).
+	AdditionalVolumeMounts *AdditionalVolumeMounts `json:"additionalVolumeMounts,omitempty"`
+}
+
+// AdditionalVolumeMounts allows mounting additional volumes to the Control Plane components.
+type AdditionalVolumeMounts struct {
+	APIServer         []corev1.VolumeMount `json:"apiServer,omitempty"`
+	ControllerManager []corev1.VolumeMount `json:"controllerManager,omitempty"`
+	Scheduler         []corev1.VolumeMount `json:"scheduler,omitempty"`
 }
 
 // ControlPlaneExtraArgs allows specifying additional arguments to the Control Plane components.
@@ -138,21 +190,42 @@ type ImageOverrideTrait struct {
 	ImageTag string `json:"imageTag,omitempty"`
 }
 
-// KonnectivitySpec defines the spec for Konnectivity.
-type KonnectivitySpec struct {
-	// Port of Konnectivity proxy server.
-	ProxyPort int32 `json:"proxyPort"`
-	// Version for Konnectivity server and agent.
+// ExtraArgs allows adding additional arguments to said component.
+// WARNING - This option can override existing konnectivity
+// parameters and cause konnectivity components to misbehave in
+// unxpected ways. Only modify if you know what you are doing.
+type ExtraArgs []string
+
+type KonnectivityServerSpec struct {
+	// The port which Konnectivity server is listening to.
+	Port int32 `json:"port"`
+	// Container image version of the Konnectivity server.
 	// +kubebuilder:default=v0.0.32
 	Version string `json:"version,omitempty"`
-	// ServerImage defines the container image for Konnectivity's server.
+	// Container image used by the Konnectivity server.
 	// +kubebuilder:default=registry.k8s.io/kas-network-proxy/proxy-server
-	ServerImage string `json:"serverImage,omitempty"`
-	// AgentImage defines the container image for Konnectivity's agent.
-	// +kubebuilder:default=registry.k8s.io/kas-network-proxy/proxy-agent
-	AgentImage string `json:"agentImage,omitempty"`
+	Image string `json:"image,omitempty"`
 	// Resources define the amount of CPU and memory to allocate to the Konnectivity server.
 	Resources *corev1.ResourceRequirements `json:"resources,omitempty"`
+	ExtraArgs ExtraArgs                    `json:"extraArgs,omitempty"`
+}
+
+type KonnectivityAgentSpec struct {
+	// AgentImage defines the container image for Konnectivity's agent.
+	// +kubebuilder:default=registry.k8s.io/kas-network-proxy/proxy-agent
+	Image string `json:"image,omitempty"`
+	// Version for Konnectivity agent.
+	// +kubebuilder:default=v0.0.32
+	Version   string    `json:"version,omitempty"`
+	ExtraArgs ExtraArgs `json:"extraArgs,omitempty"`
+}
+
+// KonnectivitySpec defines the spec for Konnectivity.
+type KonnectivitySpec struct {
+	// +kubebuilder:default={version:"v0.0.32",image:"registry.k8s.io/kas-network-proxy/proxy-server",port:8132}
+	KonnectivityServerSpec KonnectivityServerSpec `json:"server,omitempty"`
+	// +kubebuilder:default={version:"v0.0.32",image:"registry.k8s.io/kas-network-proxy/proxy-agent"}
+	KonnectivityAgentSpec KonnectivityAgentSpec `json:"agent,omitempty"`
 }
 
 // AddonsSpec defines the enabled addons and their features.
@@ -187,9 +260,10 @@ type TenantControlPlaneSpec struct {
 // +kubebuilder:subresource:scale:specpath=.spec.controlPlane.deployment.replicas,statuspath=.status.kubernetesResources.deployment.replicas,selectorpath=.status.kubernetesResources.deployment.selector
 // +kubebuilder:resource:shortName=tcp
 // +kubebuilder:printcolumn:name="Version",type="string",JSONPath=".spec.kubernetes.version",description="Kubernetes version"
-// +kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.kubernetesResources.version.status",description="Kubernetes version"
-// +kubebuilder:printcolumn:name="Control-Plane-Endpoint",type="string",JSONPath=".status.controlPlaneEndpoint",description="Tenant Control Plane Endpoint (API server)"
+// +kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.kubernetesResources.version.status",description="Status"
+// +kubebuilder:printcolumn:name="Control-Plane endpoint",type="string",JSONPath=".status.controlPlaneEndpoint",description="Tenant Control Plane Endpoint (API server)"
 // +kubebuilder:printcolumn:name="Kubeconfig",type="string",JSONPath=".status.kubeconfig.admin.secretName",description="Secret which contains admin kubeconfig"
+//+kubebuilder:printcolumn:name="Datastore",type="string",JSONPath=".status.storage.dataStoreName",description="DataStore actually used"
 // +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="Age"
 
 // TenantControlPlane is the Schema for the tenantcontrolplanes API.

api/v1alpha1/zz_generated.deepcopy.go

@@ -58,6 +58,42 @@ func (in *AdditionalMetadata) DeepCopy() *AdditionalMetadata {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *AdditionalVolumeMounts) DeepCopyInto(out *AdditionalVolumeMounts) {
+	*out = *in
+	if in.APIServer != nil {
+		in, out := &in.APIServer, &out.APIServer
+		*out = make([]v1.VolumeMount, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+	if in.ControllerManager != nil {
+		in, out := &in.ControllerManager, &out.ControllerManager
+		*out = make([]v1.VolumeMount, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+	if in.Scheduler != nil {
+		in, out := &in.Scheduler, &out.Scheduler
+		*out = make([]v1.VolumeMount, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AdditionalVolumeMounts.
+func (in *AdditionalVolumeMounts) DeepCopy() *AdditionalVolumeMounts {
+	if in == nil {
+		return nil
+	}
+	out := new(AdditionalVolumeMounts)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *AddonSpec) DeepCopyInto(out *AddonSpec) {
 	*out = *in
@@ -319,6 +355,11 @@ func (in *ControlPlaneComponentsResources) DeepCopyInto(out *ControlPlaneCompone
 		*out = new(v1.ResourceRequirements)
 		(*in).DeepCopyInto(*out)
 	}
+	if in.Kine != nil {
+		in, out := &in.Kine, &out.Kine
+		*out = new(v1.ResourceRequirements)
+		(*in).DeepCopyInto(*out)
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ControlPlaneComponentsResources.
@@ -477,7 +518,7 @@ func (in *DataStoreSpec) DeepCopyInto(out *DataStoreSpec) {
 	*out = *in
 	if in.Endpoints != nil {
 		in, out := &in.Endpoints, &out.Endpoints
-		*out = make([]string, len(*in))
+		*out = make(Endpoints, len(*in))
 		copy(*out, *in)
 	}
 	if in.BasicAuth != nil {
@@ -518,9 +559,25 @@ func (in *DataStoreStatus) DeepCopy() *DataStoreStatus {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *DatastoreUsedSecret) DeepCopyInto(out *DatastoreUsedSecret) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DatastoreUsedSecret.
+func (in *DatastoreUsedSecret) DeepCopy() *DatastoreUsedSecret {
+	if in == nil {
+		return nil
+	}
+	out := new(DatastoreUsedSecret)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *DeploymentSpec) DeepCopyInto(out *DeploymentSpec) {
 	*out = *in
+	out.RegistrySettings = in.RegistrySettings
 	if in.NodeSelector != nil {
 		in, out := &in.NodeSelector, &out.NodeSelector
 		*out = make(map[string]string, len(*in))
@@ -528,6 +585,7 @@ func (in *DeploymentSpec) DeepCopyInto(out *DeploymentSpec) {
 			(*out)[key] = val
 		}
 	}
+	in.Strategy.DeepCopyInto(&out.Strategy)
 	if in.Tolerations != nil {
 		in, out := &in.Tolerations, &out.Tolerations
 		*out = make([]v1.Toleration, len(*in))
@@ -558,6 +616,32 @@ func (in *DeploymentSpec) DeepCopyInto(out *DeploymentSpec) {
 		(*in).DeepCopyInto(*out)
 	}
 	in.AdditionalMetadata.DeepCopyInto(&out.AdditionalMetadata)
+	if in.AdditionalInitContainers != nil {
+		in, out := &in.AdditionalInitContainers, &out.AdditionalInitContainers
+		*out = make([]v1.Container, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+	if in.AdditionalContainers != nil {
+		in, out := &in.AdditionalContainers, &out.AdditionalContainers
+		*out = make([]v1.Container, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+	if in.AdditionalVolumes != nil {
+		in, out := &in.AdditionalVolumes, &out.AdditionalVolumes
+		*out = make([]v1.Volume, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+	if in.AdditionalVolumeMounts != nil {
+		in, out := &in.AdditionalVolumeMounts, &out.AdditionalVolumeMounts
+		*out = new(AdditionalVolumeMounts)
+		(*in).DeepCopyInto(*out)
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DeploymentSpec.
@@ -603,6 +687,25 @@ func (in *ETCDCertificatesStatus) DeepCopy() *ETCDCertificatesStatus {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in Endpoints) DeepCopyInto(out *Endpoints) {
+	{
+		in := &in
+		*out = make(Endpoints, len(*in))
+		copy(*out, *in)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Endpoints.
+func (in Endpoints) DeepCopy() Endpoints {
+	if in == nil {
+		return nil
+	}
+	out := new(Endpoints)
+	in.DeepCopyInto(out)
+	return *out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ExternalKubernetesObjectStatus) DeepCopyInto(out *ExternalKubernetesObjectStatus) {
 	*out = *in
@@ -619,6 +722,25 @@ func (in *ExternalKubernetesObjectStatus) DeepCopy() *ExternalKubernetesObjectSt
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in ExtraArgs) DeepCopyInto(out *ExtraArgs) {
+	{
+		in := &in
+		*out = make(ExtraArgs, len(*in))
+		copy(*out, *in)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExtraArgs.
+func (in ExtraArgs) DeepCopy() ExtraArgs {
+	if in == nil {
+		return nil
+	}
+	out := new(ExtraArgs)
+	in.DeepCopyInto(out)
+	return *out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ImageOverrideTrait) DeepCopyInto(out *ImageOverrideTrait) {
 	*out = *in
@@ -650,6 +772,26 @@ func (in *IngressSpec) DeepCopy() *IngressSpec {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *KonnectivityAgentSpec) DeepCopyInto(out *KonnectivityAgentSpec) {
+	*out = *in
+	if in.ExtraArgs != nil {
+		in, out := &in.ExtraArgs, &out.ExtraArgs
+		*out = make(ExtraArgs, len(*in))
+		copy(*out, *in)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KonnectivityAgentSpec.
+func (in *KonnectivityAgentSpec) DeepCopy() *KonnectivityAgentSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(KonnectivityAgentSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *KonnectivityConfigMap) DeepCopyInto(out *KonnectivityConfigMap) {
 	*out = *in
@@ -666,13 +808,35 @@ func (in *KonnectivityConfigMap) DeepCopy() *KonnectivityConfigMap {
 }
 
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *KonnectivitySpec) DeepCopyInto(out *KonnectivitySpec) {
+func (in *KonnectivityServerSpec) DeepCopyInto(out *KonnectivityServerSpec) {
 	*out = *in
 	if in.Resources != nil {
 		in, out := &in.Resources, &out.Resources
 		*out = new(v1.ResourceRequirements)
 		(*in).DeepCopyInto(*out)
 	}
+	if in.ExtraArgs != nil {
+		in, out := &in.ExtraArgs, &out.ExtraArgs
+		*out = make(ExtraArgs, len(*in))
+		copy(*out, *in)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KonnectivityServerSpec.
+func (in *KonnectivityServerSpec) DeepCopy() *KonnectivityServerSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(KonnectivityServerSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *KonnectivitySpec) DeepCopyInto(out *KonnectivitySpec) {
+	*out = *in
+	in.KonnectivityServerSpec.DeepCopyInto(&out.KonnectivityServerSpec)
+	in.KonnectivityAgentSpec.DeepCopyInto(&out.KonnectivityAgentSpec)
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KonnectivitySpec.
@@ -742,8 +906,6 @@ func (in *KubeadmPhaseStatus) DeepCopy() *KubeadmPhaseStatus {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *KubeadmPhasesStatus) DeepCopyInto(out *KubeadmPhasesStatus) {
 	*out = *in
-	in.UploadConfigKubeadm.DeepCopyInto(&out.UploadConfigKubeadm)
-	in.UploadConfigKubelet.DeepCopyInto(&out.UploadConfigKubelet)
 	in.BootstrapToken.DeepCopyInto(&out.BootstrapToken)
 }
 
@@ -794,6 +956,11 @@ func (in *KubeconfigsStatus) DeepCopy() *KubeconfigsStatus {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *KubeletSpec) DeepCopyInto(out *KubeletSpec) {
 	*out = *in
+	if in.PreferredAddressTypes != nil {
+		in, out := &in.PreferredAddressTypes, &out.PreferredAddressTypes
+		*out = make([]KubeletPreferredAddressType, len(*in))
+		copy(*out, *in)
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeletSpec.
@@ -858,7 +1025,7 @@ func (in *KubernetesServiceStatus) DeepCopy() *KubernetesServiceStatus {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *KubernetesSpec) DeepCopyInto(out *KubernetesSpec) {
 	*out = *in
-	out.Kubelet = in.Kubelet
+	in.Kubelet.DeepCopyInto(&out.Kubelet)
 	if in.AdmissionControllers != nil {
 		in, out := &in.AdmissionControllers, &out.AdmissionControllers
 		*out = make(AdmissionControllers, len(*in))
@@ -960,6 +1127,21 @@ func (in *PublicKeyPrivateKeyPairStatus) DeepCopy() *PublicKeyPrivateKeyPairStat
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *RegistrySettings) DeepCopyInto(out *RegistrySettings) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RegistrySettings.
+func (in *RegistrySettings) DeepCopy() *RegistrySettings {
+	if in == nil {
+		return nil
+	}
+	out := new(RegistrySettings)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *SecretReference) DeepCopyInto(out *SecretReference) {
 	*out = *in
@@ -1126,3 +1308,18 @@ func (in *TenantControlPlaneStatus) DeepCopy() *TenantControlPlaneStatus {
 	in.DeepCopyInto(out)
 	return out
 }
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *TenantControlPlaneStatusDataStore) DeepCopyInto(out *TenantControlPlaneStatusDataStore) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TenantControlPlaneStatusDataStore.
+func (in *TenantControlPlaneStatusDataStore) DeepCopy() *TenantControlPlaneStatusDataStore {
+	if in == nil {
+		return nil
+	}
+	out := new(TenantControlPlaneStatusDataStore)
+	in.DeepCopyInto(out)
+	return out
+}

assets/kamaji-logo.svg

@@ -1 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="11.85 8.10 202.80 187.55"><title>Kamaji</title><path d="M32.1 13.7c-2.4.9-6.3 3.5-8.6 5.8-7.7 7.7-7.5 5-7.5 82.5 0 77.4-.2 74.8 7.5 82.5 7.7 7.8 4.2 7.5 90 7.5s82.3.3 90-7.5c7.7-7.7 7.5-5.1 7.5-82.5s.2-74.8-7.5-82.5c-7.8-7.8-4.1-7.5-90.4-7.4-66.7 0-77.2.3-81 1.6zm160.5 9.9c1.9.9 4.4 3.1 5.7 4.8l2.2 3.1v141l-2.2 3.1c-4.8 6.7-1.1 6.4-84.8 6.4s-80 .3-84.8-6.4l-2.2-3.1v-141l2.2-3.1c4.8-6.6.8-6.4 84.6-6.4 68 0 76.3.2 79.3 1.6z"/><path d="M90.1 33.7c-5.1 2.5-7.3 6.7-6.8 13.1.3 4.1 1 5.9 3.3 8.4s2.5 3 .9 2.3c-2-.7-25.1-4.6-29-4.9-1.1 0-2 .5-2 1.4 0 1.1-1.2 1.5-4.9 1.5-6.7 0-6.8 1.9-.4 4 8.2 2.7 9 3.4 3.3 3.5-5.3 0-8.2 1.1-7.1 2.8.7 1.2-2.7 2.2-8.1 2.2-7 0-6.5 2.4 1.1 5.1l3.9 1.4-2.9.5c-4.3.8-3.2 2.3 2.8 4.1l5.3 1.5-5.2 2.7c-8.2 4.2-8.3 5.8-.4 6.1 5.6.2 7.3 1.1 4.2 2.1-2.3.7-2.8 3.1-.9 3.7.7.3-.5 2-2.8 4-5.6 5.3-4 6.4 6.2 4.5 4.4-.8 8.1-1.3 8.3-1.2.2.2-1.3 2.4-3.3 4.8-2 2.4-3.6 4.7-3.6 5.2 0 .4 1.4.5 3 .3 2.9-.4 4 .5 2 1.7-.5.3-1 1.3-1 2.2 0 1.6 2.2 1.5 6.5-.3 1.7-.7 1.6-.2-.9 3-5.4 7.2.7 6.5 13.6-1.4 2.7-1.7 5.1-3 5.4-3 .3 0-.9 2.1-2.7 4.6-4.5 6.6-2.5 7.9 3.7 2.3 4.6-4.3 4.7-4.3 3-1.2-1.9 3.8-2.1 5.6-.4 5.1.6-.2 7.1-7.1 14.3-15.4 7.2-8.2 13.7-14.9 14.5-14.9.8 0 7.3 6.7 14.6 15 7.2 8.2 13.7 15.1 14.3 15.3 1.6.5 1.4-1.4-.5-5-1.6-3.2-1.6-3.2 3.2 1 6 5.1 7.8 4 3.5-2.2-1.8-2.5-3-4.6-2.7-4.6.3 0 2.7 1.3 5.4 3 12.9 7.9 19 8.6 13.6 1.4-2.5-3.2-2.6-3.7-.9-3 5.9 2.5 7.7 1.7 5.6-2.3-.9-1.5-.6-1.7 2-1.3 3.8.6 3.7-.5-.7-5.7-2-2.3-3.5-4.4-3.2-4.6.2-.2 2.1 0 4.3.4 13.9 3 16.4 1.8 9.8-4.3-2.1-1.9-3.2-3.6-2.5-3.6 2 0 1.4-2.8-.9-3.5-3.2-1-1.3-2 4.2-2.1 7.9-.2 7.8-1.9-.4-6.1l-5.2-2.7 5.4-1.6c6.4-1.8 7.9-4 2.9-4.1h-3.3l3.9-1.5c7.3-2.6 8.4-5.4 2.2-5.4-5.1 0-9.6-1.1-9-2.2 1.1-1.7-1.8-2.8-7.1-2.8-5.7-.1-4.9-.8 3.3-3.5 6.4-2.1 6.3-4-.4-4-3.7 0-4.9-.4-4.9-1.5 0-.9-.9-1.4-2-1.4-3.9.3-27 4.2-29 4.9-1.6.7-1.4.2.9-2.3 3.7-4 4.7-11.3 2.2-16.1-4.8-9.2-18.8-9.3-23.8 0-4.4 8.3.2 18.4 9.5 20.5 3 .6 2.8.8-5.5 4l-8.8 3.3-8.7-3.3c-8.1-3.2-8.4-3.4-5.5-4.1 1.7-.3 4.3-1.5 5.7-2.7 13.1-10.3.6-30.4-14.4-23.1zm77.6 98.4c-3.6 2.1-.8 7.7 3.2 6.4 2.1-.6 3.5-3.1 2.5-4.6-1.1-1.8-4-2.7-5.7-1.8zm8.3 3.9c0 1.9.5 2.1 6.3 1.8 4.7-.2 6.2-.7 6.2-1.8s-1.5-1.6-6.2-1.8c-5.8-.3-6.3-.1-6.3 1.8zm-135.6.3c-.2.7-.3 7.4-.2 14.8l.3 13.4 3.3.3c3.1.3 3.2.2 3.2-3.4 0-2.5.7-4.6 2.1-6l2.1-2.3 5 6c3.9 4.7 5.6 5.9 7.8 5.9 1.6 0 3.1-.3 3.3-.8.3-.4-2.1-4-5.4-8.1-3.2-4-5.9-7.6-5.9-8 0-.4 2.5-3.1 5.5-6.1 3-3 5.5-5.8 5.5-6.2 0-.4-1.5-.8-3.3-.8-2.8 0-4.4 1-9.6 6.5-3.5 3.6-6.5 6.5-6.7 6.5-.2 0-.4-2.9-.4-6.5V135h-3c-1.7 0-3.3.6-3.6 1.3zm31.2 7c-1.1.8-1.5 1.9-1 3 .5 1.4 1.3 1.6 4 1.1 4.2-.8 8.4.2 8.4 2 0 .8-1.8 1.5-5.1 1.9-6 .7-8.9 2.9-8.9 6.6 0 3.2.8 4.4 3.7 6 2.9 1.5 5.2 1.4 8.6-.3 2.3-1.3 2.7-1.3 2.7 0 0 .9 1.1 1.4 3 1.4h3v-8.6c0-8.1-.1-8.7-2.9-11.5-2.5-2.5-3.7-2.9-8.3-2.9-3 0-6.2.6-7.2 1.3zm11.2 13.9c-.2 1.7-1.1 2.4-3.2 2.6-3.3.4-5.1-1-4.3-3.2.4-1.1 1.9-1.6 4.2-1.6 3.2 0 3.6.3 3.3 2.2zm13.4-4l.3 11.3h6l.5-7.8c.5-7.6 1.5-9.6 4.7-9.7 3 0 4.3 3.2 4.3 10.6v7.4h3c3 0 3 0 3-5.9 0-7.3 1.2-10.7 4.1-11.6 3.8-1.3 5.9 2.5 5.9 10.6v6.9h6v-9c0-8.3-.2-9.3-2.5-11.5-2.9-3-9.8-3.5-12.7-.8-1.7 1.5-1.9 1.5-3.6 0-2.2-2-9.2-2.3-11.1-.5-1.1 1-1.4 1-1.8 0-.3-.6-1.8-1.2-3.4-1.2h-3l.3 11.2zm45.4-9.9c-1.1.8-1.5 1.9-1 3 .5 1.4 1.3 1.6 4 1.1 4.2-.8 8.4.2 8.4 2 0 .8-1.8 1.5-5.1 1.9-6 .7-8.9 2.9-8.9 6.6 0 3.2.8 4.4 3.7 6 2.9 1.5 5.2 1.4 8.6-.3 2.3-1.3 2.7-1.3 2.7 0 0 .9 1.1 1.4 3 1.4h3v-8.6c0-8.1-.1-8.7-2.9-11.5-2.5-2.5-3.7-2.9-8.3-2.9-3 0-6.2.6-7.2 1.3zm11.2 13.9c-.2 1.7-1.1 2.4-3.2 2.6-3.3.4-5.1-1-4.3-3.2.4-1.1 1.9-1.6 4.2-1.6 3.2 0 3.6.3 3.3 2.2zm13-2.5c-.3 12.8-.3 12.8-2.7 12.8-1.5 0-2.7.8-3.1 2-2 5.4 9.4 4.3 11.9-1.2.6-1.3 1.1-7.7 1.1-14.3v-12h-6.9l-.3 12.7zm13.4-1.5l.3 11.3h6v-22l-3.3-.3-3.3-.3.3 11.3z"/></svg>

charts/kamaji/Chart.yaml

@@ -1,11 +1,9 @@
 apiVersion: v2
-appVersion: v0.1.1
-description: Kamaji is a tool aimed to build and operate a Managed Kubernetes Service
-  with a fraction of the operational burden. With Kamaji, you can deploy and operate
-  hundreds of Kubernetes clusters as a hyper-scaler.
+appVersion: v0.4.2
+description: Kamaji is a Kubernetes Control Plane Manager.
 home: https://github.com/clastix/kamaji
-icon: https://github.com/clastix/kamaji/raw/master/assets/kamaji-logo.png
-kubeVersion: 1.21 - 1.25
+icon: https://github.com/clastix/kamaji/raw/master/assets/logo-colored.png
+kubeVersion: ">=1.21.0-0"
 maintainers:
 - email: dario@tranchitella.eu
   name: Dario Tranchitella
@@ -13,14 +11,12 @@ maintainers:
   name: Massimiliano Giovagnoli
 - email: me@bsctl.io
   name: Adriano Pezzuto
-- email: iam@mendrugory.com
-  name: Gonzalo Gabriel Jiménez Fuentes
 name: kamaji
 sources:
 - https://github.com/clastix/kamaji
 type: application
-version: 0.10.0
+version: 0.15.1
 annotations:
   catalog.cattle.io/certified: partner
   catalog.cattle.io/release-name: kamaji
-  catalog.cattle.io/display-name: Kamaji - Managed Kubernetes Service
+  catalog.cattle.io/display-name: Kamaji

charts/kamaji/README.md

@@ -1,8 +1,8 @@
 # kamaji
 
-![Version: 0.10.0](https://img.shields.io/badge/Version-0.10.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v0.1.1](https://img.shields.io/badge/AppVersion-v0.1.1-informational?style=flat-square)
+![Version: 0.15.1](https://img.shields.io/badge/Version-0.15.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v0.4.2](https://img.shields.io/badge/AppVersion-v0.4.2-informational?style=flat-square)
 
-Kamaji is a tool aimed to build and operate a Managed Kubernetes Service with a fraction of the operational burden. With Kamaji, you can deploy and operate hundreds of Kubernetes clusters as a hyper-scaler.
+Kamaji is a Kubernetes Control Plane Manager.
 
 ## Maintainers
 
@@ -11,7 +11,6 @@ Kamaji is a tool aimed to build and operate a Managed Kubernetes Service with a
 | Dario Tranchitella | <dario@tranchitella.eu> |  |
 | Massimiliano Giovagnoli | <me@maxgio.it> |  |
 | Adriano Pezzuto | <me@bsctl.io> |  |
-| Gonzalo Gabriel Jiménez Fuentes | <iam@mendrugory.com> |  |
 
 ## Source Code
 
@@ -19,7 +18,7 @@ Kamaji is a tool aimed to build and operate a Managed Kubernetes Service with a
 
 ## Requirements
 
-Kubernetes: `1.21 - 1.25`
+Kubernetes: `>=1.21.0-0`
 
 [Kamaji](https://github.com/clastix/kamaji) requires a [multi-tenant `etcd`](https://github.com/clastix/kamaji-internal/blob/master/deploy/getting-started-with-kamaji.md#setup-internal-multi-tenant-etcd) cluster.
 This Helm Chart starting from v0.1.1 provides the installation of an internal `etcd` in order to streamline the local test. If you'd like to use an externally managed etcd instance, you can specify the overrides and by setting the value `etcd.deploy=false`.
@@ -67,7 +66,8 @@ Here the values you can override:
 | Key | Type | Default | Description |
 |-----|------|---------|-------------|
 | affinity | object | `{}` | Kubernetes affinity rules to apply to Kamaji controller pods |
-| configPath | string | `"./kamaji.yaml"` | Configuration file path alternative. (default "./kamaji.yaml") |
+| cfssl.image.repository | string | `"cfssl/cfssl"` |  |
+| cfssl.image.tag | string | `"latest"` |  |
 | datastore.basicAuth.passwordSecret.keyPath | string | `nil` | The Secret key where the data is stored. |
 | datastore.basicAuth.passwordSecret.name | string | `nil` | The name of the Secret containing the password used to connect to the relational database. |
 | datastore.basicAuth.passwordSecret.namespace | string | `nil` | The namespace of the Secret containing the password used to connect to the relational database. |
@@ -75,8 +75,9 @@ Here the values you can override:
 | datastore.basicAuth.usernameSecret.name | string | `nil` | The name of the Secret containing the username used to connect to the relational database. |
 | datastore.basicAuth.usernameSecret.namespace | string | `nil` | The namespace of the Secret containing the username used to connect to the relational database. |
 | datastore.driver | string | `"etcd"` | (string) The Kamaji Datastore driver, supported: etcd, MySQL, PostgreSQL (defaults=etcd). |
+| datastore.enabled | bool | `true` | (bool) Enable the Kamaji Datastore creation (default=true) |
 | datastore.endpoints | list | `[]` | (array) List of endpoints of the selected Datastore. When letting the Chart install the etcd datastore, this field is populated automatically. |
-| datastore.nameOverride | string | `nil` | The Datastore name override, if empty defaults to `default` |
+| datastore.nameOverride | string | `nil` | The Datastore name override, if empty and enabled=true defaults to `default`, if enabled=false, this is the name of the Datastore to connect to.  |
 | datastore.tlsConfig.certificateAuthority.certificate.keyPath | string | `nil` | Key of the Secret which contains the content of the certificate. |
 | datastore.tlsConfig.certificateAuthority.certificate.name | string | `nil` | Name of the Secret containing the CA required to establish the mandatory SSL/TLS connection to the datastore. |
 | datastore.tlsConfig.certificateAuthority.certificate.namespace | string | `nil` | Namespace of the Secret containing the CA required to establish the mandatory SSL/TLS connection to the datastore. |
@@ -91,7 +92,7 @@ Here the values you can override:
 | datastore.tlsConfig.clientCertificate.privateKey.namespace | string | `nil` | Namespace of the Secret containing the client certificate private key required to establish the mandatory SSL/TLS connection to the datastore. |
 | etcd.compactionInterval | int | `0` | ETCD Compaction interval (e.g. "5m0s"). (default: "0" (disabled)) |
 | etcd.deploy | bool | `true` | Install an etcd with enabled multi-tenancy along with Kamaji |
-| etcd.image | object | `{"pullPolicy":"IfNotPresent","repository":"quay.io/coreos/etcd","tag":"v3.5.4"}` | Install specific etcd image |
+| etcd.image | object | `{"pullPolicy":"IfNotPresent","repository":"quay.io/coreos/etcd","tag":"v3.5.6"}` | Install specific etcd image |
 | etcd.livenessProbe | object | `{"failureThreshold":8,"httpGet":{"path":"/health?serializable=true","port":2381,"scheme":"HTTP"},"initialDelaySeconds":10,"periodSeconds":10,"timeoutSeconds":15}` | The livenessProbe for the etcd container |
 | etcd.overrides.caSecret.name | string | `"etcd-certs"` | Name of the secret which contains CA's certificate and private key. (default: "etcd-certs") |
 | etcd.overrides.caSecret.namespace | string | `"kamaji-system"` | Namespace of the secret which contains CA's certificate and private key. (default: "kamaji-system") |
@@ -100,11 +101,13 @@ Here the values you can override:
 | etcd.overrides.endpoints | object | `{"etcd-0":"etcd-0.etcd.kamaji-system.svc.cluster.local","etcd-1":"etcd-1.etcd.kamaji-system.svc.cluster.local","etcd-2":"etcd-2.etcd.kamaji-system.svc.cluster.local"}` | (map) Dictionary of the endpoints for the etcd cluster's members, key is the name of the etcd server. Don't define the protocol (TLS is automatically inflected), or any port, inflected from .etcd.peerApiPort value. |
 | etcd.peerApiPort | int | `2380` | The peer API port which servers are listening to. |
 | etcd.persistence.accessModes[0] | string | `"ReadWriteOnce"` |  |
+| etcd.persistence.customAnnotations | object | `{}` | The custom annotations to add to the PVC |
 | etcd.persistence.size | string | `"10Gi"` |  |
-| etcd.persistence.storageClass | string | `""` |  |
+| etcd.persistence.storageClassName | string | `""` |  |
 | etcd.port | int | `2379` | The client request port. |
 | etcd.serviceAccount.create | bool | `true` | Create a ServiceAccount, required to install and provision the etcd backing storage (default: true) |
 | etcd.serviceAccount.name | string | `""` | Define the ServiceAccount name to use during the setup and provision of the etcd backing storage (default: "") |
+| etcd.tolerations | list | `[]` | (array) Kubernetes affinity rules to apply to Kamaji etcd pods |
 | extraArgs | list | `[]` | A list of extra arguments to add to the kamaji controller default ones |
 | fullnameOverride | string | `""` |  |
 | healthProbeBindAddress | string | `":8081"` | The address the probe endpoint binds to. (default ":8081") |
@@ -126,11 +129,10 @@ Here the values you can override:
 | resources.requests.cpu | string | `"100m"` |  |
 | resources.requests.memory | string | `"20Mi"` |  |
 | securityContext | object | `{"allowPrivilegeEscalation":false}` | The securityContext to apply to the Kamaji controller container only. It does not apply to the Kamaji RBAC proxy container. |
-| service.port | int | `8443` |  |
-| service.type | string | `"ClusterIP"` |  |
 | serviceAccount.annotations | object | `{}` |  |
 | serviceAccount.create | bool | `true` |  |
 | serviceAccount.name | string | `"kamaji-controller-manager"` |  |
+| serviceMonitor.enabled | bool | `false` | Toggle the ServiceMonitor true if you have Prometheus Operator installed and configured |
 | temporaryDirectoryPath | string | `"/tmp/kamaji"` | Directory which will be used to work with temporary files. (default "/tmp/kamaji") |
 | tolerations | list | `[]` | Kubernetes node taints that the Kamaji controller pods would tolerate |
 

charts/kamaji/app-readme.md

@@ -1,30 +1,12 @@
-# Kamaji - Managed Kubernetes Service
+# Kamaji
 
-Kamaji is a tool aimed to build and operate a Managed Kubernetes Service with a fraction of the operational burden.
+Kamaji deploys and operates Kubernetes at scale with a fraction of the operational burden.
 
 Useful links:
 - [Kamaji Github repository](https://github.com/clastix/kamaji)
-- [Kamaji Documentation](https://github.com/clastix/kamaji/docs/)
+- [Kamaji Documentation](https://kamaji.clastix.io)
 
 ## Requirements
 
 * Kubernetes v1.22+
-* Helm v3
-
-# Installation
-
-To install the Chart with the release name `kamaji`:
-
-        helm upgrade --install --namespace kamaji-system --create-namespace clastix/kamaji
-
-Show the status:
-
-        helm status kamaji -n kamaji-system
-
-Upgrade the Chart
-
-        helm upgrade kamaji -n kamaji-system clastix/kamaji
-
-Uninstall the Chart
-
-        helm uninstall kamaji -n kamaji-system
+* Helm v3

charts/kamaji/crds/datastore.yaml

@@ -3,8 +3,8 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.9.2
-  creationTimestamp: null
+    cert-manager.io/inject-ca-from: kamaji-system/kamaji-serving-cert
+    controller-gen.kubebuilder.io/version: v0.11.4
   name: datastores.kamaji.clastix.io
 spec:
   group: kamaji.clastix.io
@@ -15,254 +15,225 @@ spec:
     singular: datastore
   scope: Cluster
   versions:
-  - additionalPrinterColumns:
-    - description: Kamaji data store driver
-      jsonPath: .spec.driver
-      name: Driver
-      type: string
-    - description: Age
-      jsonPath: .metadata.creationTimestamp
-      name: Age
-      type: date
-    name: v1alpha1
-    schema:
-      openAPIV3Schema:
-        description: DataStore is the Schema for the datastores API.
-        properties:
-          apiVersion:
-            description: 'APIVersion defines the versioned schema of this representation
-              of an object. Servers should convert recognized schemas to the latest
-              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
-            type: string
-          kind:
-            description: 'Kind is a string value representing the REST resource this
-              object represents. Servers may infer this from the endpoint the client
-              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
-            type: string
-          metadata:
-            type: object
-          spec:
-            description: DataStoreSpec defines the desired state of DataStore.
-            properties:
-              basicAuth:
-                description: In case of authentication enabled for the given data
-                  store, specifies the username and password pair. This value is optional.
-                properties:
-                  password:
-                    properties:
-                      content:
-                        description: Bare content of the file, base64 encoded. It
-                          has precedence over the SecretReference value.
-                        format: byte
-                        type: string
-                      secretReference:
-                        properties:
-                          keyPath:
-                            description: Name of the key for the given Secret reference
-                              where the content is stored. This value is mandatory.
-                            type: string
-                          name:
-                            description: name is unique within a namespace to reference
-                              a secret resource.
-                            type: string
-                          namespace:
-                            description: namespace defines the space within which
-                              the secret name must be unique.
-                            type: string
-                        required:
-                        - keyPath
-                        type: object
-                        x-kubernetes-map-type: atomic
-                    type: object
-                  username:
-                    properties:
-                      content:
-                        description: Bare content of the file, base64 encoded. It
-                          has precedence over the SecretReference value.
-                        format: byte
-                        type: string
-                      secretReference:
-                        properties:
-                          keyPath:
-                            description: Name of the key for the given Secret reference
-                              where the content is stored. This value is mandatory.
-                            type: string
-                          name:
-                            description: name is unique within a namespace to reference
-                              a secret resource.
-                            type: string
-                          namespace:
-                            description: namespace defines the space within which
-                              the secret name must be unique.
-                            type: string
-                        required:
-                        - keyPath
-                        type: object
-                        x-kubernetes-map-type: atomic
-                    type: object
-                required:
-                - password
-                - username
-                type: object
-              driver:
-                description: The driver to use to connect to the shared datastore.
-                type: string
-              endpoints:
-                description: List of the endpoints to connect to the shared datastore.
-                  No need for protocol, just bare IP/FQDN and port.
-                items:
+    - additionalPrinterColumns:
+        - description: Kamaji data store driver
+          jsonPath: .spec.driver
+          name: Driver
+          type: string
+        - description: Age
+          jsonPath: .metadata.creationTimestamp
+          name: Age
+          type: date
+      name: v1alpha1
+      schema:
+        openAPIV3Schema:
+          description: DataStore is the Schema for the datastores API.
+          properties:
+            apiVersion:
+              description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              type: string
+            kind:
+              description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              type: string
+            metadata:
+              type: object
+            spec:
+              description: DataStoreSpec defines the desired state of DataStore.
+              properties:
+                basicAuth:
+                  description: In case of authentication enabled for the given data store, specifies the username and password pair. This value is optional.
+                  properties:
+                    password:
+                      properties:
+                        content:
+                          description: Bare content of the file, base64 encoded. It has precedence over the SecretReference value.
+                          format: byte
+                          type: string
+                        secretReference:
+                          properties:
+                            keyPath:
+                              description: Name of the key for the given Secret reference where the content is stored. This value is mandatory.
+                              minLength: 1
+                              type: string
+                            name:
+                              description: name is unique within a namespace to reference a secret resource.
+                              type: string
+                            namespace:
+                              description: namespace defines the space within which the secret name must be unique.
+                              type: string
+                          required:
+                            - keyPath
+                          type: object
+                          x-kubernetes-map-type: atomic
+                      type: object
+                    username:
+                      properties:
+                        content:
+                          description: Bare content of the file, base64 encoded. It has precedence over the SecretReference value.
+                          format: byte
+                          type: string
+                        secretReference:
+                          properties:
+                            keyPath:
+                              description: Name of the key for the given Secret reference where the content is stored. This value is mandatory.
+                              minLength: 1
+                              type: string
+                            name:
+                              description: name is unique within a namespace to reference a secret resource.
+                              type: string
+                            namespace:
+                              description: namespace defines the space within which the secret name must be unique.
+                              type: string
+                          required:
+                            - keyPath
+                          type: object
+                          x-kubernetes-map-type: atomic
+                      type: object
+                  required:
+                    - password
+                    - username
+                  type: object
+                driver:
+                  description: The driver to use to connect to the shared datastore.
+                  enum:
+                    - etcd
+                    - MySQL
+                    - PostgreSQL
                   type: string
-                type: array
-              tlsConfig:
-                description: Defines the TLS/SSL configuration required to connect
-                  to the data store in a secure way.
-                properties:
-                  certificateAuthority:
-                    description: Retrieve the Certificate Authority certificate and
-                      private key, such as bare content of the file, or a SecretReference.
-                      The key reference is required since etcd authentication is based
-                      on certificates, and Kamaji is responsible in creating this.
-                    properties:
-                      certificate:
-                        properties:
-                          content:
-                            description: Bare content of the file, base64 encoded.
-                              It has precedence over the SecretReference value.
-                            format: byte
-                            type: string
-                          secretReference:
-                            properties:
-                              keyPath:
-                                description: Name of the key for the given Secret
-                                  reference where the content is stored. This value
-                                  is mandatory.
-                                type: string
-                              name:
-                                description: name is unique within a namespace to
-                                  reference a secret resource.
-                                type: string
-                              namespace:
-                                description: namespace defines the space within which
-                                  the secret name must be unique.
-                                type: string
-                            required:
-                            - keyPath
-                            type: object
-                            x-kubernetes-map-type: atomic
-                        type: object
-                      privateKey:
-                        properties:
-                          content:
-                            description: Bare content of the file, base64 encoded.
-                              It has precedence over the SecretReference value.
-                            format: byte
-                            type: string
-                          secretReference:
-                            properties:
-                              keyPath:
-                                description: Name of the key for the given Secret
-                                  reference where the content is stored. This value
-                                  is mandatory.
-                                type: string
-                              name:
-                                description: name is unique within a namespace to
-                                  reference a secret resource.
-                                type: string
-                              namespace:
-                                description: namespace defines the space within which
-                                  the secret name must be unique.
-                                type: string
-                            required:
-                            - keyPath
-                            type: object
-                            x-kubernetes-map-type: atomic
-                        type: object
-                    required:
-                    - certificate
-                    type: object
-                  clientCertificate:
-                    description: Specifies the SSL/TLS key and private key pair used
-                      to connect to the data store.
-                    properties:
-                      certificate:
-                        properties:
-                          content:
-                            description: Bare content of the file, base64 encoded.
-                              It has precedence over the SecretReference value.
-                            format: byte
-                            type: string
-                          secretReference:
-                            properties:
-                              keyPath:
-                                description: Name of the key for the given Secret
-                                  reference where the content is stored. This value
-                                  is mandatory.
-                                type: string
-                              name:
-                                description: name is unique within a namespace to
-                                  reference a secret resource.
-                                type: string
-                              namespace:
-                                description: namespace defines the space within which
-                                  the secret name must be unique.
-                                type: string
-                            required:
-                            - keyPath
-                            type: object
-                            x-kubernetes-map-type: atomic
-                        type: object
-                      privateKey:
-                        properties:
-                          content:
-                            description: Bare content of the file, base64 encoded.
-                              It has precedence over the SecretReference value.
-                            format: byte
-                            type: string
-                          secretReference:
-                            properties:
-                              keyPath:
-                                description: Name of the key for the given Secret
-                                  reference where the content is stored. This value
-                                  is mandatory.
-                                type: string
-                              name:
-                                description: name is unique within a namespace to
-                                  reference a secret resource.
-                                type: string
-                              namespace:
-                                description: namespace defines the space within which
-                                  the secret name must be unique.
-                                type: string
-                            required:
-                            - keyPath
-                            type: object
-                            x-kubernetes-map-type: atomic
-                        type: object
-                    required:
-                    - certificate
-                    - privateKey
-                    type: object
-                required:
-                - certificateAuthority
-                - clientCertificate
-                type: object
-            required:
-            - driver
-            - endpoints
-            - tlsConfig
-            type: object
-          status:
-            description: DataStoreStatus defines the observed state of DataStore.
-            properties:
-              usedBy:
-                description: List of the Tenant Control Planes, namespaced named,
-                  using this data store.
-                items:
-                  type: string
-                type: array
-            type: object
-        type: object
-    served: true
-    storage: true
-    subresources:
-      status: {}
+                endpoints:
+                  description: List of the endpoints to connect to the shared datastore. No need for protocol, just bare IP/FQDN and port.
+                  items:
+                    type: string
+                  minItems: 1
+                  type: array
+                tlsConfig:
+                  description: Defines the TLS/SSL configuration required to connect to the data store in a secure way.
+                  properties:
+                    certificateAuthority:
+                      description: Retrieve the Certificate Authority certificate and private key, such as bare content of the file, or a SecretReference. The key reference is required since etcd authentication is based on certificates, and Kamaji is responsible in creating this.
+                      properties:
+                        certificate:
+                          properties:
+                            content:
+                              description: Bare content of the file, base64 encoded. It has precedence over the SecretReference value.
+                              format: byte
+                              type: string
+                            secretReference:
+                              properties:
+                                keyPath:
+                                  description: Name of the key for the given Secret reference where the content is stored. This value is mandatory.
+                                  minLength: 1
+                                  type: string
+                                name:
+                                  description: name is unique within a namespace to reference a secret resource.
+                                  type: string
+                                namespace:
+                                  description: namespace defines the space within which the secret name must be unique.
+                                  type: string
+                              required:
+                                - keyPath
+                              type: object
+                              x-kubernetes-map-type: atomic
+                          type: object
+                        privateKey:
+                          properties:
+                            content:
+                              description: Bare content of the file, base64 encoded. It has precedence over the SecretReference value.
+                              format: byte
+                              type: string
+                            secretReference:
+                              properties:
+                                keyPath:
+                                  description: Name of the key for the given Secret reference where the content is stored. This value is mandatory.
+                                  minLength: 1
+                                  type: string
+                                name:
+                                  description: name is unique within a namespace to reference a secret resource.
+                                  type: string
+                                namespace:
+                                  description: namespace defines the space within which the secret name must be unique.
+                                  type: string
+                              required:
+                                - keyPath
+                              type: object
+                              x-kubernetes-map-type: atomic
+                          type: object
+                      required:
+                        - certificate
+                      type: object
+                    clientCertificate:
+                      description: Specifies the SSL/TLS key and private key pair used to connect to the data store.
+                      properties:
+                        certificate:
+                          properties:
+                            content:
+                              description: Bare content of the file, base64 encoded. It has precedence over the SecretReference value.
+                              format: byte
+                              type: string
+                            secretReference:
+                              properties:
+                                keyPath:
+                                  description: Name of the key for the given Secret reference where the content is stored. This value is mandatory.
+                                  minLength: 1
+                                  type: string
+                                name:
+                                  description: name is unique within a namespace to reference a secret resource.
+                                  type: string
+                                namespace:
+                                  description: namespace defines the space within which the secret name must be unique.
+                                  type: string
+                              required:
+                                - keyPath
+                              type: object
+                              x-kubernetes-map-type: atomic
+                          type: object
+                        privateKey:
+                          properties:
+                            content:
+                              description: Bare content of the file, base64 encoded. It has precedence over the SecretReference value.
+                              format: byte
+                              type: string
+                            secretReference:
+                              properties:
+                                keyPath:
+                                  description: Name of the key for the given Secret reference where the content is stored. This value is mandatory.
+                                  minLength: 1
+                                  type: string
+                                name:
+                                  description: name is unique within a namespace to reference a secret resource.
+                                  type: string
+                                namespace:
+                                  description: namespace defines the space within which the secret name must be unique.
+                                  type: string
+                              required:
+                                - keyPath
+                              type: object
+                              x-kubernetes-map-type: atomic
+                          type: object
+                      required:
+                        - certificate
+                        - privateKey
+                      type: object
+                  required:
+                    - certificateAuthority
+                    - clientCertificate
+                  type: object
+              required:
+                - driver
+                - endpoints
+                - tlsConfig
+              type: object
+            status:
+              description: DataStoreStatus defines the observed state of DataStore.
+              properties:
+                usedBy:
+                  description: List of the Tenant Control Planes, namespaced named, using this data store.
+                  items:
+                    type: string
+                  type: array
+              type: object
+          type: object
+      served: true
+      storage: true
+      subresources:
+        status: {}

charts/kamaji/templates/_helpers.tpl

@@ -46,9 +46,9 @@ app.kubernetes.io/managed-by: {{ .Release.Service }}
 Selector labels
 */}}
 {{- define "kamaji.selectorLabels" -}}
-app.kubernetes.io/name: {{ include "kamaji.name" . }}
-app.kubernetes.io/instance: {{ .Release.Name }}
-app.kubernetes.io/component: controller-manager
+app.kubernetes.io/name: {{ default (include "kamaji.name" .) .name }}
+app.kubernetes.io/instance: {{ default .Release.Name .instance }}
+app.kubernetes.io/component: {{ default "controller-manager" .component }}
 {{- end }}
 
 {{/*
@@ -61,3 +61,31 @@ Create the name of the service account to use
 {{- default "default" .Values.serviceAccount.name }}
 {{- end }}
 {{- end }}
+
+{{/*
+Create the name of the Service to user for webhooks
+*/}}
+{{- define "kamaji.webhookServiceName" -}}
+{{- printf "%s-webhook-service" (include "kamaji.fullname" .) }}
+{{- end }}
+
+{{/*
+Create the name of the Service to user for metrics
+*/}}
+{{- define "kamaji.metricsServiceName" -}}
+{{- printf "%s-metrics-service" (include "kamaji.fullname" .) }}
+{{- end }}
+
+{{/*
+Create the name of the cert-manager secret
+*/}}
+{{- define "kamaji.webhookSecretName" -}}
+{{- printf "%s-webhook-server-cert" (include "kamaji.fullname" .) }}
+{{- end }}
+
+{{/*
+Create the name of the cert-manager Certificate
+*/}}
+{{- define "kamaji.certificateName" -}}
+{{- printf "%s-serving-cert" (include "kamaji.fullname" .) }}
+{{- end }}

charts/kamaji/templates/_helpers_datastore.tpl

@@ -2,7 +2,11 @@
 Create a default fully qualified datastore name.
 */}}
 {{- define "datastore.fullname" -}}
+{{- if .Values.datastore.enabled }}
 {{- default "default" .Values.datastore.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- required "A valid .Values.datastore.nameOverride required!" .Values.datastore.nameOverride }}
+{{- end }}
 {{- end }}
 
 {{/*

charts/kamaji/templates/certmanager_certificate.yaml

@@ -0,0 +1,16 @@
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  labels:
+    {{- $data := . | mustMergeOverwrite (dict "component" "certificate") -}}
+    {{- include "kamaji.labels" $data | nindent 4 }}
+  name: {{ include "kamaji.certificateName" . }}
+  namespace: {{ .Release.Namespace }}
+spec:
+  dnsNames:
+    - {{ include "kamaji.webhookServiceName" . }}.{{ .Release.Namespace }}.svc
+    - {{ include "kamaji.webhookServiceName" . }}.{{ .Release.Namespace }}.svc.cluster.local
+  issuerRef:
+    kind: Issuer
+    name: kamaji-selfsigned-issuer
+  secretName: {{ include "kamaji.webhookSecretName" . }}

charts/kamaji/templates/certmanager_issuer.yaml

@@ -0,0 +1,10 @@
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+  labels:
+    {{- $data := . | mustMergeOverwrite (dict "component" "issuer") -}}
+    {{- include "kamaji.labels" $data | nindent 4 }}
+  name: kamaji-selfsigned-issuer
+  namespace: {{ .Release.Namespace }}
+spec:
+  selfSigned: {}

charts/kamaji/templates/controller.yaml

@@ -28,18 +28,7 @@ spec:
       serviceAccountName: {{ include "kamaji.serviceAccountName" . }}
       containers:
       - args:
-        - --secure-listen-address=0.0.0.0:8443
-        - --upstream=http://127.0.0.1:8080/
-        - --logtostderr=true
-        - --v=10
-        image: gcr.io/kubebuilder/kube-rbac-proxy:v0.8.0
-        name: kube-rbac-proxy
-        ports:
-        - containerPort: 8443
-          name: https
-          protocol: TCP
-      - args:
-        - --config-file={{ .Values.configPath }}
+        - manager
         - --health-probe-bind-address={{ .Values.healthProbeBindAddress }}
         - --leader-elect
         - --metrics-bind-address={{ .Values.metricsBindAddress }}
@@ -52,7 +41,16 @@ spec:
         {{- toYaml . | nindent 8 }}
         {{- end }}
         command:
-        - /manager
+        - /kamaji
+        env:
+          - name: POD_NAMESPACE
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.namespace
+          - name: SERVICE_ACCOUNT
+            valueFrom:
+              fieldRef:
+                fieldPath: spec.serviceAccountName
         image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
         imagePullPolicy: {{ .Values.image.pullPolicy }}
         {{- with .Values.livenessProbe }}
@@ -61,6 +59,12 @@ spec:
         {{- end }}
         name: manager
         ports:
+        - containerPort: 9443
+          name: webhook-server
+          protocol: TCP
+        - containerPort: 8080
+          name: metrics
+          protocol: TCP
         - containerPort: 8081
           name: healthcheck
           protocol: TCP
@@ -72,7 +76,21 @@ spec:
           {{- toYaml .Values.resources | nindent 12 }}
         securityContext:
           {{- toYaml .Values.securityContext | nindent 12 }}
+        volumeMounts:
+          - mountPath: /tmp
+            name: tmp
+          - mountPath: /tmp/k8s-webhook-server/serving-certs
+            name: cert
+            readOnly: true
       terminationGracePeriodSeconds: 10
+      volumes:
+        - name: tmp
+          emptyDir:
+            medium: Memory
+        - name: cert
+          secret:
+            defaultMode: 420
+            secretName: {{ include "kamaji.webhookSecretName" . }}
       {{- with .Values.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}

charts/kamaji/templates/datastore.yaml

@@ -1,7 +1,10 @@
+{{- if .Values.datastore.enabled}}
 apiVersion: kamaji.clastix.io/v1alpha1
 kind: DataStore
 metadata:
   name: {{ include "datastore.fullname" . }}
+  annotations:
+    "helm.sh/hook": pre-install
   labels:
     {{- include "datastore.labels" . | nindent 4 }}
 spec:
@@ -10,10 +13,16 @@ spec:
     {{- include "datastore.endpoints" . | indent 4 }}
 {{- if (and .Values.datastore.basicAuth.usernameSecret.name .Values.datastore.basicAuth.passwordSecret.name) }}
   basicAuth:
-    {{- .Values.datastore.basicAuth | toYaml | nindent 4 }}
+    username:
+      secretReference:
+        {{- .Values.datastore.basicAuth.usernameSecret | toYaml | nindent 8 }}
+    password:
+      secretReference:
+        {{- .Values.datastore.basicAuth.passwordSecret | toYaml | nindent 8 }}
 {{- end }}
   tlsConfig:
     certificateAuthority:
       {{- include "datastore.certificateAuthority" . | indent 6 }}
     clientCertificate:
       {{- include "datastore.clientCertificate" . | indent 6 }}
+{{- end}}

charts/kamaji/templates/etcd_cm.yaml

@@ -6,6 +6,10 @@ metadata:
     {{- include "etcd.labels" . | nindent 4 }}
   name: {{ include "etcd.csrConfigMapName" . }}
   namespace: {{ .Release.Namespace }}
+  annotations:
+    "helm.sh/hook": pre-install
+    "helm.sh/hook-weight": "-5"
+    "helm.sh/hook-delete-policy": "hook-succeeded,hook-failed"
 data:
   ca-csr.json: |-
     {

charts/kamaji/templates/etcd_job_postdelete.yaml

@@ -28,4 +28,8 @@ spec:
             - --ignore-not-found=true
             - {{ include "etcd.caSecretName" . }}
             - {{ include "etcd.clientSecretName" . }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
 {{- end }}

charts/kamaji/templates/etcd_job_postinstall.yaml

@@ -18,45 +18,27 @@ spec:
       serviceAccountName: {{ include "etcd.serviceAccountName" . }}
       restartPolicy: Never
       initContainers:
-        - name: cfssl
-          image: cfssl/cfssl:latest
-          command:
-            - bash
-            - -c
-            - |-
-              cfssl gencert -initca /csr/ca-csr.json | cfssljson -bare /certs/ca &&
-              mv /certs/ca.pem /certs/ca.crt && mv /certs/ca-key.pem /certs/ca.key &&
-              cfssl gencert -ca=/certs/ca.crt -ca-key=/certs/ca.key -config=/csr/config.json -profile=peer-authentication /csr/peer-csr.json | cfssljson -bare /certs/peer &&
-              cfssl gencert -ca=/certs/ca.crt -ca-key=/certs/ca.key -config=/csr/config.json -profile=peer-authentication /csr/server-csr.json | cfssljson -bare /certs/server &&
-              cfssl gencert -ca=/certs/ca.crt -ca-key=/certs/ca.key -config=/csr/config.json -profile=client-authentication /csr/root-client-csr.json | cfssljson -bare /certs/root-client
-          volumeMounts:
-            - mountPath: /certs
-              name: certs
-            - mountPath: /csr
-              name: csr
         - name: kubectl
           image: {{ printf "clastix/kubectl:%s" (include "etcd.jobsTagKubeVersion" .) }}
           command:
             - sh
             - -c
             - |-
-              kubectl --namespace={{ .Release.Namespace }} delete secret --ignore-not-found=true {{ include "etcd.caSecretName" . }} {{ include "etcd.clientSecretName" . }} &&
-              kubectl --namespace={{ .Release.Namespace }} create secret generic {{ include "etcd.caSecretName" . }} --from-file=/certs/ca.crt --from-file=/certs/ca.key --from-file=/certs/peer-key.pem --from-file=/certs/peer.pem --from-file=/certs/server-key.pem --from-file=/certs/server.pem &&
-              kubectl --namespace={{ .Release.Namespace }} create secret tls {{ include "etcd.clientSecretName" . }} --key=/certs/root-client-key.pem --cert=/certs/root-client.pem &&
               kubectl --namespace={{ .Release.Namespace }} rollout status sts/etcd --timeout=300s
-          volumeMounts:
-            - mountPath: /certs
-              name: certs
       containers:
         - command:
             - bash
             - -c
             - |-
-              etcdctl member list -w table &&
-              etcdctl user add --no-password=true root &&
-              etcdctl role add root &&
-              etcdctl user grant-role root root &&
-              etcdctl auth enable
+              etcdctl member list -w table
+              if etcdctl user get root &>/dev/null; then
+                echo "User already exists, nothing to do"
+              else
+                etcdctl user add --no-password=true root &&
+                etcdctl role add root &&
+                etcdctl user grant-role root root &&
+                etcdctl auth enable
+              fi
           env:
             - name: ETCDCTL_ENDPOINTS
               value: https://etcd-0.{{ include "etcd.serviceName" . }}.{{ .Release.Namespace }}.svc.cluster.local:2379
@@ -82,10 +64,11 @@ spec:
         - name: root-certs
           secret:
             secretName: {{ include "etcd.clientSecretName" . }}
-            optional: true
-        - name: csr
-          configMap:
-            name: {{ include "etcd.csrConfigMapName" . }}
         - name: certs
-          emptyDir: {}
+          secret:
+            secretName: {{ include "etcd.caSecretName" . }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
 {{- end }}

charts/kamaji/templates/etcd_job_preinstall.yaml

@@ -0,0 +1,72 @@
+{{- if .Values.etcd.deploy }}
+apiVersion: batch/v1
+kind: Job
+metadata:
+  labels:
+    {{- include "etcd.labels" . | nindent 4 }}
+  annotations:
+    "helm.sh/hook": pre-install
+    "helm.sh/hook-weight": "-5"
+    "helm.sh/hook-delete-policy": "hook-succeeded"
+  name: "{{ .Release.Name }}-etcd-certs"
+  namespace: {{ .Release.Namespace }}
+spec:
+  template:
+    metadata:
+      name: "{{ .Release.Name }}"
+    spec:
+      serviceAccountName: {{ include "etcd.serviceAccountName" . }}
+      restartPolicy: Never
+      initContainers:
+        - name: cfssl
+          image: "{{ .Values.cfssl.image.repository }}:{{ .Values.cfssl.image.tag }}"
+          command:
+            - bash
+            - -c
+            - |-
+              cfssl gencert -initca /csr/ca-csr.json | cfssljson -bare /certs/ca &&
+              mv /certs/ca.pem /certs/ca.crt && mv /certs/ca-key.pem /certs/ca.key &&
+              cfssl gencert -ca=/certs/ca.crt -ca-key=/certs/ca.key -config=/csr/config.json -profile=peer-authentication /csr/peer-csr.json | cfssljson -bare /certs/peer &&
+              cfssl gencert -ca=/certs/ca.crt -ca-key=/certs/ca.key -config=/csr/config.json -profile=peer-authentication /csr/server-csr.json | cfssljson -bare /certs/server &&
+              cfssl gencert -ca=/certs/ca.crt -ca-key=/certs/ca.key -config=/csr/config.json -profile=client-authentication /csr/root-client-csr.json | cfssljson -bare /certs/root-client
+          volumeMounts:
+            - mountPath: /certs
+              name: certs
+            - mountPath: /csr
+              name: csr
+      containers:
+        - name: kubectl
+          image: {{ printf "clastix/kubectl:%s" (include "etcd.jobsTagKubeVersion" .) }}
+          command: ["/bin/sh", "-c"]
+          args:
+            - |
+              if kubectl get secret {{ include "etcd.caSecretName" . }} --namespace={{ .Release.Namespace }} &>/dev/null; then
+                echo "Secret {{ include "etcd.caSecretName" . }} already exists"
+              else
+                echo "Creating secret {{ include "etcd.caSecretName" . }}"
+                kubectl --namespace={{ .Release.Namespace }} create secret generic {{ include "etcd.caSecretName" . }} --from-file=/certs/ca.crt --from-file=/certs/ca.key --from-file=/certs/peer-key.pem --from-file=/certs/peer.pem --from-file=/certs/server-key.pem --from-file=/certs/server.pem
+              fi
+              if kubectl get secret {{ include "etcd.clientSecretName" . }} --namespace={{ .Release.Namespace }} &>/dev/null; then
+                echo "Secret {{ include "etcd.clientSecretName" . }} already exists"
+              else
+                echo "Creating secret {{ include "etcd.clientSecretName" . }}"
+                kubectl --namespace={{ .Release.Namespace }} create secret tls {{ include "etcd.clientSecretName" . }} --key=/certs/root-client-key.pem --cert=/certs/root-client.pem
+              fi
+          volumeMounts:
+            - mountPath: /certs
+              name: certs
+      securityContext:
+        runAsUser: 1000
+        runAsGroup: 1000
+        fsGroup: 1000
+      volumes:
+        - name: csr
+          configMap:
+            name: {{ include "etcd.csrConfigMapName" . }}
+        - name: certs
+          emptyDir: {}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+{{- end }}

charts/kamaji/templates/etcd_rbac.yaml

@@ -5,6 +5,9 @@ metadata:
   labels:
     {{- include "etcd.labels" . | nindent 4 }}
   name: etcd-gen-certs-role
+  annotations:
+    "helm.sh/hook": pre-install
+    "helm.sh/hook-weight": "-5"
   namespace: {{ .Release.Namespace }}
 rules:
   - apiGroups:
@@ -12,6 +15,7 @@ rules:
     resources:
       - secrets
     verbs:
+      - get
       - delete
     resourceNames:
       - {{ include "etcd.caSecretName" . }}
@@ -38,6 +42,9 @@ metadata:
     {{- include "etcd.labels" . | nindent 4 }}
   name: etcd-gen-certs-rolebiding
   namespace: {{ .Release.Namespace }}
+  annotations:
+    "helm.sh/hook": pre-install
+    "helm.sh/hook-weight": "-5"
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: Role

charts/kamaji/templates/etcd_sa.yaml

@@ -5,5 +5,8 @@ metadata:
   labels:
     {{- include "etcd.labels" . | nindent 4 }}
   name: {{ include "etcd.serviceAccountName" . }}
+  annotations:
+    "helm.sh/hook": pre-install
+    "helm.sh/hook-weight": "-5"
   namespace: {{ .Release.Namespace }}
 {{- end }}

charts/kamaji/templates/etcd_sts.yaml

@@ -22,6 +22,10 @@ spec:
         - name: certs
           secret:
             secretName: {{ include "etcd.caSecretName" . }}
+      {{- with .Values.etcd.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
       containers:
         - name: etcd
           image: {{ .Values.etcd.image.repository }}:{{ .Values.etcd.image.tag | default "v3.5.4" }}
@@ -81,6 +85,10 @@ spec:
   volumeClaimTemplates:
   - metadata:
       name: data
+      {{- with .Values.etcd.persistence.customAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
     spec:
       storageClassName: {{ .Values.etcd.persistence.storageClassName }}
       accessModes:

charts/kamaji/templates/mutatingwebhookconfiguration.yaml

@@ -0,0 +1,30 @@
+apiVersion: admissionregistration.k8s.io/v1
+kind: MutatingWebhookConfiguration
+metadata:
+  annotations:
+    cert-manager.io/inject-ca-from: {{ .Release.Namespace }}/{{ include "kamaji.certificateName" . }}
+  labels:
+    {{- $data := . | mustMergeOverwrite (dict "instance" "mutating-webhook-configuration") -}}
+    {{- include "kamaji.labels" $data | nindent 4 }}
+  name: kamaji-mutating-webhook-configuration
+webhooks:
+  - admissionReviewVersions:
+      - v1
+    clientConfig:
+      service:
+        name: {{ include "kamaji.webhookServiceName" . }}
+        namespace: {{ .Release.Namespace }}
+        path: /mutate-kamaji-clastix-io-v1alpha1-tenantcontrolplane
+    failurePolicy: Fail
+    name: mtenantcontrolplane.kb.io
+    rules:
+      - apiGroups:
+          - kamaji.clastix.io
+        apiVersions:
+          - v1alpha1
+        operations:
+          - CREATE
+          - UPDATE
+        resources:
+          - tenantcontrolplanes
+    sideEffects: None

charts/kamaji/templates/rbac.yaml

@@ -66,6 +66,16 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+    - batch
+  resources:
+    - jobs
+  verbs:
+    - create
+    - delete
+    - get
+    - list
+    - watch
 - apiGroups:
   - ""
   resources:
@@ -114,12 +124,6 @@ rules:
     - patch
     - update
     - watch
-- apiGroups:
-    - kamaji.clastix.io
-  resources:
-    - datastores/finalizers
-  verbs:
-    - update
 - apiGroups:
     - kamaji.clastix.io
   resources:

charts/kamaji/templates/service.yaml

@@ -1,16 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  name: {{ include "kamaji.fullname" . }}
-  labels:
-    {{- include "kamaji.labels" . | nindent 4 }}
-  namespace: {{ .Release.Namespace }}
-spec:
-  type: {{ .Values.service.type }}
-  ports:
-  - name: https
-    port: {{ .Values.service.port }}
-    protocol: TCP
-    targetPort: https
-  selector:
-    {{- include "kamaji.selectorLabels" . | nindent 4 }}

charts/kamaji/templates/service_metrics.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    {{- $data := . | mustMergeOverwrite (dict "component" "metrics") -}}
+    {{- include "kamaji.labels" $data | nindent 4 }}
+  name: {{ include "kamaji.metricsServiceName" . }}
+  namespace: {{ .Release.Namespace }}
+spec:
+  ports:
+    - port: 8080
+      name: metrics
+      protocol: TCP
+      targetPort: metrics
+  selector:
+    {{- include "kamaji.selectorLabels" . | nindent 4 }}

charts/kamaji/templates/service_webhook.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    {{- $data := . | mustMergeOverwrite (dict "component" "webhook" "instance" "webhook-service") -}}
+    {{- include "kamaji.labels" $data | nindent 4 }}
+  name: {{ include "kamaji.webhookServiceName" . }}
+  namespace: {{ .Release.Namespace }}
+spec:
+  ports:
+    - port: 443
+      protocol: TCP
+      name: webhook-server
+      targetPort: webhook-server
+  selector:
+    {{- include "kamaji.selectorLabels" . | nindent 4 }}

charts/kamaji/templates/servicemonitor.yaml

@@ -0,0 +1,21 @@
+{{- if and (.Capabilities.APIVersions.Has "monitoring.coreos.com/v1") .Values.serviceMonitor.enabled }}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  labels:
+    {{- $data := . | mustMergeOverwrite (dict "component" "servicemonitor") -}}
+    {{- include "kamaji.labels" $data | nindent 4 }}
+  name: {{ include "kamaji.fullname" . }}
+  namespace: {{ .Release.Namespace }}
+spec:
+  endpoints:
+    - path: /metrics
+      port: metrics
+      scheme: http
+  namespaceSelector:
+    matchNames:
+      - {{ .Release.Namespace }}
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: {{ include "kamaji.name" . }}
+{{- end }}

charts/kamaji/templates/validatingwebhookconfiguration.yaml

@@ -0,0 +1,70 @@
+apiVersion: admissionregistration.k8s.io/v1
+kind: ValidatingWebhookConfiguration
+metadata:
+  annotations:
+    cert-manager.io/inject-ca-from: {{ .Release.Namespace }}/{{ include "kamaji.certificateName" . }}
+  labels:
+    {{- $data := . | mustMergeOverwrite (dict "instance" "validating-webhook-configuration") -}}
+    {{- include "kamaji.labels" $data | nindent 4 }}
+  name: kamaji-validating-webhook-configuration
+webhooks:
+  - admissionReviewVersions:
+      - v1
+    clientConfig:
+      service:
+        name: {{ include "kamaji.webhookServiceName" . }}
+        namespace: {{ .Release.Namespace }}
+        path: /validate--v1-secret
+    failurePolicy: Ignore
+    name: vdatastoresecrets.kb.io
+    rules:
+      - apiGroups:
+          - ""
+        apiVersions:
+          - v1
+        operations:
+          - DELETE
+        resources:
+          - secrets
+    sideEffects: None
+  - admissionReviewVersions:
+      - v1
+    clientConfig:
+      service:
+        name: {{ include "kamaji.webhookServiceName" . }}
+        namespace: {{ .Release.Namespace }}
+        path: /validate-kamaji-clastix-io-v1alpha1-datastore
+    failurePolicy: Fail
+    name: vdatastore.kb.io
+    rules:
+      - apiGroups:
+          - kamaji.clastix.io
+        apiVersions:
+          - v1alpha1
+        operations:
+          - CREATE
+          - UPDATE
+          - DELETE
+        resources:
+          - datastores
+    sideEffects: None
+  - admissionReviewVersions:
+      - v1
+    clientConfig:
+      service:
+        name: {{ include "kamaji.webhookServiceName" . }}
+        namespace: {{ .Release.Namespace }}
+        path: /validate-kamaji-clastix-io-v1alpha1-tenantcontrolplane
+    failurePolicy: Fail
+    name: vtenantcontrolplane.kb.io
+    rules:
+      - apiGroups:
+          - kamaji.clastix.io
+        apiVersions:
+          - v1alpha1
+        operations:
+          - CREATE
+          - UPDATE
+        resources:
+          - tenantcontrolplanes
+    sideEffects: None

charts/kamaji/values.yaml

@@ -15,8 +15,10 @@ image:
 # -- A list of extra arguments to add to the kamaji controller default ones
 extraArgs: []
 
-# -- Configuration file path alternative. (default "./kamaji.yaml")
-configPath: "./kamaji.yaml"
+
+serviceMonitor:
+  # -- Toggle the ServiceMonitor true if you have Prometheus Operator installed and configured
+  enabled: false
 
 etcd:
   # -- Install an etcd with enabled multi-tenancy along with Kamaji
@@ -31,7 +33,7 @@ etcd:
   # -- Install specific etcd image
   image:
     repository: quay.io/coreos/etcd
-    tag: "v3.5.4"
+    tag: "v3.5.6"
     pullPolicy: IfNotPresent
 
   # -- The livenessProbe for the etcd container
@@ -52,9 +54,15 @@ etcd:
     name: ""
   persistence:
     size: 10Gi
-    storageClass: ""
+    storageClassName: ""
     accessModes:
     - ReadWriteOnce
+    # -- The custom annotations to add to the PVC
+    customAnnotations: {}
+    #  volumeType: local
+  
+  # -- (array) Kubernetes affinity rules to apply to Kamaji etcd pods
+  tolerations: []
 
   overrides:
     caSecret:
@@ -127,10 +135,6 @@ securityContext:
   # runAsNonRoot: true
   # runAsUser: 1000
 
-service:
-  type: ClusterIP
-  port: 8443
-
 resources:
   limits:
     cpu: 200m
@@ -156,7 +160,9 @@ loggingDevel:
   enable: false
 
 datastore:
-  # -- (string) The Datastore name override, if empty defaults to `default`
+  # -- (bool) Enable the Kamaji Datastore creation (default=true)
+  enabled: true
+  # -- (string) The Datastore name override, if empty and enabled=true defaults to `default`, if enabled=false, this is the name of the Datastore to connect to. 
   nameOverride:
   # -- (string) The Kamaji Datastore driver, supported: etcd, MySQL, PostgreSQL (defaults=etcd).
   driver: etcd
@@ -208,3 +214,8 @@ datastore:
         namespace:
         # -- Key of the Secret which contains the content of the private key.
         keyPath:
+
+cfssl:
+  image:
+    repository: cfssl/cfssl
+    tag: latest

cmd/manager/cmd.go

@@ -0,0 +1,271 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package manager
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	goRuntime "runtime"
+	"time"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/rest"
+	"k8s.io/klog/v2"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/cache"
+	"sigs.k8s.io/controller-runtime/pkg/healthz"
+	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
+	ctrlwebhook "sigs.k8s.io/controller-runtime/pkg/webhook"
+
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+	cmdutils "github.com/clastix/kamaji/cmd/utils"
+	"github.com/clastix/kamaji/controllers"
+	"github.com/clastix/kamaji/controllers/soot"
+	"github.com/clastix/kamaji/internal"
+	"github.com/clastix/kamaji/internal/builders/controlplane"
+	datastoreutils "github.com/clastix/kamaji/internal/datastore/utils"
+	"github.com/clastix/kamaji/internal/webhook"
+	"github.com/clastix/kamaji/internal/webhook/handlers"
+	"github.com/clastix/kamaji/internal/webhook/routes"
+)
+
+//nolint:maintidx
+func NewCmd(scheme *runtime.Scheme) *cobra.Command {
+	// CLI flags
+	var (
+		metricsBindAddress         string
+		healthProbeBindAddress     string
+		leaderElect                bool
+		tmpDirectory               string
+		kineImage                  string
+		controllerReconcileTimeout time.Duration
+		cacheResyncPeriod          time.Duration
+		datastore                  string
+		managerNamespace           string
+		managerServiceAccountName  string
+		managerServiceName         string
+		webhookCABundle            []byte
+		migrateJobImage            string
+		maxConcurrentReconciles    int
+
+		webhookCAPath string
+	)
+
+	ctx := ctrl.SetupSignalHandler()
+
+	cmd := &cobra.Command{
+		Use:           "manager",
+		Short:         "Start the Kamaji Kubernetes Operator",
+		SilenceErrors: false,
+		SilenceUsage:  true,
+		PreRunE: func(cmd *cobra.Command, args []string) (err error) {
+			// Avoid to pollute Kamaji stdout with useless details by the underlying klog implementations
+			klog.SetOutput(io.Discard)
+			klog.LogToStderr(false)
+
+			if err = cmdutils.CheckFlags(cmd.Flags(), []string{"kine-image", "datastore", "migrate-image", "tmp-directory", "pod-namespace", "webhook-service-name", "serviceaccount-name", "webhook-ca-path"}...); err != nil {
+				return err
+			}
+
+			if webhookCABundle, err = os.ReadFile(webhookCAPath); err != nil {
+				return fmt.Errorf("unable to read webhook CA: %w", err)
+			}
+
+			if err = datastoreutils.CheckExists(ctx, scheme, datastore); err != nil {
+				return err
+			}
+
+			if controllerReconcileTimeout.Seconds() == 0 {
+				return fmt.Errorf("the controller reconcile timeout must be greater than zero")
+			}
+
+			return nil
+		},
+		RunE: func(cmd *cobra.Command, args []string) error {
+			setupLog := ctrl.Log.WithName("setup")
+
+			setupLog.Info(fmt.Sprintf("Kamaji version %s %s%s", internal.GitTag, internal.GitCommit, internal.GitDirty))
+			setupLog.Info(fmt.Sprintf("Build from: %s", internal.GitRepo))
+			setupLog.Info(fmt.Sprintf("Build date: %s", internal.BuildTime))
+			setupLog.Info(fmt.Sprintf("Go Version: %s", goRuntime.Version()))
+			setupLog.Info(fmt.Sprintf("Go OS/Arch: %s/%s", goRuntime.GOOS, goRuntime.GOARCH))
+
+			mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
+				Scheme: scheme,
+				Metrics: metricsserver.Options{
+					BindAddress: metricsBindAddress,
+				},
+				WebhookServer: ctrlwebhook.NewServer(ctrlwebhook.Options{
+					Port: 9443,
+				}),
+				HealthProbeBindAddress:  healthProbeBindAddress,
+				LeaderElection:          leaderElect,
+				LeaderElectionNamespace: managerNamespace,
+				LeaderElectionID:        "799b98bc.clastix.io",
+				NewCache: func(config *rest.Config, opts cache.Options) (cache.Cache, error) {
+					opts.SyncPeriod = &cacheResyncPeriod
+
+					return cache.New(config, opts)
+				},
+			})
+			if err != nil {
+				setupLog.Error(err, "unable to start manager")
+
+				return err
+			}
+
+			tcpChannel, certChannel := make(controllers.TenantControlPlaneChannel), make(controllers.CertificateChannel)
+
+			if err = (&controllers.DataStore{Client: mgr.GetClient(), TenantControlPlaneTrigger: tcpChannel}).SetupWithManager(mgr); err != nil {
+				setupLog.Error(err, "unable to create controller", "controller", "DataStore")
+
+				return err
+			}
+
+			reconciler := &controllers.TenantControlPlaneReconciler{
+				Client:    mgr.GetClient(),
+				APIReader: mgr.GetAPIReader(),
+				Config: controllers.TenantControlPlaneReconcilerConfig{
+					ReconcileTimeout:     controllerReconcileTimeout,
+					DefaultDataStoreName: datastore,
+					KineContainerImage:   kineImage,
+					TmpBaseDirectory:     tmpDirectory,
+				},
+				CertificateChan:         certChannel,
+				TriggerChan:             tcpChannel,
+				KamajiNamespace:         managerNamespace,
+				KamajiServiceAccount:    managerServiceAccountName,
+				KamajiService:           managerServiceName,
+				KamajiMigrateImage:      migrateJobImage,
+				MaxConcurrentReconciles: maxConcurrentReconciles,
+			}
+
+			if err = reconciler.SetupWithManager(mgr); err != nil {
+				setupLog.Error(err, "unable to create controller", "controller", "Namespace")
+
+				return err
+			}
+
+			if err = (&controllers.CertificateLifecycle{Channel: certChannel}).SetupWithManager(mgr); err != nil {
+				setupLog.Error(err, "unable to create controller", "controller", "CertificateLifecycle")
+
+				return err
+			}
+
+			if err = (&kamajiv1alpha1.DatastoreUsedSecret{}).SetupWithManager(ctx, mgr); err != nil {
+				setupLog.Error(err, "unable to create indexer", "indexer", "DatastoreUsedSecret")
+
+				return err
+			}
+
+			if err = (&kamajiv1alpha1.TenantControlPlaneStatusDataStore{}).SetupWithManager(ctx, mgr); err != nil {
+				setupLog.Error(err, "unable to create indexer", "indexer", "TenantControlPlaneStatusDataStore")
+
+				return err
+			}
+
+			err = webhook.Register(mgr, map[routes.Route][]handlers.Handler{
+				routes.TenantControlPlaneMigrate{}: {
+					handlers.Freeze{},
+				},
+				routes.TenantControlPlaneDefaults{}: {
+					handlers.TenantControlPlaneDefaults{DefaultDatastore: datastore},
+				},
+				routes.TenantControlPlaneValidate{}: {
+					handlers.TenantControlPlaneName{},
+					handlers.TenantControlPlaneVersion{},
+					handlers.TenantControlPlaneKubeletAddresses{},
+					handlers.TenantControlPlaneDataStore{Client: mgr.GetClient()},
+					handlers.TenantControlPlaneDeployment{
+						Client: mgr.GetClient(),
+						DeploymentBuilder: controlplane.Deployment{
+							Client:             mgr.GetClient(),
+							KineContainerImage: kineImage,
+						},
+						KonnectivityBuilder: controlplane.Konnectivity{
+							Scheme: *mgr.GetScheme(),
+						},
+					},
+				},
+				routes.DataStoreValidate{}: {
+					handlers.DataStoreValidation{Client: mgr.GetClient()},
+				},
+				routes.DataStoreSecrets{}: {
+					handlers.DataStoreSecretValidation{Client: mgr.GetClient()},
+				},
+			})
+			if err != nil {
+				setupLog.Error(err, "unable to create webhook")
+
+				return err
+			}
+
+			if err = (&soot.Manager{
+				MigrateCABundle:         webhookCABundle,
+				MigrateServiceName:      managerServiceName,
+				MigrateServiceNamespace: managerNamespace,
+				AdminClient:             mgr.GetClient(),
+			}).SetupWithManager(mgr); err != nil {
+				setupLog.Error(err, "unable to set up soot manager")
+
+				return err
+			}
+
+			if err = mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
+				setupLog.Error(err, "unable to set up health check")
+
+				return err
+			}
+			if err = mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil {
+				setupLog.Error(err, "unable to set up ready check")
+
+				return err
+			}
+
+			setupLog.Info("starting manager")
+			if err = mgr.Start(ctx); err != nil {
+				setupLog.Error(err, "problem running manager")
+
+				return err
+			}
+
+			return nil
+		},
+	}
+
+	// Setting zap logger
+	zapfs := flag.NewFlagSet("zap", flag.ExitOnError)
+	opts := zap.Options{
+		Development: true,
+	}
+	opts.BindFlags(zapfs)
+	cmd.Flags().AddGoFlagSet(zapfs)
+	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
+	// Setting CLI flags
+	cmd.Flags().StringVar(&metricsBindAddress, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.")
+	cmd.Flags().StringVar(&healthProbeBindAddress, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
+	cmd.Flags().BoolVar(&leaderElect, "leader-elect", true, "Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.")
+	cmd.Flags().StringVar(&tmpDirectory, "tmp-directory", "/tmp/kamaji", "Directory which will be used to work with temporary files.")
+	cmd.Flags().StringVar(&kineImage, "kine-image", "rancher/kine:v0.9.2-amd64", "Container image along with tag to use for the Kine sidecar container (used only if etcd-storage-type is set to one of kine strategies).")
+	cmd.Flags().StringVar(&datastore, "datastore", "etcd", "The default DataStore that should be used by Kamaji to setup the required storage.")
+	cmd.Flags().StringVar(&migrateJobImage, "migrate-image", fmt.Sprintf("clastix/kamaji:%s", internal.GitTag), "Specify the container image to launch when a TenantControlPlane is migrated to a new datastore.")
+	cmd.Flags().IntVar(&maxConcurrentReconciles, "max-concurrent-tcp-reconciles", 1, "Specify the number of workers for the Tenant Control Plane controller (beware of CPU consumption)")
+	cmd.Flags().StringVar(&managerNamespace, "pod-namespace", os.Getenv("POD_NAMESPACE"), "The Kubernetes Namespace on which the Operator is running in, required for the TenantControlPlane migration jobs.")
+	cmd.Flags().StringVar(&managerServiceName, "webhook-service-name", "kamaji-webhook-service", "The Kamaji webhook server Service name which is used to get validation webhooks, required for the TenantControlPlane migration jobs.")
+	cmd.Flags().StringVar(&managerServiceAccountName, "serviceaccount-name", os.Getenv("SERVICE_ACCOUNT"), "The Kubernetes Namespace on which the Operator is running in, required for the TenantControlPlane migration jobs.")
+	cmd.Flags().StringVar(&webhookCAPath, "webhook-ca-path", "/tmp/k8s-webhook-server/serving-certs/ca.crt", "Path to the Manager webhook server CA, required for the TenantControlPlane migration jobs.")
+	cmd.Flags().DurationVar(&controllerReconcileTimeout, "controller-reconcile-timeout", 30*time.Second, "The reconciliation request timeout before the controller withdraw the external resource calls, such as dealing with the Datastore, or the Tenant Control Plane API endpoint.")
+	cmd.Flags().DurationVar(&cacheResyncPeriod, "cache-resync-period", 10*time.Hour, "The controller-runtime.Manager cache resync period.")
+
+	cobra.OnInitialize(func() {
+		viper.AutomaticEnv()
+	})
+
+	return cmd
+}

cmd/migrate/cmd.go

@@ -0,0 +1,119 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package migrate
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/spf13/cobra"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
+	ctrl "sigs.k8s.io/controller-runtime"
+	ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
+
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+	"github.com/clastix/kamaji/internal/datastore"
+)
+
+func NewCmd(scheme *runtime.Scheme) *cobra.Command {
+	// CLI flags
+	var (
+		tenantControlPlane string
+		targetDataStore    string
+		timeout            time.Duration
+	)
+
+	cmd := &cobra.Command{
+		Use:          "migrate",
+		Short:        "Migrate the data of a TenantControlPlane to another compatible DataStore",
+		SilenceUsage: true,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			ctx, cancelFn := context.WithTimeout(context.Background(), timeout)
+			defer cancelFn()
+
+			log := ctrl.Log
+
+			log.Info("generating the controller-runtime client")
+
+			client, err := ctrlclient.New(ctrl.GetConfigOrDie(), ctrlclient.Options{
+				Scheme: scheme,
+			})
+			if err != nil {
+				return err
+			}
+
+			parts := strings.Split(tenantControlPlane, string(types.Separator))
+			if len(parts) != 2 {
+				return fmt.Errorf("non well-formed namespaced name for the tenant control plane, expected <NAMESPACE>/NAME, fot %s", tenantControlPlane)
+			}
+
+			log.Info("retrieving the TenantControlPlane")
+
+			tcp := &kamajiv1alpha1.TenantControlPlane{}
+			if err = client.Get(ctx, types.NamespacedName{Namespace: parts[0], Name: parts[1]}, tcp); err != nil {
+				return err
+			}
+
+			log.Info("retrieving the TenantControlPlane used DataStore")
+
+			originDs := &kamajiv1alpha1.DataStore{}
+			if err = client.Get(ctx, types.NamespacedName{Name: tcp.Status.Storage.DataStoreName}, originDs); err != nil {
+				return err
+			}
+
+			log.Info("retrieving the target DataStore")
+
+			targetDs := &kamajiv1alpha1.DataStore{}
+			if err = client.Get(ctx, types.NamespacedName{Name: targetDataStore}, targetDs); err != nil {
+				return err
+			}
+
+			if tcp.Status.Storage.Driver != string(targetDs.Spec.Driver) {
+				return fmt.Errorf("migration between DataStore with different driver is not supported")
+			}
+
+			if tcp.Status.Storage.DataStoreName == targetDs.GetName() {
+				return fmt.Errorf("cannot migrate to the same DataStore")
+			}
+
+			log.Info("generating the origin storage connection")
+
+			originConnection, err := datastore.NewStorageConnection(ctx, client, *originDs)
+			if err != nil {
+				return err
+			}
+			defer originConnection.Close()
+
+			log.Info("generating the target storage connection")
+
+			targetConnection, err := datastore.NewStorageConnection(ctx, client, *targetDs)
+			if err != nil {
+				return err
+			}
+			defer targetConnection.Close()
+			// Start migrating from the old Datastore to the new one
+			log.Info("migration from origin to target started")
+
+			if err = originConnection.Migrate(ctx, *tcp, targetConnection); err != nil {
+				return fmt.Errorf("unable to migrate data from %s to %s: %w", originDs.GetName(), targetDs.GetName(), err)
+			}
+
+			log.Info("migration completed")
+
+			return nil
+		},
+	}
+
+	cmd.Flags().StringVar(&tenantControlPlane, "tenant-control-plane", "", "Namespaced-name of the TenantControlPlane that must be migrated (e.g.: default/test)")
+	cmd.Flags().StringVar(&targetDataStore, "target-datastore", "", "Name of the Datastore to which the TenantControlPlane will be migrated")
+	cmd.Flags().DurationVar(&timeout, "timeout", 5*time.Minute, "Amount of time for the context timeout")
+
+	_ = cmd.MarkFlagRequired("tenant-control-plane")
+	_ = cmd.MarkFlagRequired("target-datastore")
+
+	return cmd
+}

cmd/root.go

@@ -0,0 +1,27 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package cmd
+
+import (
+	"github.com/spf13/cobra"
+	_ "go.uber.org/automaxprocs" // Automatically set `GOMAXPROCS` to match Linux container CPU quota.
+	"k8s.io/apimachinery/pkg/runtime"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	appsv1 "k8s.io/kubernetes/pkg/apis/apps/v1"
+
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+)
+
+func NewCmd(scheme *runtime.Scheme) *cobra.Command {
+	return &cobra.Command{
+		Use:   "kamaji",
+		Short: "Build and operate Kubernetes at scale with a fraction of operational burden.",
+		PersistentPreRun: func(cmd *cobra.Command, args []string) {
+			utilruntime.Must(clientgoscheme.AddToScheme(scheme))
+			utilruntime.Must(kamajiv1alpha1.AddToScheme(scheme))
+			utilruntime.Must(appsv1.RegisterDefaults(scheme))
+		},
+	}
+}

cmd/utils/check_flags.go

@@ -0,0 +1,22 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package utils
+
+import (
+	"fmt"
+
+	"github.com/spf13/pflag"
+)
+
+func CheckFlags(flags *pflag.FlagSet, args ...string) error {
+	for _, arg := range args {
+		v, _ := flags.GetString(arg)
+
+		if len(v) == 0 {
+			return fmt.Errorf("expecting a value for --%s arg", arg)
+		}
+	}
+
+	return nil
+}

config/certmanager/certificate.yaml

@@ -0,0 +1,39 @@
+# The following manifests contain a self-signed issuer CR and a certificate CR.
+# More document can be found at https://docs.cert-manager.io
+# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes.
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+  labels:
+    app.kubernetes.io/name: issuer
+    app.kubernetes.io/instance: selfsigned-issuer
+    app.kubernetes.io/component: certificate
+    app.kubernetes.io/created-by: operator
+    app.kubernetes.io/part-of: operator
+    app.kubernetes.io/managed-by: kustomize
+  name: selfsigned-issuer
+  namespace: system
+spec:
+  selfSigned: {}
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  labels:
+    app.kubernetes.io/name: certificate
+    app.kubernetes.io/instance: serving-cert
+    app.kubernetes.io/component: certificate
+    app.kubernetes.io/created-by: operator
+    app.kubernetes.io/part-of: operator
+    app.kubernetes.io/managed-by: kustomize
+  name: serving-cert  # this name should match the one appeared in kustomizeconfig.yaml
+  namespace: system
+spec:
+  # $(SERVICE_NAME) and $(SERVICE_NAMESPACE) will be substituted by kustomize
+  dnsNames:
+  - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc
+  - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc.cluster.local
+  issuerRef:
+    kind: Issuer
+    name: selfsigned-issuer
+  secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize

config/certmanager/kustomization.yaml

@@ -0,0 +1,5 @@
+resources:
+- certificate.yaml
+
+configurations:
+- kustomizeconfig.yaml

config/certmanager/kustomizeconfig.yaml

@@ -0,0 +1,16 @@
+# This configuration is for teaching kustomize how to update name ref and var substitution 
+nameReference:
+- kind: Issuer
+  group: cert-manager.io
+  fieldSpecs:
+  - kind: Certificate
+    group: cert-manager.io
+    path: spec/issuerRef/name
+
+varReference:
+- kind: Certificate
+  group: cert-manager.io
+  path: spec/commonName
+- kind: Certificate
+  group: cert-manager.io
+  path: spec/dnsNames

config/crd/bases/kamaji.clastix.io_datastores.yaml

@@ -3,8 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.9.2
-  creationTimestamp: null
+    controller-gen.kubebuilder.io/version: v0.11.4
   name: datastores.kamaji.clastix.io
 spec:
   group: kamaji.clastix.io
@@ -60,6 +59,7 @@ spec:
                           keyPath:
                             description: Name of the key for the given Secret reference
                               where the content is stored. This value is mandatory.
+                            minLength: 1
                             type: string
                           name:
                             description: name is unique within a namespace to reference
@@ -86,6 +86,7 @@ spec:
                           keyPath:
                             description: Name of the key for the given Secret reference
                               where the content is stored. This value is mandatory.
+                            minLength: 1
                             type: string
                           name:
                             description: name is unique within a namespace to reference
@@ -106,12 +107,17 @@ spec:
                 type: object
               driver:
                 description: The driver to use to connect to the shared datastore.
+                enum:
+                - etcd
+                - MySQL
+                - PostgreSQL
                 type: string
               endpoints:
                 description: List of the endpoints to connect to the shared datastore.
                   No need for protocol, just bare IP/FQDN and port.
                 items:
                   type: string
+                minItems: 1
                 type: array
               tlsConfig:
                 description: Defines the TLS/SSL configuration required to connect
@@ -136,6 +142,7 @@ spec:
                                 description: Name of the key for the given Secret
                                   reference where the content is stored. This value
                                   is mandatory.
+                                minLength: 1
                                 type: string
                               name:
                                 description: name is unique within a namespace to
@@ -163,6 +170,7 @@ spec:
                                 description: Name of the key for the given Secret
                                   reference where the content is stored. This value
                                   is mandatory.
+                                minLength: 1
                                 type: string
                               name:
                                 description: name is unique within a namespace to
@@ -197,6 +205,7 @@ spec:
                                 description: Name of the key for the given Secret
                                   reference where the content is stored. This value
                                   is mandatory.
+                                minLength: 1
                                 type: string
                               name:
                                 description: name is unique within a namespace to
@@ -224,6 +233,7 @@ spec:
                                 description: Name of the key for the given Secret
                                   reference where the content is stored. This value
                                   is mandatory.
+                                minLength: 1
                                 type: string
                               name:
                                 description: name is unique within a namespace to

config/crd/kustomization.yaml

@@ -7,14 +7,11 @@ resources:
 #+kubebuilder:scaffold:crdkustomizeresource
 
 patchesStrategicMerge:
-# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix.
-# patches here are for enabling the conversion webhook for each CRD
-#- patches/webhook_in_clusters.yaml
+- patches/webhook_in_clusters.yaml
 #+kubebuilder:scaffold:crdkustomizewebhookpatch
 
-# [CERTMANAGER] To enable cert-manager, uncomment all the sections with [CERTMANAGER] prefix.
-# patches here are for enabling the CA injection for each CRD
-#- patches/cainjection_in_clusters.yaml
+- patches/cainjection_in_clusters.yaml
+- patches/cainjection_in_datastores.yaml
 #+kubebuilder:scaffold:crdkustomizecainjectionpatch
 
 # the following config is for teaching kustomize how to do kustomization for CRDs.

config/default/kustomization.yaml

@@ -9,67 +9,50 @@ namespace: kamaji-system
 namePrefix: kamaji-
 
 # Labels to add to all resources and selectors.
-#commonLabels:
-#  someName: someValue
+commonLabels:
+  cluster.x-k8s.io/provider: "kamaji-core"
 
 bases:
 - ../crd
 - ../rbac
 - ../manager
 - ../samples
-# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
-# crd/kustomization.yaml
-#- ../webhook
-# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required.
-#- ../certmanager
-# [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'.
-#- ../prometheus
+- ../webhook
+- ../certmanager
+- ../prometheus
 
 patchesStrategicMerge:
-# Protect the /metrics endpoint by putting it behind auth.
-# If you want your controller-manager to expose the /metrics
-# endpoint w/o any authn/z, please comment the following line.
-- manager_auth_proxy_patch.yaml
-
 # Mount the controller config file for loading manager configurations
 # through a ComponentConfig type
 #- manager_config_patch.yaml
-
-# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
-# crd/kustomization.yaml
-#- manager_webhook_patch.yaml
-
-# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'.
-# Uncomment 'CERTMANAGER' sections in crd/kustomization.yaml to enable the CA injection in the admission webhooks.
-# 'CERTMANAGER' needs to be enabled to use ca injection
-#- webhookcainjection_patch.yaml
+- manager_webhook_patch.yaml
+- webhookcainjection_patch.yaml
 
 # the following config is for teaching kustomize how to do var substitution
 vars:
-# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix.
-#- name: CERTIFICATE_NAMESPACE # namespace of the certificate CR
-#  objref:
-#    kind: Certificate
-#    group: cert-manager.io
-#    version: v1
-#    name: serving-cert # this name should match the one in certificate.yaml
-#  fieldref:
-#    fieldpath: metadata.namespace
-#- name: CERTIFICATE_NAME
-#  objref:
-#    kind: Certificate
-#    group: cert-manager.io
-#    version: v1
-#    name: serving-cert # this name should match the one in certificate.yaml
-#- name: SERVICE_NAMESPACE # namespace of the service
-#  objref:
-#    kind: Service
-#    version: v1
-#    name: webhook-service
-#  fieldref:
-#    fieldpath: metadata.namespace
-#- name: SERVICE_NAME
-#  objref:
-#    kind: Service
-#    version: v1
-#    name: webhook-service
+- name: CERTIFICATE_NAMESPACE # namespace of the certificate CR
+  objref:
+    kind: Certificate
+    group: cert-manager.io
+    version: v1
+    name: serving-cert # this name should match the one in certificate.yaml
+  fieldref:
+    fieldpath: metadata.namespace
+- name: CERTIFICATE_NAME
+  objref:
+    kind: Certificate
+    group: cert-manager.io
+    version: v1
+    name: serving-cert # this name should match the one in certificate.yaml
+- name: SERVICE_NAMESPACE # namespace of the service
+  objref:
+    kind: Service
+    version: v1
+    name: webhook-service
+  fieldref:
+    fieldpath: metadata.namespace
+- name: SERVICE_NAME
+  objref:
+    kind: Service
+    version: v1
+    name: webhook-service

config/default/manager_webhook_patch.yaml

@@ -0,0 +1,23 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: controller-manager
+  namespace: system
+spec:
+  template:
+    spec:
+      containers:
+      - name: manager
+        ports:
+        - containerPort: 9443
+          name: webhook-server
+          protocol: TCP
+        volumeMounts:
+        - mountPath: /tmp/k8s-webhook-server/serving-certs
+          name: cert
+          readOnly: true
+      volumes:
+      - name: cert
+        secret:
+          defaultMode: 420
+          secretName: webhook-server-cert

config/default/webhookcainjection_patch.yaml

@@ -0,0 +1,29 @@
+# This patch add annotation to admission webhook config and
+# the variables $(CERTIFICATE_NAMESPACE) and $(CERTIFICATE_NAME) will be substituted by kustomize.
+apiVersion: admissionregistration.k8s.io/v1
+kind: MutatingWebhookConfiguration
+metadata:
+  labels:
+    app.kubernetes.io/name: mutatingwebhookconfiguration
+    app.kubernetes.io/instance: mutating-webhook-configuration
+    app.kubernetes.io/component: webhook
+    app.kubernetes.io/created-by: operator
+    app.kubernetes.io/part-of: operator
+    app.kubernetes.io/managed-by: kustomize
+  name: mutating-webhook-configuration
+  annotations:
+    cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME)
+---
+apiVersion: admissionregistration.k8s.io/v1
+kind: ValidatingWebhookConfiguration
+metadata:
+  labels:
+    app.kubernetes.io/name: validatingwebhookconfiguration
+    app.kubernetes.io/instance: validating-webhook-configuration
+    app.kubernetes.io/component: webhook
+    app.kubernetes.io/created-by: operator
+    app.kubernetes.io/part-of: operator
+    app.kubernetes.io/managed-by: kustomize
+  name: validating-webhook-configuration
+  annotations:
+    cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME)

config/manager/kustomization.yaml

@@ -13,4 +13,4 @@ kind: Kustomization
 images:
 - name: controller
   newName: clastix/kamaji
-  newTag: v0.1.1
+  newTag: v0.4.2

config/manager/manager.yaml

@@ -26,12 +26,27 @@ spec:
         runAsNonRoot: true
       containers:
       - command:
-        - /manager
+        - /kamaji
         args:
+        - manager
         - --leader-elect
+        - --datastore=kamaji-etcd
+        env:
+          - name: POD_NAMESPACE
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.namespace
+          - name: SERVICE_ACCOUNT
+            valueFrom:
+              fieldRef:
+                fieldPath: spec.serviceAccountName
         image: controller:latest
         imagePullPolicy: Always
         name: manager
+        ports:
+          - containerPort: 8080
+            name: metrics
+            protocol: TCP
         securityContext:
           allowPrivilegeEscalation: false
         livenessProbe:

config/metadata.yaml

@@ -0,0 +1,10 @@
+# maps release series of major.minor to cluster-api contract version
+# the contract version may change between minor or major versions, but *not*
+# between patch versions.
+#
+# update this file only when a new major or minor version is released
+apiVersion: clusterctl.cluster.x-k8s.io/v1alpha3
+releaseSeries:
+  - major: 0
+    minor: 3
+    contract: v1beta1

config/prometheus/kustomization.yaml

@@ -1,2 +1,5 @@
 resources:
 - monitor.yaml
+
+configurations:
+- kustomizeconfig.yaml

config/prometheus/kustomizeconfig.yaml

@@ -0,0 +1,4 @@
+varReference:
+  - kind: ServiceMonitor
+    group: monitoring.coreos.com
+    path: spec/namespaceSelector/matchNames

config/prometheus/monitor.yaml

@@ -1,4 +1,3 @@
-
 # Prometheus Monitor Service (Metrics)
 apiVersion: monitoring.coreos.com/v1
 kind: ServiceMonitor
@@ -10,11 +9,11 @@ metadata:
 spec:
   endpoints:
     - path: /metrics
-      port: https
-      scheme: https
-      bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
-      tlsConfig:
-        insecureSkipVerify: true
+      port: metrics
+      scheme: http
+  namespaceSelector:
+    matchNames:
+      - $(SERVICE_NAMESPACE)
   selector:
     matchLabels:
       control-plane: controller-manager

config/rbac/role.yaml

@@ -2,7 +2,6 @@
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
-  creationTimestamp: null
   name: manager-role
 rules:
 - apiGroups:
@@ -17,6 +16,16 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - batch
+  resources:
+  - jobs
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - watch
 - apiGroups:
   - ""
   resources:

config/samples/kamaji_v1alpha1_datastore_mysql_bronze.yaml

@@ -0,0 +1,34 @@
+apiVersion: kamaji.clastix.io/v1alpha1
+kind: DataStore
+metadata:
+  name: mysql-bronze
+spec:
+  driver: MySQL
+  endpoints:
+    - bronze.mysql-system.svc:3306
+  basicAuth:
+    username:
+      content: cm9vdA==
+    password:
+      secretReference:
+        name: mysql-bronze-config
+        namespace: mysql-system
+        keyPath: MYSQL_ROOT_PASSWORD
+  tlsConfig:
+    certificateAuthority:
+      certificate:
+        secretReference:
+          name: mysql-bronze-config
+          namespace: mysql-system
+          keyPath: "ca.crt"
+    clientCertificate:
+      certificate:
+        secretReference:
+          name: mysql-bronze-config
+          namespace: mysql-system
+          keyPath: "server.crt"
+      privateKey:
+        secretReference:
+          name: mysql-bronze-config
+          namespace: mysql-system
+          keyPath: "server.key"

config/samples/kamaji_v1alpha1_datastore_mysql_gold.yaml

@@ -1,34 +1,34 @@
 apiVersion: kamaji.clastix.io/v1alpha1
 kind: DataStore
 metadata:
-  name: mysql
+  name: mysql-gold
 spec:
   driver: MySQL
   endpoints:
-    - mariadb.kamaji-system.svc:3306
+    - gold.mysql-system.svc:3306
   basicAuth:
     username:
       content: cm9vdA==
     password:
       secretReference:
-        name: mysql-config
-        namespace: kamaji-system
+        name: mysql-gold-config
+        namespace: mysql-system
         keyPath: MYSQL_ROOT_PASSWORD
   tlsConfig:
     certificateAuthority:
       certificate:
         secretReference:
-          name: mysql-config
-          namespace: kamaji-system
+          name: mysql-gold-config
+          namespace: mysql-system
           keyPath: "ca.crt"
     clientCertificate:
       certificate:
         secretReference:
-          name: mysql-config
-          namespace: kamaji-system
+          name: mysql-gold-config
+          namespace: mysql-system
           keyPath: "server.crt"
       privateKey:
         secretReference:
-          name: mysql-config
-          namespace: kamaji-system
+          name: mysql-gold-config
+          namespace: mysql-system
           keyPath: "server.key"

config/samples/kamaji_v1alpha1_datastore_mysql_silver.yaml

@@ -0,0 +1,34 @@
+apiVersion: kamaji.clastix.io/v1alpha1
+kind: DataStore
+metadata:
+  name: mysql-silver
+spec:
+  driver: MySQL
+  endpoints:
+    - silver.mysql-system.svc:3306
+  basicAuth:
+    username:
+      content: cm9vdA==
+    password:
+      secretReference:
+        name: mysql-silver-config
+        namespace: mysql-system
+        keyPath: MYSQL_ROOT_PASSWORD
+  tlsConfig:
+    certificateAuthority:
+      certificate:
+        secretReference:
+          name: mysql-silver-config
+          namespace: mysql-system
+          keyPath: "ca.crt"
+    clientCertificate:
+      certificate:
+        secretReference:
+          name: mysql-silver-config
+          namespace: mysql-system
+          keyPath: "server.crt"
+      privateKey:
+        secretReference:
+          name: mysql-silver-config
+          namespace: mysql-system
+          keyPath: "server.key"

config/samples/kamaji_v1alpha1_datastore_postgresql_bronze.yaml

@@ -0,0 +1,37 @@
+apiVersion: kamaji.clastix.io/v1alpha1
+kind: DataStore
+metadata:
+  name: postgresql-bronze
+spec:
+  driver: PostgreSQL
+  endpoints:
+    - postgres-bronze-rw.postgres-system.svc:5432
+  basicAuth:
+    username:
+      secretReference:
+        name: postgres-bronze-superuser
+        namespace: postgres-system
+        keyPath: username
+    password:
+      secretReference:
+        name: postgres-bronze-superuser
+        namespace: postgres-system
+        keyPath: password
+  tlsConfig:
+    certificateAuthority:
+      certificate:
+        secretReference:
+          name: postgres-bronze-ca
+          namespace: postgres-system
+          keyPath: ca.crt
+    clientCertificate:
+      certificate:
+        secretReference:
+          name: postgres-bronze-root-cert
+          namespace: postgres-system
+          keyPath: tls.crt
+      privateKey:
+        secretReference:
+          name: postgres-bronze-root-cert
+          namespace: postgres-system
+          keyPath: tls.key

config/samples/kamaji_v1alpha1_datastore_postgresql_gold.yaml

@@ -1,37 +1,37 @@
 apiVersion: kamaji.clastix.io/v1alpha1
 kind: DataStore
 metadata:
-  name: postgresql
+  name: postgresql-gold
 spec:
   driver: PostgreSQL
   endpoints:
-    - postgresql-rw.kamaji-system.svc:5432
+    - postgres-gold-rw.postgres-system.svc:5432
   basicAuth:
     username:
       secretReference:
-        name: postgresql-superuser
-        namespace: kamaji-system
+        name: postgres-gold-superuser
+        namespace: postgres-system
         keyPath: username
     password:
       secretReference:
-        name: postgresql-superuser
-        namespace: kamaji-system
+        name: postgres-gold-superuser
+        namespace: postgres-system
         keyPath: password
   tlsConfig:
     certificateAuthority:
       certificate:
         secretReference:
-          name: postgresql-ca
-          namespace: kamaji-system
+          name: postgres-gold-ca
+          namespace: postgres-system
           keyPath: ca.crt
     clientCertificate:
       certificate:
         secretReference:
-          name: postgres-root-cert
-          namespace: kamaji-system
+          name: postgres-gold-root-cert
+          namespace: postgres-system
           keyPath: tls.crt
       privateKey:
         secretReference:
-          name: postgres-root-cert
-          namespace: kamaji-system
+          name: postgres-gold-root-cert
+          namespace: postgres-system
           keyPath: tls.key

config/samples/kamaji_v1alpha1_datastore_postgresql_silver.yaml

@@ -0,0 +1,37 @@
+apiVersion: kamaji.clastix.io/v1alpha1
+kind: DataStore
+metadata:
+  name: postgresql-silver
+spec:
+  driver: PostgreSQL
+  endpoints:
+    - postgres-silver-rw.postgres-system.svc:5432
+  basicAuth:
+    username:
+      secretReference:
+        name: postgres-silver-superuser
+        namespace: postgres-system
+        keyPath: username
+    password:
+      secretReference:
+        name: postgres-silver-superuser
+        namespace: postgres-system
+        keyPath: password
+  tlsConfig:
+    certificateAuthority:
+      certificate:
+        secretReference:
+          name: postgres-silver-ca
+          namespace: postgres-system
+          keyPath: ca.crt
+    clientCertificate:
+      certificate:
+        secretReference:
+          name: postgres-silver-root-cert
+          namespace: postgres-system
+          keyPath: tls.crt
+      privateKey:
+        secretReference:
+          name: postgres-silver-root-cert
+          namespace: postgres-system
+          keyPath: tls.key

config/samples/kamaji_v1alpha1_tenantcontrolplane.yaml

@@ -1,22 +1,24 @@
 apiVersion: kamaji.clastix.io/v1alpha1
 kind: TenantControlPlane
 metadata:
-  name: test
+  name: k8s-126
+  labels:
+    tenant.clastix.io: k8s-126
 spec:
   controlPlane:
     deployment:
-      replicas: 1
+      replicas: 2
     service:
       serviceType: LoadBalancer
   kubernetes:
-    version: "v1.23.1"
+    version: "v1.26.0"
     kubelet:
-      cgroupfs: cgroupfs
-    admissionControllers:
-      - ResourceQuota
-      - LimitRanger
+      cgroupfs: systemd
   networkProfile:
     port: 6443
   addons:
     coreDNS: {}
     kubeProxy: {}
+    konnectivity:
+      server:
+        port: 8132

config/samples/kamaji_v1alpha1_tenantcontrolplane_additionalcontainers.yaml

@@ -0,0 +1,32 @@
+apiVersion: kamaji.clastix.io/v1alpha1
+kind: TenantControlPlane
+metadata:
+  name: additionalcontainers
+  labels:
+    tenant.clastix.io: additionalcontainers
+spec:
+  dataStore: postgresql-bronze
+  controlPlane:
+    deployment:
+      replicas: 1
+      additionalInitContainers:
+        - name: init
+          image: registry.k8s.io/e2e-test-images/busybox:1.29-4
+          command:
+            - /bin/sh
+            - -c
+            - echo hello world
+      additionalContainers:
+        - name: nginx
+          image: registry.k8s.io/e2e-test-images/nginx:1.15-4
+    service:
+      serviceType: LoadBalancer
+  kubernetes:
+    version: "v1.26.0"
+    kubelet:
+      cgroupfs: systemd
+  networkProfile:
+    port: 6443
+  addons:
+    coreDNS: {}
+    kubeProxy: {}

config/samples/kamaji_v1alpha1_tenantcontrolplane_additionalvolumes.yaml

@@ -0,0 +1,62 @@
+apiVersion: kamaji.clastix.io/v1alpha1
+kind: TenantControlPlane
+metadata:
+  name: additional-volumes
+  labels:
+    tenant.clastix.io: additional-volumes
+spec:
+  controlPlane:
+    deployment:
+      replicas: 1
+      additionalVolumes:
+        - name: api-server-volume
+          configMap:
+            name: api-server-extra-cm
+        - name: controller-manager-volume
+          configMap:
+            name: controller-manager-extra-cm
+        - name: scheduler-volume
+          configMap:
+            name: scheduler-extra-cm
+      additionalVolumeMounts:
+        apiServer:
+          - name: api-server-volume
+            mountPath: "/tmp/api-server"
+        controllerManager:
+          - name: controller-manager-volume
+            mountPath: "/tmp/controller-manager"
+        scheduler:
+          - name: scheduler-volume
+            mountPath: "/tmp/scheduler"
+    service:
+      serviceType: LoadBalancer
+  kubernetes:
+    version: "v1.26.0"
+    kubelet:
+      cgroupfs: systemd
+  networkProfile:
+    port: 6443
+  addons:
+    coreDNS: {}
+    kubeProxy: {}
+---
+apiVersion: v1
+data:
+  api-server: "This is an API Server volume"
+kind: ConfigMap
+metadata:
+  name: api-server-extra-cm
+---
+apiVersion: v1
+data:
+  controller-manager: "This is a Controller Manager volume"
+kind: ConfigMap
+metadata:
+  name: controller-manager-extra-cm
+---
+apiVersion: v1
+data:
+  controller-manager: "This is a Scheduler volume"
+kind: ConfigMap
+metadata:
+  name: scheduler-extra-cm

config/samples/kamaji_v1alpha1_tenantcontrolplane_kine.yaml

@@ -0,0 +1,20 @@
+apiVersion: kamaji.clastix.io/v1alpha1
+kind: TenantControlPlane
+metadata:
+  name: kine
+  labels:
+    tenant.clastix.io: kine
+spec:
+  addons:
+    coreDNS: {}
+    kubeProxy: {}
+  controlPlane:
+    deployment:
+      replicas: 1
+    service:
+      serviceType: LoadBalancer
+  dataStore: postgresql-bronze
+  kubernetes:
+    kubelet:
+      cgroupfs: systemd
+    version: v1.26.0

config/samples/kamaji_v1alpha1_tenantcontrolplane_konnectivity.yaml

@@ -0,0 +1,23 @@
+apiVersion: kamaji.clastix.io/v1alpha1
+kind: TenantControlPlane
+metadata:
+  name: konnectivity-addon
+  labels:
+    tenant.clastix.io: konnectivity-addon
+spec:
+  deployment:
+    replicas: 2
+  service:
+    serviceType: LoadBalancer
+  kubernetes:
+    version: "v1.26.0"
+    kubelet:
+      cgroupfs: systemd
+  networkProfile:
+    port: 6443
+  addons:
+    coreDNS: {}
+    kubeProxy: {}
+    konnectivity:
+      server:
+        port: 8132

config/webhook/kustomization.yaml

@@ -0,0 +1,6 @@
+resources:
+- manifests.yaml
+- service.yaml
+
+configurations:
+- kustomizeconfig.yaml

config/webhook/kustomizeconfig.yaml

@@ -0,0 +1,25 @@
+# the following config is for teaching kustomize where to look at when substituting vars.
+# It requires kustomize v2.1.0 or newer to work properly.
+nameReference:
+- kind: Service
+  version: v1
+  fieldSpecs:
+  - kind: MutatingWebhookConfiguration
+    group: admissionregistration.k8s.io
+    path: webhooks/clientConfig/service/name
+  - kind: ValidatingWebhookConfiguration
+    group: admissionregistration.k8s.io
+    path: webhooks/clientConfig/service/name
+
+namespace:
+- kind: MutatingWebhookConfiguration
+  group: admissionregistration.k8s.io
+  path: webhooks/clientConfig/service/namespace
+  create: true
+- kind: ValidatingWebhookConfiguration
+  group: admissionregistration.k8s.io
+  path: webhooks/clientConfig/service/namespace
+  create: true
+
+varReference:
+- path: metadata/annotations

config/webhook/manifests.yaml

@@ -0,0 +1,92 @@
+---
+apiVersion: admissionregistration.k8s.io/v1
+kind: MutatingWebhookConfiguration
+metadata:
+  name: mutating-webhook-configuration
+webhooks:
+- admissionReviewVersions:
+  - v1
+  clientConfig:
+    service:
+      name: webhook-service
+      namespace: system
+      path: /mutate-kamaji-clastix-io-v1alpha1-tenantcontrolplane
+  failurePolicy: Fail
+  name: mtenantcontrolplane.kb.io
+  rules:
+  - apiGroups:
+    - kamaji.clastix.io
+    apiVersions:
+    - v1alpha1
+    operations:
+    - CREATE
+    - UPDATE
+    resources:
+    - tenantcontrolplanes
+  sideEffects: None
+---
+apiVersion: admissionregistration.k8s.io/v1
+kind: ValidatingWebhookConfiguration
+metadata:
+  name: validating-webhook-configuration
+webhooks:
+- admissionReviewVersions:
+  - v1
+  clientConfig:
+    service:
+      name: webhook-service
+      namespace: system
+      path: /validate--v1-secret
+  failurePolicy: Ignore
+  name: vdatastoresecrets.kb.io
+  rules:
+  - apiGroups:
+    - ""
+    apiVersions:
+    - v1
+    operations:
+    - DELETE
+    resources:
+    - secrets
+  sideEffects: None
+- admissionReviewVersions:
+  - v1
+  clientConfig:
+    service:
+      name: webhook-service
+      namespace: system
+      path: /validate-kamaji-clastix-io-v1alpha1-datastore
+  failurePolicy: Fail
+  name: vdatastore.kb.io
+  rules:
+  - apiGroups:
+    - kamaji.clastix.io
+    apiVersions:
+    - v1alpha1
+    operations:
+    - CREATE
+    - UPDATE
+    - DELETE
+    resources:
+    - datastores
+  sideEffects: None
+- admissionReviewVersions:
+  - v1
+  clientConfig:
+    service:
+      name: webhook-service
+      namespace: system
+      path: /validate-kamaji-clastix-io-v1alpha1-tenantcontrolplane
+  failurePolicy: Fail
+  name: vtenantcontrolplane.kb.io
+  rules:
+  - apiGroups:
+    - kamaji.clastix.io
+    apiVersions:
+    - v1alpha1
+    operations:
+    - CREATE
+    - UPDATE
+    resources:
+    - tenantcontrolplanes
+  sideEffects: None

config/webhook/service.yaml

@@ -0,0 +1,20 @@
+
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app.kubernetes.io/name: service
+    app.kubernetes.io/instance: webhook-service
+    app.kubernetes.io/component: webhook
+    app.kubernetes.io/created-by: operator
+    app.kubernetes.io/part-of: operator
+    app.kubernetes.io/managed-by: kustomize
+  name: webhook-service
+  namespace: system
+spec:
+  ports:
+    - port: 443
+      protocol: TCP
+      targetPort: 9443
+  selector:
+    control-plane: controller-manager

controllers/cert_channel.go

@@ -0,0 +1,10 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package controllers
+
+import (
+	"sigs.k8s.io/controller-runtime/pkg/event"
+)
+
+type CertificateChannel chan event.GenericEvent

controllers/certificate_lifecycle_controller.go

@@ -0,0 +1,163 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package controllers
+
+import (
+	"context"
+	"crypto/x509"
+	"fmt"
+	"time"
+
+	"github.com/pkg/errors"
+	corev1 "k8s.io/api/core/v1"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/sets"
+	clientcmdapiv1 "k8s.io/client-go/tools/clientcmd/api/v1"
+	controllerruntime "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/builder"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/event"
+	"sigs.k8s.io/controller-runtime/pkg/log"
+	"sigs.k8s.io/controller-runtime/pkg/predicate"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+	"github.com/clastix/kamaji/internal/constants"
+	"github.com/clastix/kamaji/internal/crypto"
+	"github.com/clastix/kamaji/internal/utilities"
+)
+
+type CertificateLifecycle struct {
+	Channel CertificateChannel
+	client  client.Client
+}
+
+func (s *CertificateLifecycle) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) {
+	logger := log.FromContext(ctx)
+
+	logger.Info("starting CertificateLifecycle handling")
+
+	secret := corev1.Secret{}
+	err := s.client.Get(ctx, request.NamespacedName, &secret)
+	if k8serrors.IsNotFound(err) {
+		logger.Info("resource have been deleted, skipping")
+
+		return reconcile.Result{}, nil
+	}
+	if err != nil {
+		logger.Error(err, "cannot retrieve the required resource")
+
+		return reconcile.Result{}, err
+	}
+
+	checkType, ok := secret.GetLabels()[constants.ControllerLabelResource]
+	if !ok {
+		logger.Info("missing controller label, shouldn't happen")
+
+		return reconcile.Result{}, nil
+	}
+
+	var crt *x509.Certificate
+
+	switch checkType {
+	case "x509":
+		crt, err = s.extractCertificateFromBareSecret(secret)
+	case "kubeconfig":
+		crt, err = s.extractCertificateFromKubeconfig(secret)
+	default:
+		err = fmt.Errorf("unsupported strategy, %s", checkType)
+	}
+
+	if err != nil {
+		logger.Error(err, "skipping reconciliation")
+
+		return reconcile.Result{}, nil
+	}
+
+	deadline := time.Now().AddDate(0, 0, 1)
+
+	if deadline.After(crt.NotAfter) {
+		logger.Info("certificate near expiration, must be rotated")
+
+		s.Channel <- event.GenericEvent{Object: &kamajiv1alpha1.TenantControlPlane{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      secret.GetOwnerReferences()[0].Name,
+				Namespace: secret.Namespace,
+			},
+		}}
+
+		logger.Info("certificate rotation triggered")
+
+		return reconcile.Result{}, nil
+	}
+
+	after := crt.NotAfter.Sub(deadline)
+
+	logger.Info("certificate is still valid, enqueuing back", "after", after.String())
+
+	return reconcile.Result{Requeue: true, RequeueAfter: after}, nil
+}
+
+func (s *CertificateLifecycle) extractCertificateFromBareSecret(secret corev1.Secret) (*x509.Certificate, error) {
+	var crt *x509.Certificate
+	var err error
+
+	for _, v := range secret.Data {
+		if crt, err = crypto.ParseCertificateBytes(v); err == nil {
+			break
+		}
+	}
+
+	if crt == nil {
+		return nil, fmt.Errorf("none of the provided keys is containing a valid x509 certificate")
+	}
+
+	return crt, nil
+}
+
+func (s *CertificateLifecycle) extractCertificateFromKubeconfig(secret corev1.Secret) (*x509.Certificate, error) {
+	var kc *clientcmdapiv1.Config
+	var err error
+
+	for k := range secret.Data {
+		if kc, err = utilities.DecodeKubeconfig(secret, k); err == nil {
+			break
+		}
+	}
+
+	if kc == nil {
+		return nil, fmt.Errorf("none of the provided keys is containing a valid kubeconfig")
+	}
+
+	crt, err := crypto.ParseCertificateBytes(kc.AuthInfos[0].AuthInfo.ClientCertificateData)
+	if err != nil {
+		return nil, errors.Wrap(err, "cannot parse kubeconfig certificate bytes")
+	}
+
+	return crt, nil
+}
+
+func (s *CertificateLifecycle) SetupWithManager(mgr controllerruntime.Manager) error {
+	s.client = mgr.GetClient()
+
+	supportedStrategies := sets.New[string]("x509", "kubeconfig")
+
+	return controllerruntime.NewControllerManagedBy(mgr).
+		For(&corev1.Secret{}, builder.WithPredicates(predicate.NewPredicateFuncs(func(object client.Object) bool {
+			labels := object.GetLabels()
+
+			if labels == nil {
+				return false
+			}
+
+			value, ok := labels[constants.ControllerLabelResource]
+			if !ok {
+				return false
+			}
+
+			return supportedStrategies.Has(value)
+		}))).
+		Complete(s)
+}

controllers/datastore_controller.go

@@ -5,7 +5,6 @@ package controllers
 
 import (
 	"context"
-	"fmt"
 
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/fields"
@@ -15,7 +14,6 @@ import (
 	controllerruntime "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/builder"
 	"sigs.k8s.io/controller-runtime/pkg/client"
-	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 	"sigs.k8s.io/controller-runtime/pkg/event"
 	"sigs.k8s.io/controller-runtime/pkg/handler"
 	"sigs.k8s.io/controller-runtime/pkg/log"
@@ -24,15 +22,10 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/source"
 
 	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
-	"github.com/clastix/kamaji/indexers"
-)
-
-const (
-	dataStoreFinalizer = "finalizer.kamaji.clastix.io/datastore"
 )
 
 type DataStore struct {
-	client client.Client
+	Client client.Client
 	// TenantControlPlaneTrigger is the channel used to communicate across the controllers:
 	// if a Data Source is updated we have to be sure that the reconciliation of the certificates content
 	// for each Tenant Control Plane is put in place properly.
@@ -46,75 +39,22 @@ func (r *DataStore) Reconcile(ctx context.Context, request reconcile.Request) (r
 	log := log.FromContext(ctx)
 
 	ds := &kamajiv1alpha1.DataStore{}
-	if err := r.client.Get(ctx, request.NamespacedName, ds); err != nil {
-		if k8serrors.IsNotFound(err) {
-			return reconcile.Result{}, nil
-		}
+	err := r.Client.Get(ctx, request.NamespacedName, ds)
+	if k8serrors.IsNotFound(err) {
+		log.Info("resource have been deleted, skipping")
 
-		log.Error(err, "unable to retrieve the request")
-
-		return reconcile.Result{}, err
+		return reconcile.Result{}, nil
 	}
-	// Managing the finalizer, required to don't drop a DataSource if this is still used by a Tenant Control Plane.
-	switch {
-	case ds.DeletionTimestamp != nil && controllerutil.ContainsFinalizer(ds, dataStoreFinalizer):
-		log.Info("marked for deletion, checking conditions")
-
-		if len(ds.Status.UsedBy) == 0 {
-			log.Info("resource is no more used by any Tenant Control Plane")
-
-			controllerutil.RemoveFinalizer(ds, dataStoreFinalizer)
-
-			return reconcile.Result{}, r.client.Update(ctx, ds)
-		}
-
-		log.Info("DataStore is still used by some Tenant Control Planes, cannot be removed")
-	case ds.DeletionTimestamp == nil && !controllerutil.ContainsFinalizer(ds, dataStoreFinalizer):
-		log.Info("the resource is missing the required finalizer, adding it")
-
-		controllerutil.AddFinalizer(ds, dataStoreFinalizer)
-
-		return reconcile.Result{}, r.client.Update(ctx, ds)
-	}
-	// A Data Source can trigger several Tenant Control Planes and requires a minimum validation:
-	// we have to ensure the data provided by the Data Source is valid and referencing an existing Secret object.
-	if _, err := ds.Spec.TLSConfig.CertificateAuthority.Certificate.GetContent(ctx, r.client); err != nil {
-		log.Error(err, "invalid Certificate Authority data")
-
-		return reconcile.Result{}, err
-	}
-
-	if ds.Spec.Driver == kamajiv1alpha1.EtcdDriver {
-		if ds.Spec.TLSConfig.CertificateAuthority.PrivateKey == nil {
-			err := fmt.Errorf("a valid private key is required for the etcd driver")
-
-			log.Error(err, "missing Certificate Authority private key data")
-
-			return reconcile.Result{}, err
-		}
-		if _, err := ds.Spec.TLSConfig.CertificateAuthority.PrivateKey.GetContent(ctx, r.client); err != nil {
-			log.Error(err, "invalid Certificate Authority private key data")
-
-			return reconcile.Result{}, err
-		}
-	}
-
-	if _, err := ds.Spec.TLSConfig.ClientCertificate.Certificate.GetContent(ctx, r.client); err != nil {
-		log.Error(err, "invalid Client Certificate data")
-
-		return reconcile.Result{}, err
-	}
-
-	if _, err := ds.Spec.TLSConfig.ClientCertificate.PrivateKey.GetContent(ctx, r.client); err != nil {
-		log.Error(err, "invalid Client Certificate private key data")
+	if err != nil {
+		log.Error(err, "cannot retrieve the required resource")
 
 		return reconcile.Result{}, err
 	}
 
 	tcpList := kamajiv1alpha1.TenantControlPlaneList{}
 
-	if err := r.client.List(ctx, &tcpList, client.MatchingFieldsSelector{
-		Selector: fields.OneTermEqualSelector(indexers.TenantControlPlaneUsedDataStoreKey, ds.GetName()),
+	if err := r.Client.List(ctx, &tcpList, client.MatchingFieldsSelector{
+		Selector: fields.OneTermEqualSelector(kamajiv1alpha1.TenantControlPlaneUsedDataStoreKey, ds.GetName()),
 	}); err != nil {
 		log.Error(err, "cannot retrieve list of the Tenant Control Plane using the following instance")
 
@@ -128,7 +68,7 @@ func (r *DataStore) Reconcile(ctx context.Context, request reconcile.Request) (r
 
 	ds.Status.UsedBy = tcpSets.List()
 
-	if err := r.client.Status().Update(ctx, ds); err != nil {
+	if err := r.Client.Status().Update(ctx, ds); err != nil {
 		log.Error(err, "cannot update the status for the given instance")
 
 		return reconcile.Result{}, err
@@ -143,12 +83,6 @@ func (r *DataStore) Reconcile(ctx context.Context, request reconcile.Request) (r
 	return reconcile.Result{}, nil
 }
 
-func (r *DataStore) InjectClient(client client.Client) error {
-	r.client = client
-
-	return nil
-}
-
 func (r *DataStore) SetupWithManager(mgr controllerruntime.Manager) error {
 	enqueueFn := func(tcp *kamajiv1alpha1.TenantControlPlane, limitingInterface workqueue.RateLimitingInterface) {
 		if dataStoreName := tcp.Status.Storage.DataStoreName; len(dataStoreName) > 0 {
@@ -164,15 +98,15 @@ func (r *DataStore) SetupWithManager(mgr controllerruntime.Manager) error {
 		For(&kamajiv1alpha1.DataStore{}, builder.WithPredicates(
 			predicate.ResourceVersionChangedPredicate{},
 		)).
-		Watches(&source.Kind{Type: &kamajiv1alpha1.TenantControlPlane{}}, handler.Funcs{
-			CreateFunc: func(createEvent event.CreateEvent, limitingInterface workqueue.RateLimitingInterface) {
+		WatchesRawSource(source.Kind(mgr.GetCache(), &kamajiv1alpha1.TenantControlPlane{}), handler.Funcs{
+			CreateFunc: func(_ context.Context, createEvent event.CreateEvent, limitingInterface workqueue.RateLimitingInterface) {
 				enqueueFn(createEvent.Object.(*kamajiv1alpha1.TenantControlPlane), limitingInterface)
 			},
-			UpdateFunc: func(updateEvent event.UpdateEvent, limitingInterface workqueue.RateLimitingInterface) {
+			UpdateFunc: func(_ context.Context, updateEvent event.UpdateEvent, limitingInterface workqueue.RateLimitingInterface) {
 				enqueueFn(updateEvent.ObjectOld.(*kamajiv1alpha1.TenantControlPlane), limitingInterface)
 				enqueueFn(updateEvent.ObjectNew.(*kamajiv1alpha1.TenantControlPlane), limitingInterface)
 			},
-			DeleteFunc: func(deleteEvent event.DeleteEvent, limitingInterface workqueue.RateLimitingInterface) {
+			DeleteFunc: func(_ context.Context, deleteEvent event.DeleteEvent, limitingInterface workqueue.RateLimitingInterface) {
 				enqueueFn(deleteEvent.Object.(*kamajiv1alpha1.TenantControlPlane), limitingInterface)
 			},
 		}).

controllers/finalizers/tcp.go

@@ -0,0 +1,11 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package finalizers
+
+const (
+	// DatastoreFinalizer is using a wrong name, since it's related to the underlying datastore.
+	DatastoreFinalizer       = "finalizer.kamaji.clastix.io"
+	DatastoreSecretFinalizer = "finalizer.kamaji.clastix.io/datastore-secret"
+	SootFinalizer            = "finalizer.kamaji.clastix.io/soot"
+)

controllers/resources.go

@@ -10,8 +10,11 @@ import (
 	"github.com/google/uuid"
 	k8stypes "k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 
 	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+	"github.com/clastix/kamaji/controllers/finalizers"
+	builder "github.com/clastix/kamaji/internal/builders/controlplane"
 	"github.com/clastix/kamaji/internal/datastore"
 	"github.com/clastix/kamaji/internal/resources"
 	ds "github.com/clastix/kamaji/internal/resources/datastore"
@@ -19,20 +22,25 @@ import (
 )
 
 type GroupResourceBuilderConfiguration struct {
-	client              client.Client
-	log                 logr.Logger
-	tcpReconcilerConfig TenantControlPlaneReconcilerConfig
-	tenantControlPlane  kamajiv1alpha1.TenantControlPlane
-	Connection          datastore.Connection
-	DataStore           kamajiv1alpha1.DataStore
+	client               client.Client
+	log                  logr.Logger
+	tcpReconcilerConfig  TenantControlPlaneReconcilerConfig
+	tenantControlPlane   kamajiv1alpha1.TenantControlPlane
+	Connection           datastore.Connection
+	DataStore            kamajiv1alpha1.DataStore
+	KamajiNamespace      string
+	KamajiServiceAccount string
+	KamajiService        string
+	KamajiMigrateImage   string
 }
 
-type GroupDeleteableResourceBuilderConfiguration struct {
+type GroupDeletableResourceBuilderConfiguration struct {
 	client              client.Client
 	log                 logr.Logger
 	tcpReconcilerConfig TenantControlPlaneReconcilerConfig
 	tenantControlPlane  kamajiv1alpha1.TenantControlPlane
 	connection          datastore.Connection
+	dataStore           kamajiv1alpha1.DataStore
 }
 
 // GetResources returns a list of resources that will be used to provide tenant control planes
@@ -44,32 +52,60 @@ func GetResources(config GroupResourceBuilderConfiguration) []resources.Resource
 
 // GetDeletableResources returns a list of resources that have to be deleted when tenant control planes are deleted
 // Currently there is only a default approach
-// TODO: the idea of this function is to become a factory to return the group of deleteable resources according to the given configuration.
-func GetDeletableResources(config GroupDeleteableResourceBuilderConfiguration) []resources.DeleteableResource {
-	return getDefaultDeleteableResources(config)
+// TODO: the idea of this function is to become a factory to return the group of deletable resources according to the given configuration.
+func GetDeletableResources(tcp *kamajiv1alpha1.TenantControlPlane, config GroupDeletableResourceBuilderConfiguration) []resources.DeletableResource {
+	var res []resources.DeletableResource
+
+	if controllerutil.ContainsFinalizer(tcp, finalizers.DatastoreFinalizer) {
+		res = append(res, &ds.Setup{
+			Client:     config.client,
+			Connection: config.connection,
+		})
+		res = append(res, &ds.Config{
+			Client:     config.client,
+			ConnString: config.connection.GetConnectionString(),
+			DataStore:  config.dataStore,
+		})
+	}
+
+	return res
 }
 
 func getDefaultResources(config GroupResourceBuilderConfiguration) []resources.Resource {
-	resources := append(getUpgradeResources(config.client), getKubernetesServiceResources(config.client)...)
+	resources := getDataStoreMigratingResources(config.client, config.KamajiNamespace, config.KamajiMigrateImage, config.KamajiServiceAccount, config.KamajiService)
+	resources = append(resources, getUpgradeResources(config.client)...)
+	resources = append(resources, getKubernetesServiceResources(config.client)...)
 	resources = append(resources, getKubeadmConfigResources(config.client, getTmpDirectory(config.tcpReconcilerConfig.TmpBaseDirectory, config.tenantControlPlane), config.DataStore)...)
 	resources = append(resources, getKubernetesCertificatesResources(config.client, config.tcpReconcilerConfig, config.tenantControlPlane)...)
 	resources = append(resources, getKubeconfigResources(config.client, config.tcpReconcilerConfig, config.tenantControlPlane)...)
 	resources = append(resources, getKubernetesStorageResources(config.client, config.Connection, config.DataStore)...)
-	resources = append(resources, getInternalKonnectivityResources(config.client)...)
+	resources = append(resources, getKonnectivityServerRequirementsResources(config.client)...)
 	resources = append(resources, getKubernetesDeploymentResources(config.client, config.tcpReconcilerConfig, config.DataStore)...)
+	resources = append(resources, getKonnectivityServerPatchResources(config.client)...)
+	resources = append(resources, getDataStoreMigratingCleanup(config.client, config.KamajiNamespace)...)
 	resources = append(resources, getKubernetesIngressResources(config.client)...)
-	resources = append(resources, getKubeadmPhaseResources(config.client)...)
-	resources = append(resources, getKubeadmAddonResources(config.client)...)
-	resources = append(resources, getExternalKonnectivityResources(config.client)...)
 
 	return resources
 }
 
-func getDefaultDeleteableResources(config GroupDeleteableResourceBuilderConfiguration) []resources.DeleteableResource {
-	return []resources.DeleteableResource{
-		&ds.Setup{
-			Client:     config.client,
-			Connection: config.connection,
+func getDataStoreMigratingCleanup(c client.Client, kamajiNamespace string) []resources.Resource {
+	return []resources.Resource{
+		&ds.Migrate{
+			Client:          c,
+			KamajiNamespace: kamajiNamespace,
+			ShouldCleanUp:   true,
+		},
+	}
+}
+
+func getDataStoreMigratingResources(c client.Client, kamajiNamespace, migrateImage string, kamajiServiceAccount, kamajiService string) []resources.Resource {
+	return []resources.Resource{
+		&ds.Migrate{
+			Client:               c,
+			MigrateImage:         migrateImage,
+			KamajiNamespace:      kamajiNamespace,
+			KamajiServiceAccount: kamajiServiceAccount,
+			KamajiServiceName:    kamajiService,
 		},
 	}
 }
@@ -146,6 +182,12 @@ func getKubeconfigResources(c client.Client, tcpReconcilerConfig TenantControlPl
 			KubeConfigFileName: resources.AdminKubeConfigFileName,
 			TmpDirectory:       getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
 		},
+		&resources.KubeconfigResource{
+			Name:               "admin-kubeconfig",
+			Client:             c,
+			KubeConfigFileName: resources.SuperAdminKubeConfigFileName,
+			TmpDirectory:       getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
+		},
 		&resources.KubeconfigResource{
 			Name:               "controller-manager-kubeconfig",
 			Client:             c,
@@ -198,52 +240,15 @@ func getKubernetesIngressResources(c client.Client) []resources.Resource {
 	}
 }
 
-func getKubeadmPhaseResources(c client.Client) []resources.Resource {
-	return []resources.Resource{
-		&resources.KubeadmPhase{
-			Name:   "upload-config-kubeadm",
-			Client: c,
-			Phase:  resources.PhaseUploadConfigKubeadm,
-		},
-		&resources.KubeadmPhase{
-			Name:   "upload-config-kubelet",
-			Client: c,
-			Phase:  resources.PhaseUploadConfigKubelet,
-		},
-		&resources.KubeadmPhase{
-			Name:   "bootstrap-token",
-			Client: c,
-			Phase:  resources.PhaseBootstrapToken,
-		},
-	}
-}
-
-func getKubeadmAddonResources(c client.Client) []resources.Resource {
-	return []resources.Resource{
-		&resources.KubeadmAddonResource{
-			Name:         "coredns",
-			Client:       c,
-			KubeadmAddon: resources.AddonCoreDNS,
-		},
-		&resources.KubeadmAddonResource{
-			Name:         "kubeproxy",
-			Client:       c,
-			KubeadmAddon: resources.AddonKubeProxy,
-		},
-	}
-}
-
-func getExternalKonnectivityResources(c client.Client) []resources.Resource {
+func GetExternalKonnectivityResources(c client.Client) []resources.Resource {
 	return []resources.Resource{
+		&konnectivity.Agent{Client: c},
 		&konnectivity.ServiceAccountResource{Client: c},
 		&konnectivity.ClusterRoleBindingResource{Client: c},
-		&konnectivity.KubernetesDeploymentResource{Client: c},
-		&konnectivity.ServiceResource{Client: c},
-		&konnectivity.Agent{Client: c},
 	}
 }
 
-func getInternalKonnectivityResources(c client.Client) []resources.Resource {
+func getKonnectivityServerRequirementsResources(c client.Client) []resources.Resource {
 	return []resources.Resource{
 		&konnectivity.EgressSelectorConfigurationResource{Client: c},
 		&konnectivity.CertificateResource{Client: c},
@@ -251,6 +256,13 @@ func getInternalKonnectivityResources(c client.Client) []resources.Resource {
 	}
 }
 
+func getKonnectivityServerPatchResources(c client.Client) []resources.Resource {
+	return []resources.Resource{
+		&konnectivity.KubernetesDeploymentResource{Builder: builder.Konnectivity{Scheme: *c.Scheme()}, Client: c},
+		&konnectivity.ServiceResource{Client: c},
+	}
+}
+
 func getNamespacedName(namespace string, name string) k8stypes.NamespacedName {
 	return k8stypes.NamespacedName{Namespace: namespace, Name: name}
 }

controllers/soot/controllers/coredns.go

@@ -0,0 +1,89 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package controllers
+
+import (
+	"context"
+
+	"github.com/go-logr/logr"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
+	controllerruntime "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/builder"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+	"sigs.k8s.io/controller-runtime/pkg/event"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
+	"sigs.k8s.io/controller-runtime/pkg/manager"
+	"sigs.k8s.io/controller-runtime/pkg/predicate"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+	"sigs.k8s.io/controller-runtime/pkg/source"
+
+	"github.com/clastix/kamaji/controllers/utils"
+	"github.com/clastix/kamaji/internal/kubeadm"
+	"github.com/clastix/kamaji/internal/resources"
+	"github.com/clastix/kamaji/internal/resources/addons"
+)
+
+type CoreDNS struct {
+	logger logr.Logger
+
+	AdminClient               client.Client
+	GetTenantControlPlaneFunc utils.TenantControlPlaneRetrievalFn
+	TriggerChannel            chan event.GenericEvent
+}
+
+func (c *CoreDNS) Reconcile(ctx context.Context, _ reconcile.Request) (reconcile.Result, error) {
+	tcp, err := c.GetTenantControlPlaneFunc()
+	if err != nil {
+		c.logger.Error(err, "cannot retrieve TenantControlPlane")
+
+		return reconcile.Result{}, err
+	}
+
+	c.logger.Info("start processing")
+
+	resource := &addons.CoreDNS{Client: c.AdminClient}
+
+	result, handlingErr := resources.Handle(ctx, resource, tcp)
+	if handlingErr != nil {
+		c.logger.Error(handlingErr, "resource process failed", "resource", resource.GetName())
+
+		return reconcile.Result{}, handlingErr
+	}
+
+	if result == controllerutil.OperationResultNone {
+		c.logger.Info("reconciliation completed")
+
+		return reconcile.Result{}, nil
+	}
+
+	if err = utils.UpdateStatus(ctx, c.AdminClient, tcp, resource); err != nil {
+		c.logger.Error(err, "update status failed", "resource", resource.GetName())
+
+		return reconcile.Result{}, err
+	}
+
+	c.logger.Info("reconciliation processed")
+
+	return reconcile.Result{}, nil
+}
+
+func (c *CoreDNS) SetupWithManager(mgr manager.Manager) error {
+	c.logger = mgr.GetLogger().WithName("coredns")
+	c.TriggerChannel = make(chan event.GenericEvent)
+
+	return controllerruntime.NewControllerManagedBy(mgr).
+		For(&rbacv1.ClusterRoleBinding{}, builder.WithPredicates(predicate.NewPredicateFuncs(func(object client.Object) bool {
+			return object.GetName() == kubeadm.CoreDNSClusterRoleBindingName
+		}))).
+		WatchesRawSource(&source.Channel{Source: c.TriggerChannel}, &handler.EnqueueRequestForObject{}).
+		Owns(&rbacv1.ClusterRole{}).
+		Owns(&corev1.ServiceAccount{}).
+		Owns(&corev1.Service{}).
+		Owns(&corev1.ConfigMap{}).
+		Owns(&appsv1.Deployment{}).
+		Complete(c)
+}

controllers/soot/controllers/konnectivity.go

@@ -0,0 +1,112 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package controllers
+
+import (
+	"context"
+
+	"github.com/go-logr/logr"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	v1 "k8s.io/api/rbac/v1"
+	"k8s.io/apimachinery/pkg/types"
+	controllerruntime "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/builder"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+	"sigs.k8s.io/controller-runtime/pkg/event"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
+	"sigs.k8s.io/controller-runtime/pkg/manager"
+	"sigs.k8s.io/controller-runtime/pkg/predicate"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+	"sigs.k8s.io/controller-runtime/pkg/source"
+
+	"github.com/clastix/kamaji/controllers"
+	"github.com/clastix/kamaji/controllers/utils"
+	"github.com/clastix/kamaji/internal/resources"
+	"github.com/clastix/kamaji/internal/resources/konnectivity"
+)
+
+type KonnectivityAgent struct {
+	logger logr.Logger
+
+	AdminClient               client.Client
+	GetTenantControlPlaneFunc utils.TenantControlPlaneRetrievalFn
+	TriggerChannel            chan event.GenericEvent
+}
+
+func (k *KonnectivityAgent) Reconcile(ctx context.Context, _ reconcile.Request) (reconcile.Result, error) {
+	tcp, err := k.GetTenantControlPlaneFunc()
+	if err != nil {
+		k.logger.Error(err, "cannot retrieve TenantControlPlane")
+
+		return reconcile.Result{}, err
+	}
+
+	for _, resource := range controllers.GetExternalKonnectivityResources(k.AdminClient) {
+		k.logger.Info("start processing", "resource", resource.GetName())
+
+		result, handlingErr := resources.Handle(ctx, resource, tcp)
+		if handlingErr != nil {
+			k.logger.Error(handlingErr, "resource process failed", "resource", resource.GetName())
+
+			return reconcile.Result{}, handlingErr
+		}
+
+		if result == controllerutil.OperationResultNone {
+			k.logger.Info("resource processed", "resource", resource.GetName())
+
+			continue
+		}
+
+		if err = utils.UpdateStatus(ctx, k.AdminClient, tcp, resource); err != nil {
+			k.logger.Error(err, "update status failed", "resource", resource.GetName())
+
+			return reconcile.Result{}, err
+		}
+	}
+
+	k.logger.Info("reconciliation completed")
+
+	return reconcile.Result{}, nil
+}
+
+func (k *KonnectivityAgent) SetupWithManager(mgr manager.Manager) error {
+	k.logger = mgr.GetLogger().WithName("konnectivity_agent")
+	k.TriggerChannel = make(chan event.GenericEvent)
+
+	return controllerruntime.NewControllerManagedBy(mgr).
+		For(&appsv1.DaemonSet{}, builder.WithPredicates(predicate.NewPredicateFuncs(func(object client.Object) bool {
+			return object.GetName() == konnectivity.AgentName && object.GetNamespace() == konnectivity.AgentNamespace
+		}))).
+		WatchesRawSource(source.Kind(mgr.GetCache(), &corev1.ServiceAccount{}), handler.EnqueueRequestsFromMapFunc(func(_ context.Context, object client.Object) []reconcile.Request {
+			if object.GetName() == konnectivity.AgentName && object.GetNamespace() == konnectivity.AgentNamespace {
+				return []reconcile.Request{
+					{
+						NamespacedName: types.NamespacedName{
+							Namespace: object.GetNamespace(),
+							Name:      object.GetName(),
+						},
+					},
+				}
+			}
+
+			return nil
+		})).
+		WatchesRawSource(source.Kind(mgr.GetCache(), &v1.ClusterRoleBinding{}), handler.EnqueueRequestsFromMapFunc(func(_ context.Context, object client.Object) []reconcile.Request {
+			if object.GetName() == konnectivity.CertCommonName {
+				return []reconcile.Request{
+					{
+						NamespacedName: types.NamespacedName{
+							Name: konnectivity.CertCommonName,
+						},
+					},
+				}
+			}
+
+			return nil
+		})).
+		WatchesRawSource(&source.Channel{Source: k.TriggerChannel}, &handler.EnqueueRequestForObject{}).
+		Complete(k)
+}

controllers/soot/controllers/kubeadm_phase.go

@@ -0,0 +1,72 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package controllers
+
+import (
+	"context"
+
+	"github.com/go-logr/logr"
+	controllerruntime "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/builder"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+	"sigs.k8s.io/controller-runtime/pkg/event"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
+	"sigs.k8s.io/controller-runtime/pkg/manager"
+	"sigs.k8s.io/controller-runtime/pkg/predicate"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+	"sigs.k8s.io/controller-runtime/pkg/source"
+
+	"github.com/clastix/kamaji/controllers/utils"
+	"github.com/clastix/kamaji/internal/resources"
+)
+
+type KubeadmPhase struct {
+	GetTenantControlPlaneFunc utils.TenantControlPlaneRetrievalFn
+	TriggerChannel            chan event.GenericEvent
+	Phase                     resources.KubeadmPhaseResource
+
+	logger logr.Logger
+}
+
+func (k *KubeadmPhase) Reconcile(ctx context.Context, _ reconcile.Request) (reconcile.Result, error) {
+	tcp, err := k.GetTenantControlPlaneFunc()
+	if err != nil {
+		return reconcile.Result{}, err
+	}
+
+	k.logger.Info("start processing")
+
+	result, handlingErr := resources.Handle(ctx, k.Phase, tcp)
+	if handlingErr != nil {
+		k.logger.Error(handlingErr, "resource process failed")
+
+		return reconcile.Result{}, handlingErr
+	}
+
+	if result == controllerutil.OperationResultNone {
+		k.logger.Info("reconciliation completed")
+
+		return reconcile.Result{}, nil
+	}
+
+	if err = utils.UpdateStatus(ctx, k.Phase.GetClient(), tcp, k.Phase); err != nil {
+		k.logger.Error(err, "update status failed")
+
+		return reconcile.Result{}, err
+	}
+
+	k.logger.Info("reconciliation processed")
+
+	return reconcile.Result{}, nil
+}
+
+func (k *KubeadmPhase) SetupWithManager(mgr manager.Manager) error {
+	k.logger = mgr.GetLogger().WithName(k.Phase.GetName())
+	k.TriggerChannel = make(chan event.GenericEvent)
+
+	return controllerruntime.NewControllerManagedBy(mgr).
+		For(k.Phase.GetWatchedObject(), builder.WithPredicates(predicate.NewPredicateFuncs(k.Phase.GetPredicateFunc()))).
+		WatchesRawSource(&source.Channel{Source: k.TriggerChannel}, &handler.EnqueueRequestForObject{}).
+		Complete(k)
+}

controllers/soot/controllers/kubeproxy.go

@@ -0,0 +1,89 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package controllers
+
+import (
+	"context"
+
+	"github.com/go-logr/logr"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
+	controllerruntime "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/builder"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+	"sigs.k8s.io/controller-runtime/pkg/event"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
+	"sigs.k8s.io/controller-runtime/pkg/manager"
+	"sigs.k8s.io/controller-runtime/pkg/predicate"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+	"sigs.k8s.io/controller-runtime/pkg/source"
+
+	"github.com/clastix/kamaji/controllers/utils"
+	"github.com/clastix/kamaji/internal/kubeadm"
+	"github.com/clastix/kamaji/internal/resources"
+	"github.com/clastix/kamaji/internal/resources/addons"
+)
+
+type KubeProxy struct {
+	AdminClient               client.Client
+	GetTenantControlPlaneFunc utils.TenantControlPlaneRetrievalFn
+	TriggerChannel            chan event.GenericEvent
+
+	logger logr.Logger
+}
+
+func (k *KubeProxy) Reconcile(ctx context.Context, _ reconcile.Request) (reconcile.Result, error) {
+	tcp, err := k.GetTenantControlPlaneFunc()
+	if err != nil {
+		k.logger.Error(err, "cannot retrieve TenantControlPlane")
+
+		return reconcile.Result{}, err
+	}
+
+	k.logger.Info("start processing")
+
+	resource := &addons.KubeProxy{Client: k.AdminClient}
+
+	result, handlingErr := resources.Handle(ctx, resource, tcp)
+	if handlingErr != nil {
+		k.logger.Error(handlingErr, "resource process failed", "resource", resource.GetName())
+
+		return reconcile.Result{}, handlingErr
+	}
+
+	if result == controllerutil.OperationResultNone {
+		k.logger.Info("reconciliation completed")
+
+		return reconcile.Result{}, nil
+	}
+
+	if err = utils.UpdateStatus(ctx, k.AdminClient, tcp, resource); err != nil {
+		k.logger.Error(err, "update status failed")
+
+		return reconcile.Result{}, err
+	}
+
+	k.logger.Info("reconciliation processed")
+
+	return reconcile.Result{}, nil
+}
+
+func (k *KubeProxy) SetupWithManager(mgr manager.Manager) error {
+	k.logger = mgr.GetLogger().WithName("kube_proxy")
+	k.TriggerChannel = make(chan event.GenericEvent)
+
+	return controllerruntime.NewControllerManagedBy(mgr).
+		For(&rbacv1.ClusterRoleBinding{}, builder.WithPredicates(predicate.NewPredicateFuncs(func(object client.Object) bool {
+			return object.GetName() == kubeadm.KubeProxyClusterRoleBindingName
+		}))).
+		WatchesRawSource(&source.Channel{Source: k.TriggerChannel}, &handler.EnqueueRequestForObject{}).
+		Owns(&corev1.ServiceAccount{}).
+		Owns(&rbacv1.Role{}).
+		Owns(&rbacv1.RoleBinding{}).
+		Owns(&corev1.ConfigMap{}).
+		Owns(&appsv1.DaemonSet{}).
+		Complete(k)
+}

controllers/soot/controllers/migrate.go

@@ -0,0 +1,200 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package controllers
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/go-logr/logr"
+	admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	pointer "k8s.io/utils/ptr"
+	controllerruntime "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/builder"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/event"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
+	"sigs.k8s.io/controller-runtime/pkg/manager"
+	"sigs.k8s.io/controller-runtime/pkg/predicate"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+	"sigs.k8s.io/controller-runtime/pkg/source"
+
+	"github.com/clastix/kamaji/api/v1alpha1"
+	"github.com/clastix/kamaji/controllers/utils"
+	"github.com/clastix/kamaji/internal/utilities"
+)
+
+type Migrate struct {
+	client client.Client
+	logger logr.Logger
+
+	GetTenantControlPlaneFunc utils.TenantControlPlaneRetrievalFn
+	WebhookNamespace          string
+	WebhookServiceName        string
+	WebhookCABundle           []byte
+	TriggerChannel            chan event.GenericEvent
+}
+
+func (m *Migrate) Reconcile(ctx context.Context, _ reconcile.Request) (reconcile.Result, error) {
+	tcp, err := m.GetTenantControlPlaneFunc()
+	if err != nil {
+		return reconcile.Result{}, err
+	}
+	// Cannot detect the status of the TenantControlPlane, enqueuing back
+	if tcp.Status.Kubernetes.Version.Status == nil {
+		return reconcile.Result{Requeue: true}, nil
+	}
+
+	switch *tcp.Status.Kubernetes.Version.Status {
+	case v1alpha1.VersionMigrating:
+		err = m.createOrUpdate(ctx)
+	case v1alpha1.VersionReady:
+		err = m.cleanup(ctx)
+	}
+
+	if err != nil {
+		m.logger.Error(err, "reconciliation failed")
+
+		return reconcile.Result{}, err
+	}
+
+	return reconcile.Result{}, nil
+}
+
+func (m *Migrate) cleanup(ctx context.Context) error {
+	if err := m.client.Delete(ctx, m.object()); err != nil {
+		if errors.IsNotFound(err) {
+			return nil
+		}
+
+		return fmt.Errorf("unable to clean-up ValidationWebhook required for migration: %w", err)
+	}
+
+	return nil
+}
+
+func (m *Migrate) createOrUpdate(ctx context.Context) error {
+	obj := m.object()
+
+	_, err := utilities.CreateOrUpdateWithConflict(ctx, m.client, obj, func() error {
+		obj.Webhooks = []admissionregistrationv1.ValidatingWebhook{
+			{
+				Name: "leases.migrate.kamaji.clastix.io",
+				ClientConfig: admissionregistrationv1.WebhookClientConfig{
+					URL:      pointer.To(fmt.Sprintf("https://%s.%s.svc:443/migrate", m.WebhookServiceName, m.WebhookNamespace)),
+					CABundle: m.WebhookCABundle,
+				},
+				Rules: []admissionregistrationv1.RuleWithOperations{
+					{
+						Operations: []admissionregistrationv1.OperationType{
+							admissionregistrationv1.Create,
+							admissionregistrationv1.Delete,
+						},
+						Rule: admissionregistrationv1.Rule{
+							APIGroups:   []string{"*"},
+							APIVersions: []string{"*"},
+							Resources:   []string{"*"},
+							Scope: func(v admissionregistrationv1.ScopeType) *admissionregistrationv1.ScopeType {
+								return &v
+							}(admissionregistrationv1.NamespacedScope),
+						},
+					},
+				},
+				FailurePolicy: func(v admissionregistrationv1.FailurePolicyType) *admissionregistrationv1.FailurePolicyType {
+					return &v
+				}(admissionregistrationv1.Fail),
+				MatchPolicy: func(v admissionregistrationv1.MatchPolicyType) *admissionregistrationv1.MatchPolicyType {
+					return &v
+				}(admissionregistrationv1.Equivalent),
+				NamespaceSelector: &metav1.LabelSelector{
+					MatchExpressions: []metav1.LabelSelectorRequirement{
+						{
+							Key:      "kubernetes.io/metadata.name",
+							Operator: metav1.LabelSelectorOpIn,
+							Values: []string{
+								"kube-node-lease",
+							},
+						},
+					},
+				},
+				SideEffects: func(v admissionregistrationv1.SideEffectClass) *admissionregistrationv1.SideEffectClass {
+					return &v
+				}(admissionregistrationv1.SideEffectClassNoneOnDryRun),
+				AdmissionReviewVersions: []string{"v1"},
+			},
+			{
+				Name: "catchall.migrate.kamaji.clastix.io",
+				ClientConfig: admissionregistrationv1.WebhookClientConfig{
+					URL:      pointer.To(fmt.Sprintf("https://%s.%s.svc:443/migrate", m.WebhookServiceName, m.WebhookNamespace)),
+					CABundle: m.WebhookCABundle,
+				},
+				Rules: []admissionregistrationv1.RuleWithOperations{
+					{
+						Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.OperationAll},
+						Rule: admissionregistrationv1.Rule{
+							APIGroups:   []string{"*"},
+							APIVersions: []string{"*"},
+							Resources:   []string{"*"},
+							Scope: func(v admissionregistrationv1.ScopeType) *admissionregistrationv1.ScopeType {
+								return &v
+							}(admissionregistrationv1.AllScopes),
+						},
+					},
+				},
+				FailurePolicy: func(v admissionregistrationv1.FailurePolicyType) *admissionregistrationv1.FailurePolicyType {
+					return &v
+				}(admissionregistrationv1.Fail),
+				MatchPolicy: func(v admissionregistrationv1.MatchPolicyType) *admissionregistrationv1.MatchPolicyType {
+					return &v
+				}(admissionregistrationv1.Equivalent),
+				NamespaceSelector: &metav1.LabelSelector{
+					MatchExpressions: []metav1.LabelSelectorRequirement{
+						{
+							Key:      "kubernetes.io/metadata.name",
+							Operator: metav1.LabelSelectorOpNotIn,
+							Values: []string{
+								"kube-system",
+								"kube-node-lease",
+							},
+						},
+					},
+				},
+				SideEffects: func(v admissionregistrationv1.SideEffectClass) *admissionregistrationv1.SideEffectClass {
+					return &v
+				}(admissionregistrationv1.SideEffectClassNoneOnDryRun),
+				TimeoutSeconds:          nil,
+				AdmissionReviewVersions: []string{"v1"},
+			},
+		}
+
+		return nil
+	})
+
+	return err
+}
+
+func (m *Migrate) SetupWithManager(mgr manager.Manager) error {
+	m.client = mgr.GetClient()
+	m.logger = mgr.GetLogger().WithName("migrate")
+	m.TriggerChannel = make(chan event.GenericEvent)
+
+	return controllerruntime.NewControllerManagedBy(mgr).
+		For(&admissionregistrationv1.ValidatingWebhookConfiguration{}, builder.WithPredicates(predicate.NewPredicateFuncs(func(object client.Object) bool {
+			vwc := m.object()
+
+			return object.GetName() == vwc.GetName()
+		}))).
+		WatchesRawSource(&source.Channel{Source: m.TriggerChannel}, &handler.EnqueueRequestForObject{}).
+		Complete(m)
+}
+
+func (m *Migrate) object() *admissionregistrationv1.ValidatingWebhookConfiguration {
+	return &admissionregistrationv1.ValidatingWebhookConfiguration{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "kamaji-freeze",
+		},
+	}
+}