chore(helm): bumping to v0.1.0

The TCP Deployment container kube-apiserver is deeply hacked with extra
details for konnectivity: most of them weren't cleaned-up properly, and
the function wasn't entirely idempotent in toggling the feature.

This fix is addressing this situation, and rearranging the code
according to the latest polish.

.github/workflows/ci.yaml

@@ -12,13 +12,14 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - uses: actions/setup-go@v2
+      - uses: actions/setup-go@v3
         with:
           go-version: '1.18'
+          check-latest: true
       - name: Run golangci-lint
-        uses: golangci/golangci-lint-action@v2.3.0
+        uses: golangci/golangci-lint-action@v3.2.0
         with:
-          version: v1.45.2
+          version: v1.49.0
           only-new-issues: false
           args: --timeout 5m --config .golangci.yml
   diff:
@@ -28,9 +29,10 @@ jobs:
       - uses: actions/checkout@v2
         with:
           fetch-depth: 0
-      - uses: actions/setup-go@v2
+      - uses: actions/setup-go@v3
         with:
           go-version: '1.18'
+          check-latest: true
       - run: make yaml-installation-file
       - name: Checking if YAML installer file is not aligned
         run: if [[ $(git diff | wc -l) -gt 0 ]]; then echo ">>> Untracked generated files have not been committed" && git --no-pager diff && exit 1; fi

.github/workflows/e2e.yaml

@@ -34,9 +34,10 @@ jobs:
       - uses: actions/checkout@v2
         with:
           fetch-depth: 0
-      - uses: actions/setup-go@v2
+      - uses: actions/setup-go@v3
         with:
           go-version: '1.18'
+          check-latest: true
       - run: |
           sudo apt-get update
           sudo apt-get install -y golang-cfssl

.github/workflows/helm.yaml

@@ -3,11 +3,21 @@ name: Helm Chart
 on:
   push:
     branches: [ "*" ]
-    tags: [ "helm-v" ]
+    tags: [ "helm-v*" ]
   pull_request:
     branches: [ "*" ]
 
 jobs:
+  diff:
+    name: diff
+    runs-on: ubuntu-18.04
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - run: make -C charts/kamaji docs
+      - name: Checking if Helm docs is not aligned
+        run: if [[ $(git diff | wc -l) -gt 0 ]]; then echo ">>> Untracked changes have not been committed" && git --no-pager diff && exit 1; fi
   lint:
     runs-on: ubuntu-latest
     steps:
@@ -16,7 +26,7 @@ jobs:
         with:
           version: 3.3.4
       - name: Linting Chart
-        run: helm lint ./helm/kamaji
+        run: helm lint ./charts/kamaji
   release:
     if: startsWith(github.ref, 'refs/tags/helm-v')
     runs-on: ubuntu-latest
@@ -26,7 +36,7 @@ jobs:
         uses: stefanprodan/helm-gh-pages@master
         with:
           token: ${{ secrets.BOT_GITHUB_TOKEN }}
-          charts_dir: helm
+          charts_dir: charts
           charts_url: https://clastix.github.io/charts
           owner: clastix
           repository: charts

.golangci.yml

@@ -14,9 +14,6 @@ linters:
     - wrapcheck
     - gomnd
     - scopelint
-    - golint
-    - interfacer
-    - maligned
     - varnamelen
     - testpackage
     - tagliatelle
@@ -27,6 +24,10 @@ linters:
     - exhaustivestruct
     - wsl
     - exhaustive
+    - nosprintfhostport
+    - nonamedreturns
+    - interfacebloat
+    - exhaustruct
     - lll
     - gosec
     - gomoddirectives
@@ -34,6 +35,14 @@ linters:
     - gochecknoinits
     - funlen
     - dupl
-    - maintidx
     - cyclop
+    # deprecated linters
+    - deadcode
+    - golint
+    - interfacer
+    - structcheck
+    - varcheck
+    - nosnakecase
+    - ifshort
+    - maligned
   enable-all: true

Dockerfile

@@ -22,6 +22,7 @@ COPY main.go main.go
 COPY api/ api/
 COPY controllers/ controllers/
 COPY internal/ internal/
+COPY indexers/ indexers/
 
 # Build
 RUN CGO_ENABLED=0 GOOS=linux GOARCH=$TARGETARCH go build \

Makefile

@@ -3,7 +3,7 @@
 # To re-generate a bundle for another specific version without changing the standard setup, you can:
 # - use the VERSION as arg of the bundle target (e.g make bundle VERSION=0.0.2)
 # - use environment variables to overwrite this value (e.g export VERSION=0.0.2)
-VERSION ?= 0.0.1
+VERSION ?= 0.1.0
 
 # CHANNELS define the bundle channels used in the bundle.
 # Add a new line here if you would like to change its default config. (E.g CHANNELS = "candidate,fast,stable")
@@ -36,9 +36,7 @@ IMAGE_TAG_BASE ?= clastix.io/operator
 BUNDLE_IMG ?= $(IMAGE_TAG_BASE)-bundle:v$(VERSION)
 
 # Image URL to use all building/pushing image targets
-IMG ?= clastix/kamaji:latest
-# Produce CRDs that work back to Kubernetes 1.11 (no version conversion)
-CRD_OPTIONS ?= "crd:trivialVersions=true,preserveUnknownFields=false"
+IMG ?= clastix/kamaji:v$(VERSION)
 
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
 ifeq (,$(shell go env GOBIN))
@@ -87,17 +85,22 @@ kind: ## Download kind locally if necessary.
 
 CONTROLLER_GEN = $(shell pwd)/bin/controller-gen
 controller-gen: ## Download controller-gen locally if necessary.
-	$(call go-install-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.6.1)
+	$(call go-install-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.9.2)
 
 KUSTOMIZE = $(shell pwd)/bin/kustomize
 kustomize: ## Download kustomize locally if necessary.
 	$(call install-kustomize,$(KUSTOMIZE),3.8.7)
 
+APIDOCS_GEN = $(shell pwd)/bin/crdoc
+apidocs-gen: ## Download crdoc locally if necessary.
+	$(call go-install-tool,$(APIDOCS_GEN),fybrik.io/crdoc@latest)
+
 ##@ Development
 
 manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects.
-	$(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=manager-role webhook paths="./..." output:crd:artifacts:config=config/crd/bases
-	cp config/crd/bases/kamaji.clastix.io_tenantcontrolplanes.yaml helm/kamaji/crds/tenantcontrolplane.yaml
+	$(CONTROLLER_GEN) rbac:roleName=manager-role crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases
+	cp config/crd/bases/kamaji.clastix.io_tenantcontrolplanes.yaml charts/kamaji/crds/tenantcontrolplane.yaml
+	cp config/crd/bases/kamaji.clastix.io_datastores.yaml charts/kamaji/crds/datastore.yaml
 
 generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations.
 	$(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..."
@@ -244,5 +247,10 @@ env:
 
 .PHONY: e2e
 e2e: env load helm ginkgo ## Create a KinD cluster, install Kamaji on it and run the test suite.
-	$(HELM) upgrade --debug --install kamaji ./helm/kamaji --create-namespace --namespace kamaji-system --set "image.pullPolicy=Never"
+	$(HELM) upgrade --debug --install kamaji ./charts/kamaji --create-namespace --namespace kamaji-system --set "image.pullPolicy=Never"
 	$(GINKGO) -v ./e2e
+
+##@ Document
+
+apidoc: apidocs-gen
+	$(APIDOCS_GEN) crdoc --resources config/crd/bases --output docs/apireference.md --template docs/templates/reference-cr.tmpl

PROJECT

@@ -16,4 +16,12 @@ resources:
   kind: TenantControlPlane
   path: github.com/clastix/kamaji/api/v1alpha1
   version: v1alpha1
+- api:
+    crdVersion: v1
+    namespaced: false
+  domain: clastix.io
+  group: kamaji
+  kind: DataStore
+  path: github.com/clastix/kamaji/api/v1alpha1
+  version: v1alpha1
 version: "3"

README.md

@@ -8,25 +8,26 @@
   </a>
 </p>
 
-**Kamaji** is a tool aimed to build and operate a **Managed Kubernetes Service** with a fraction of the operational burden. With **Kamaji**, you can deploy and operate hundreds of Kubernetes clusters as a hyper-scale cloud provider.
+**Kamaji** deploys and operates **Kubernetes** at scale with a fraction of the operational burden.
 
 <p align="center" style="padding: 6px 6px">
   <img src="assets/kamaji-logo.png" />
 </p>
 
-> This project is still in the early development stage which means it's not ready for production as APIs, commands, flags, etc. are subject to change, but also that your feedback can still help to shape it. Please try it out and let us know what you like, dislike, what works, what doesn't, etc.
-
 ## Why we are building it?
-Global hyper-scalers are leading the Managed Kubernetes space, while regional cloud providers, as well as large corporations, are struggling to offer the same level of experience to their developers because of the lack of the right tools. Also, Kubernetes solutions for the on-premises are designed with an enterprise-first approach and they are too costly when deployed at scale. Project Kamaji aims to solve this pain by leveraging multi-tenancy and simplifying how to run Kubernetes clusters at scale with a fraction of the operational burden.
+Global hyper-scalers are leading the Managed Kubernetes space, while other cloud providers, as well as large corporations, are struggling to offer the same experience to their DevOps teams because of the lack of the right tools. Also, current Kubernetes solutions are mainly designed with an enterprise-first approach and they are too costly when deployed at scale.
+
+**Kamaji** aims to solve these pains by leveraging multi-tenancy and simplifying how to run multiple control planes on the same infrastructure with a fraction of the operational burden.
 
 ## How it works
-Kamaji turns any CNCF conformant Kubernetes cluster into an _“admin cluster”_ to orchestrate other Kubernetes clusters we're calling _“tenant clusters”_. As with every Kubernetes cluster, a tenant cluster has a set of nodes and a control plane, composed of several components: `APIs server`, `scheduler`, `controller manager`. What Kamaji does is deploy those components as a pod running in the admin cluster.
-
-
-And what about the tenant worker nodes? They are just worker nodes: regular instances, e.g. virtual or bare metal, connecting to the APIs server of the tenant cluster. Kamaji's goal is to manage the lifecycle of hundreds of these clusters, not only one, so how can we add another tenant cluster? As you could expect, Kamaji just deploys a new tenant control plane as a new pod in the admin cluster, and then it joins the new tenant worker nodes.
+Kamaji turns any Kubernetes cluster into an _“admin cluster”_ to orchestrate other Kubernetes clusters called _“tenant clusters”_. What makes Kamaji special is that Control Planes of _“tenant clusters”_ are just regular pods running in the _“admin cluster”_ instead of dedicated Virtual Machines. This solution makes running control planes at scale cheaper and easier to deploy and operate. View [Core Concepts](./docs/concepts.md) for a deeper understanding of principles behind Kamaji's design.
 
 <p align="center">
-  <img src="assets/kamaji-light.png" />
+  <img src="assets/kamaji-light.png#gh-light-mode-only" />
+</p>
+
+<p align="center">
+  <img src="assets/kamaji-dark.png#gh-dark-mode-only" />
 </p>
 
 All the tenant clusters built with Kamaji are fully compliant CNCF Kubernetes clusters and are compatible with the standard Kubernetes toolchains everybody knows and loves.
@@ -35,69 +36,48 @@ All the tenant clusters built with Kamaji are fully compliant CNCF Kubernetes cl
   <img src="assets/screenshot.png" />
 </p>
 
-## Save the state
-Putting the tenant cluster control plane in a pod is the easiest part. Also, we have to make sure each tenant cluster saves the state to be able to store and retrieve data.
-
-A dedicated `etcd` cluster for each tenant cluster doesn’t scale well for a managed service because `etcd` data persistence can be cumbersome at scale, rising the operational effort to mitigate it. So we have to find an alternative keeping in mind our goal for a resilient and cost-optimized solution at the same time. As we can deploy any Kubernetes cluster with an external `etcd` cluster, we explored this option for the tenant control planes. On the admin cluster, we deploy a multi-tenant `etcd` cluster storing the state of multiple tenant clusters.
-
-With this solution, the resiliency is guaranteed by the usual `etcd` mechanism, and the pods' count remains under control, so it solves the main goal of resiliency and costs optimization. The trade-off here is that we have to operate an external `etcd` cluster and manage the access to be sure that each tenant cluster uses only its data. Also, there are limits in size in `etcd`, defaulted to 2GB and configurable to a maximum of 8GB. We’re solving this issue by pooling multiple `etcd` and sharding the tenant control planes.
-
 ## Getting started
 
 Please refer to the [Getting Started guide](./docs/getting-started-with-kamaji.md) to deploy a minimal setup of Kamaji on KinD.
 
+> This project is still in the early development stage which means it's not ready for production as APIs, commands, flags, etc. are subject to change, but also that your feedback can still help to shape it. Please try it out and let us know what you like, dislike, what works, what doesn't, etc.
+
 ## Use cases
-Kamaji project has been initially started as a solution for actual and common problems such as minimizing the Total Cost of Ownership while running Kubernetes at large scale. However, it can open a wider range of use cases. Here are a few:
+Kamaji project has been initially started as a solution for actual and common problems such as minimizing the Total Cost of Ownership while running Kubernetes at large scale. However, it can open a wider range of use cases.
 
-### Managed Kubernetes
-Enabling companies to provide Cloud Native Infrastructure with ease by introducing a strong separation of concerns between management and workloads. Centralize clusters management, monitoring, and observability by leaving developers to focus on the applications, increase productivity and reduce operational costs. 
+Here are a few:
 
-### Kubernetes as a Service
-Provide Kubernetes clusters in a self-service fashion by running management and workloads on different infrastructures and cost centers with the option of Bring Your Own Device - BYOD.
-
-### Control Plane as a Service
-Provide multiple Kubernetes control planes running on top of a single Kubernetes cluster. Tenants who use namespaces based isolation often still need access to cluster wide resources like Cluster Roles, Admission Webhooks, or Custom Resource Definitions.
-
-### Edge Computing
-Distribute Kubernetes workloads across edge computing locations without having to manage multiple clusters across various providers. Centralize management of hundreds of control planes while leaving workloads to run isolated on their own dedicated infrastructure.
-
-### Cluster Simulation
-Check new Kubernetes API or experimental flag or a new tool without impacting production operations. Kamaji will let you simulate such things in a safe and controlled environment.
-
-### Workloads Testing
-Check the behaviour of your workloads on different and multiple versions of Kubernetes with ease by deploying multiple Control Planes in a single cluster.
+- **Managed Kubernetes:** enable companies to provide Cloud Native Infrastructure with ease by introducing a strong separation of concerns between management and workloads. Centralize clusters management, monitoring, and observability by leaving developers to focus on applications, increase productivity and reduce operational costs.
+- **Kubernetes as a Service:** provide Kubernetes clusters in a self-service fashion by running management and workloads on different infrastructures with the option of Bring Your Own Device, BYOD.
+- **Control Plane as a Service:** provide multiple Kubernetes control planes running on top of a single Kubernetes cluster. Tenants who use namespaces based isolation often still need access to cluster wide resources like Cluster Roles, Admission Webhooks, or Custom Resource Definitions.
+- **Edge Computing:** distribute Kubernetes workloads across edge computing locations without having to manage multiple clusters across various providers. Centralize management of hundreds of control planes while leaving workloads to run isolated on their own dedicated infrastructure.
+- **Cluster Simulation:** check new Kubernetes API or experimental flag or a new tool without impacting production operations. Kamaji will let you simulate such things in a safe and controlled environment.
+- **Workloads Testing:** check the behaviour of your workloads on different and multiple versions of Kubernetes with ease by deploying multiple Control Planes in a single cluster.
 
 ## Features
 
-### Self Service Kubernetes
-Leave users the freedom to self-provision their Kubernetes clusters according to the assigned boundaries.
-
-### Multi-cluster Management
-Centrally manage multiple tenant clusters from a single admin cluster. Happy SREs. 
-
-### Cheaper Control Planes
-Place multiple tenant control planes on a single node, instead of having three nodes for a single control plane.
-
-### Stronger Multi-Tenancy
-Leave tenants to access the control plane with admin permissions while keeping the tenant isolated at the infrastructure level.
-
-### Kubernetes Inception
-Use Kubernetes to manage Kubernetes by re-using all the Kubernetes goodies you already know and love.
-
-### Full APIs compliant
-Tenant clusters are fully CNCF compliant built with upstream Kubernetes binaries. A client does not see differences between a Kamaji provisioned cluster and a dedicated cluster.
+- **Self Service Kubernetes:** leave users the freedom to self-provision their Kubernetes clusters according to the assigned boundaries.
+- **Multi-cluster Management:** centrally manage multiple tenant clusters from a single admin cluster. Happy SREs. 
+- **Cheaper Control Planes:** place multiple tenant control planes on a single node, instead of having three nodes for a single control plane.
+- **Stronger Multi-Tenancy:** leave tenants to access the control plane with admin permissions while keeping the tenant isolated at the infrastructure level.
+- **Kubernetes Inception:** use Kubernetes to manage Kubernetes by re-using all the Kubernetes goodies you already know and love.
+- **Full APIs compliant:** tenant clusters are fully CNCF compliant built with upstream Kubernetes binaries. A user does not see differences between a Kamaji provisioned cluster and a dedicated cluster.
 
 ## Roadmap
 
 - [ ] Benchmarking and stress-test
 - [x] Support for dynamic address allocation on native Load Balancer
 - [x] Zero Downtime Tenant Control Plane upgrade
-- [ ] `konnectivity` integration
+- [x] `konnectivity` integration
 - [ ] Provisioning of Tenant Control Plane through Cluster APIs
-- [ ] Prometheus metrics for monitoring and alerting
-- [ ] `kine` integration, i.e. use MySQL, SQLite, PostgreSQL as datastore
-- [ ] Deeper `kubeadm` integration
-- [ ] `etcd` pooling
+- [ ] Terraform provider
+- [ ] Custom Prometheus metrics for monitoring and alerting
+- [x] `kine` integration for MySQL as datastore
+- [x] `kine` integration for PostgreSQL as datastore
+- [x] Pool of multiple datastores
+- [ ] Automatic assigning of Tenant Control Plane to a datastore
+- [ ] Autoscaling of Tenant Control Plane pods
+
 
 ## Documentation
 Please, check the project's [documentation](./docs/) for getting started with Kamaji.
@@ -108,27 +88,23 @@ Kamaji is Open Source with Apache 2 license and any contribution is welcome.
 ## Community
 Join the [Kubernetes Slack Workspace](https://slack.k8s.io/) and the [`#kamaji`](https://kubernetes.slack.com/archives/C03GLTTMWNN) channel to meet end-users and contributors.
 
-## FAQ
+## FAQs
 Q. What does Kamaji means?
 
 A. Kamaji is named as the character _Kamaji_ from the Japanese movie [_Spirited Away_](https://en.wikipedia.org/wiki/Spirited_Away).
 
 Q. Is Kamaji another Kubernetes distribution?
 
-A. No, Kamaji is a tool you can install on top of any CNCF conformant Kubernetes to provide hundreds of managed Tenant clusters as a service. We tested Kamaji on vanilla Kubernetes 1.23+, KinD, and MS Azure AKS. We expect it to work smoothly on other Kubernetes distributions. The tenant clusters made with Kamaji are conformant CNCF Kubernetes vanilla clusters built with `kubeadm`.
+A. No, Kamaji is a Kubernetes Operator you can install on top of any Kubernetes cluster to provide hundreds of managed Kubernetes clusters as a service. We tested Kamaji on vanilla Kubernetes 1.22+, KinD, and Azure AKS. We expect it to work smoothly on other Kubernetes distributions. The tenant clusters made with Kamaji are conformant CNCF Kubernetes clusters as we leverage on [`kubeadm`](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/).
 
-Q. Is it safe to run Kubernetes control plane components in a pod?
+Q. Is it safe to run Kubernetes control plane components in a pod instead of dedicated virtual machines?
 
-A. Yes, the tenant control plane components are packaged in the same way they are running in bare metal or virtual nodes. We leverage the `kubeadm` code to set up the control plane components as they were running on a server. The same unchanged images of upstream `APIs server`, `scheduler`, `controller manager` are used.
+A. Yes, the tenant control plane components are packaged in the same way they are running in bare metal or virtual nodes. We leverage the `kubeadm` code to set up the control plane components as they were running on their own server. The unchanged images of upstream `kube-apiserver`, `kube-scheduler`, and `kube-controller-manager` are used.
 
-Q. And what about multi-tenant `etcd`? I never heard of it.
+Q. You already provide a Kubernetes multi-tenancy solution with [Capsule](https://capsule.clastix.io). Why does Kamaji matter?
 
-A. Even if multi-tenant deployment for `etcd` is not a common practice, multi-tenancy, RBAC, and client authentication has been [supported](https://etcd.io/docs/v3.5/op-guide/authentication/) in `etcd` from a long time. 
+A. A multi-tenancy solution, like Capsule shares the Kubernetes control plane among all tenants keeping tenant namespaces isolated by policies. While the solution is the right choice by balancing between features and ease of usage, there are cases where a tenant user requires access to the control plane, for example, when a tenant requires to manage CRDs on his own. With Kamaji, you can provide cluster admin permissions to the tenant.
 
-Q. You already provide a Kubernetes multi-tenancy solution with [Capsule](capsule.clastix.io). Why does Kamaji matter?
+Q. Well you convinced me, how to get a try?
 
-A. Lighter Multi-Tenancy solutions, like Capsule shares the Kubernetes control plane among all tenants keeping tenant namespaces isolated by policies. While these solutions are the right choice by balancing between features and ease of usage, there are cases where a tenant user requires access to the control plane, for example, when a tenant requires to manage CRDs on his own. With Kamaji, you can provide admin permissions to the tenants.
-
-Q. So I need a costly cloud infrastructure to try Kamaji?
-
-A. No, it is possible to getting started Kamaji on your laptop with [KinD](./docs/getting-started-with-kamaji.md).
+A. It is possible to get started with Kamaji on a laptop with [KinD](./docs/getting-started-with-kamaji.md) installed.

api/v1alpha1/datastore_funcs.go

@@ -0,0 +1,39 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package v1alpha1
+
+import (
+	"context"
+	"fmt"
+
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+// GetContent is the resolver for the container of the Secret.
+// The bare content has priority over the external reference.
+func (in *ContentRef) GetContent(ctx context.Context, client client.Client) ([]byte, error) {
+	if content := in.Content; len(content) > 0 {
+		return content, nil
+	}
+
+	secretRef := in.SecretRef
+
+	if secretRef == nil {
+		return nil, fmt.Errorf("no bare content and no external Secret reference")
+	}
+
+	secret, namespacedName := &corev1.Secret{}, types.NamespacedName{Name: secretRef.Name, Namespace: secretRef.Namespace}
+	if err := client.Get(ctx, namespacedName, secret); err != nil {
+		return nil, err
+	}
+
+	v, ok := secret.Data[secretRef.KeyPath]
+	if !ok {
+		return nil, fmt.Errorf("secret %s does not have key %s", namespacedName.String(), secretRef.KeyPath)
+	}
+
+	return v, nil
+}

api/v1alpha1/datastore_types.go

@@ -0,0 +1,104 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package v1alpha1
+
+import (
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+type Driver string //+kubebuilder:validation:Enum=etcd;MySQL;PostgreSQL
+
+var (
+	EtcdDriver           Driver = "etcd"
+	KineMySQLDriver      Driver = "MySQL"
+	KinePostgreSQLDriver Driver = "PostgreSQL"
+)
+
+// DataStoreSpec defines the desired state of DataStore.
+type DataStoreSpec struct {
+	// The driver to use to connect to the shared datastore.
+	Driver Driver `json:"driver"`
+	// List of the endpoints to connect to the shared datastore.
+	// No need for protocol, just bare IP/FQDN and port.
+	Endpoints []string `json:"endpoints"` //+kubebuilder:validation:MinLength=1
+	// In case of authentication enabled for the given data store, specifies the username and password pair.
+	// This value is optional.
+	BasicAuth *BasicAuth `json:"basicAuth,omitempty"`
+	// Defines the TLS/SSL configuration required to connect to the data store in a secure way.
+	TLSConfig TLSConfig `json:"tlsConfig"`
+}
+
+// TLSConfig contains the information used to connect to the data store using a secured connection.
+type TLSConfig struct {
+	// Retrieve the Certificate Authority certificate and private key, such as bare content of the file, or a SecretReference.
+	// The key reference is required since etcd authentication is based on certificates, and Kamaji is responsible in creating this.
+	CertificateAuthority CertKeyPair `json:"certificateAuthority"`
+	// Specifies the SSL/TLS key and private key pair used to connect to the data store.
+	ClientCertificate ClientCertificate `json:"clientCertificate"`
+}
+
+type ClientCertificate struct {
+	Certificate ContentRef `json:"certificate"`
+	PrivateKey  ContentRef `json:"privateKey"`
+}
+
+type CertKeyPair struct {
+	Certificate ContentRef  `json:"certificate"`
+	PrivateKey  *ContentRef `json:"privateKey,omitempty"`
+}
+
+// BasicAuth contains the required information to perform the connection using user credentials to the data store.
+type BasicAuth struct {
+	Username ContentRef `json:"username"`
+	Password ContentRef `json:"password"`
+}
+
+type ContentRef struct {
+	// Bare content of the file, base64 encoded.
+	// It has precedence over the SecretReference value.
+	Content   []byte           `json:"content,omitempty"`
+	SecretRef *SecretReference `json:"secretReference,omitempty"`
+}
+
+type SecretReference struct {
+	corev1.SecretReference `json:",inline"`
+	// Name of the key for the given Secret reference where the content is stored.
+	// This value is mandatory.
+	KeyPath string `json:"keyPath"`
+}
+
+// DataStoreStatus defines the observed state of DataStore.
+type DataStoreStatus struct {
+	// List of the Tenant Control Planes, namespaced named, using this data store.
+	UsedBy []string `json:"usedBy,omitempty"`
+}
+
+//+kubebuilder:object:root=true
+//+kubebuilder:subresource:status
+//+kubebuilder:resource:scope=Cluster
+//+kubebuilder:printcolumn:name="Driver",type="string",JSONPath=".spec.driver",description="Kamaji data store driver"
+//+kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="Age"
+
+// DataStore is the Schema for the datastores API.
+type DataStore struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec   DataStoreSpec   `json:"spec,omitempty"`
+	Status DataStoreStatus `json:"status,omitempty"`
+}
+
+//+kubebuilder:object:root=true
+
+// DataStoreList contains a list of DataStore.
+type DataStoreList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []DataStore `json:"items"`
+}
+
+func init() {
+	SchemeBuilder.Register(&DataStore{}, &DataStoreList{})
+}

api/v1alpha1/groupversion_info.go

@@ -2,8 +2,9 @@
 // SPDX-License-Identifier: Apache-2.0
 
 // Package v1alpha1 contains API Schema definitions for the kamaji v1alpha1 API group
-//+kubebuilder:object:generate=true
-//+groupName=kamaji.clastix.io
+// +kubebuilder:object:generate=true
+// +groupName=kamaji.clastix.io
+//nolint
 package v1alpha1
 
 import (

api/v1alpha1/tenantcontrolplane_status.go

@@ -8,42 +8,43 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	networkingv1 "k8s.io/api/networking/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-
-	"github.com/clastix/kamaji/internal/etcd"
 )
 
 // APIServerCertificatesStatus defines the observed state of ETCD Certificate for API server.
 type APIServerCertificatesStatus struct {
 	SecretName string      `json:"secretName,omitempty"`
 	LastUpdate metav1.Time `json:"lastUpdate,omitempty"`
+	Checksum   string      `json:"checksum,omitempty"`
 }
 
 // ETCDCertificateStatus defines the observed state of ETCD Certificate for API server.
 type ETCDCertificateStatus struct {
 	SecretName string      `json:"secretName,omitempty"`
 	LastUpdate metav1.Time `json:"lastUpdate,omitempty"`
+	Checksum   string      `json:"checksum,omitempty"`
 }
 
 // ETCDCertificatesStatus defines the observed state of ETCD Certificate for API server.
 type ETCDCertificatesStatus struct {
-	APIServer ETCDCertificateStatus `json:"apiServer,omitempty"`
-	CA        ETCDCertificateStatus `json:"ca,omitempty"`
+	APIServer APIServerCertificatesStatus `json:"apiServer,omitempty"`
+	CA        ETCDCertificateStatus       `json:"ca,omitempty"`
 }
 
 // CertificatePrivateKeyPairStatus defines the status.
 type CertificatePrivateKeyPairStatus struct {
-	SecretName      string      `json:"secretName,omitempty"`
-	LastUpdate      metav1.Time `json:"lastUpdate,omitempty"`
-	ResourceVersion string      `json:"resourceVersion,omitempty"`
+	SecretName string      `json:"secretName,omitempty"`
+	LastUpdate metav1.Time `json:"lastUpdate,omitempty"`
+	Checksum   string      `json:"checksum,omitempty"`
 }
 
 // PublicKeyPrivateKeyPairStatus defines the status.
 type PublicKeyPrivateKeyPairStatus struct {
 	SecretName string      `json:"secretName,omitempty"`
 	LastUpdate metav1.Time `json:"lastUpdate,omitempty"`
+	Checksum   string      `json:"checksum,omitempty"`
 }
 
-// CertificatesStatus defines the observed state of ETCD Certificates.
+// CertificatesStatus defines the observed state of ETCD TLSConfig.
 type CertificatesStatus struct {
 	CA                     CertificatePrivateKeyPairStatus `json:"ca,omitempty"`
 	APIServer              CertificatePrivateKeyPairStatus `json:"apiServer,omitempty"`
@@ -54,40 +55,31 @@ type CertificatesStatus struct {
 	ETCD                   *ETCDCertificatesStatus         `json:"etcd,omitempty"`
 }
 
-// ETCDStatus defines the observed state of ETCDStatus.
-type ETCDStatus struct {
-	Role etcd.Role `json:"role,omitempty"`
-	User etcd.User `json:"user,omitempty"`
+type DataStoreCertificateStatus struct {
+	SecretName string      `json:"secretName,omitempty"`
+	Checksum   string      `json:"checksum,omitempty"`
+	LastUpdate metav1.Time `json:"lastUpdate,omitempty"`
 }
 
-type SQLCertificateStatus struct {
-	SecretName      string      `json:"secretName,omitempty"`
-	ResourceVersion string      `json:"resourceVersion,omitempty"`
-	LastUpdate      metav1.Time `json:"lastUpdate,omitempty"`
+type DataStoreConfigStatus struct {
+	SecretName string `json:"secretName,omitempty"`
+	Checksum   string `json:"checksum,omitempty"`
 }
 
-type SQLConfigStatus struct {
-	SecretName      string `json:"secretName,omitempty"`
-	ResourceVersion string `json:"resourceVersion,omitempty"`
-}
-
-type SQLSetupStatus struct {
-	Schema                   string      `json:"schema,omitempty"`
-	User                     string      `json:"user,omitempty"`
-	LastUpdate               metav1.Time `json:"lastUpdate,omitempty"`
-	SQLConfigResourceVersion string      `json:"sqlConfigResourceVersion,omitempty"`
-}
-
-type KineMySQLStatus struct {
-	Config      SQLConfigStatus      `json:"config,omitempty"`
-	Setup       SQLSetupStatus       `json:"setup,omitempty"`
-	Certificate SQLCertificateStatus `json:"certificate,omitempty"`
+type DataStoreSetupStatus struct {
+	Schema     string      `json:"schema,omitempty"`
+	User       string      `json:"user,omitempty"`
+	LastUpdate metav1.Time `json:"lastUpdate,omitempty"`
+	Checksum   string      `json:"checksum,omitempty"`
 }
 
 // StorageStatus defines the observed state of StorageStatus.
 type StorageStatus struct {
-	ETCD      *ETCDStatus      `json:"etcd,omitempty"`
-	KineMySQL *KineMySQLStatus `json:"kineMySQL,omitempty"`
+	Driver        string                     `json:"driver,omitempty"`
+	DataStoreName string                     `json:"dataStoreName,omitempty"`
+	Config        DataStoreConfigStatus      `json:"config,omitempty"`
+	Setup         DataStoreSetupStatus       `json:"setup,omitempty"`
+	Certificate   DataStoreCertificateStatus `json:"certificate,omitempty"`
 }
 
 // KubeconfigStatus contains information about the generated kubeconfig.
@@ -128,22 +120,26 @@ type KubeadmPhasesStatus struct {
 type ExternalKubernetesObjectStatus struct {
 	Name      string `json:"name,omitempty"`
 	Namespace string `json:"namespace,omitempty"`
-	// Resource version of k8s object
-	RV string `json:"resourceVersion,omitempty"`
+	Checksum  string `json:"checksum,omitempty"`
 	// Last time when k8s object was updated
 	LastUpdate metav1.Time `json:"lastUpdate,omitempty"`
 }
 
 // KonnectivityStatus defines the status of Konnectivity as Addon.
 type KonnectivityStatus struct {
-	Enabled                     bool                            `json:"enabled"`
-	EgressSelectorConfiguration string                          `json:"egressSelectorConfiguration,omitempty"`
-	Certificate                 CertificatePrivateKeyPairStatus `json:"certificate,omitempty"`
-	Kubeconfig                  KubeconfigStatus                `json:"kubeconfig,omitempty"`
-	ServiceAccount              ExternalKubernetesObjectStatus  `json:"sa,omitempty"`
-	ClusterRoleBinding          ExternalKubernetesObjectStatus  `json:"clusterrolebinding,omitempty"`
-	Agent                       ExternalKubernetesObjectStatus  `json:"agent,omitempty"`
-	Service                     KubernetesServiceStatus         `json:"service,omitempty"`
+	Enabled            bool                            `json:"enabled"`
+	ConfigMap          KonnectivityConfigMap           `json:"configMap,omitempty"`
+	Certificate        CertificatePrivateKeyPairStatus `json:"certificate,omitempty"`
+	Kubeconfig         KubeconfigStatus                `json:"kubeconfig,omitempty"`
+	ServiceAccount     ExternalKubernetesObjectStatus  `json:"sa,omitempty"`
+	ClusterRoleBinding ExternalKubernetesObjectStatus  `json:"clusterrolebinding,omitempty"`
+	Agent              ExternalKubernetesObjectStatus  `json:"agent,omitempty"`
+	Service            KubernetesServiceStatus         `json:"service,omitempty"`
+}
+
+type KonnectivityConfigMap struct {
+	Name     string `json:"name,omitempty"`
+	Checksum string `json:"checksum,omitempty"`
 }
 
 // AddonStatus defines the observed state of an Addon.
@@ -155,9 +151,8 @@ type AddonStatus struct {
 
 // AddonsStatus defines the observed state of the different Addons.
 type AddonsStatus struct {
-	CoreDNS   AddonStatus `json:"coreDNS,omitempty"`
-	KubeProxy AddonStatus `json:"kubeProxy,omitempty"`
-
+	CoreDNS      AddonStatus        `json:"coreDNS,omitempty"`
+	KubeProxy    AddonStatus        `json:"kubeProxy,omitempty"`
 	Konnectivity KonnectivityStatus `json:"konnectivity,omitempty"`
 }
 
@@ -189,7 +184,7 @@ type KubernetesStatus struct {
 	Version    KubernetesVersion          `json:"version,omitempty"`
 	Deployment KubernetesDeploymentStatus `json:"deployment,omitempty"`
 	Service    KubernetesServiceStatus    `json:"service,omitempty"`
-	Ingress    KubernetesIngressStatus    `json:"ingress,omitempty"`
+	Ingress    *KubernetesIngressStatus   `json:"ingress,omitempty"`
 }
 
 // +kubebuilder:validation:Enum=Provisioning;Upgrading;Ready;NotReady
@@ -207,12 +202,14 @@ type KubernetesVersion struct {
 	Version string `json:"version,omitempty"`
 	// +kubebuilder:default=Provisioning
 	// Status returns the current status of the Kubernetes version, such as its provisioning state, or completed upgrade.
-	Status *KubernetesVersionStatus `json:"status"`
+	Status *KubernetesVersionStatus `json:"status,omitempty"`
 }
 
 // KubernetesDeploymentStatus defines the status for the Tenant Control Plane Deployment in the management cluster.
 type KubernetesDeploymentStatus struct {
 	appsv1.DeploymentStatus `json:",inline"`
+	// Selector is the label selector used to group the Tenant Control Plane Pods used by the scale subresource.
+	Selector string `json:"selector"`
 	// The name of the Deployment for the given cluster.
 	Name string `json:"name"`
 	// The namespace which the Deployment for the given cluster is deployed.

api/v1alpha1/tenantcontrolplane_types.go

@@ -64,13 +64,12 @@ type ControlPlane struct {
 	// Defining the options for the Tenant Control Plane Service resource.
 	Service ServiceSpec `json:"service"`
 	// Defining the options for an Optional Ingress which will expose API Server of the Tenant Control Plane
-	Ingress IngressSpec `json:"ingress,omitempty"`
+	Ingress *IngressSpec `json:"ingress,omitempty"`
 }
 
 // IngressSpec defines the options for the ingress which will expose API Server of the Tenant Control Plane.
 type IngressSpec struct {
 	AdditionalMetadata AdditionalMetadata `json:"additionalMetadata,omitempty"`
-	Enabled            bool               `json:"enabled"`
 	IngressClassName   string             `json:"ingressClassName,omitempty"`
 	// Hostname is an optional field which will be used as Ingress's Host. If it is not defined,
 	// Ingress's host will be "<tenant>.<namespace>.<domain>", where domain is specified under NetworkProfileSpec
@@ -86,6 +85,11 @@ type ControlPlaneComponentsResources struct {
 type DeploymentSpec struct {
 	// +kubebuilder:default=2
 	Replicas int32 `json:"replicas,omitempty"`
+	// TopologySpreadConstraints describes how the Tenant Control Plane pods ought to spread across topology
+	// domains. Scheduler will schedule pods in a way which abides by the constraints.
+	// In case of nil underlying LabelSelector, the Kamaji one for the given Tenant Control Plane will be used.
+	// All topologySpreadConstraints are ANDed.
+	TopologySpreadConstraints []corev1.TopologySpreadConstraint `json:"topologySpreadConstraints,omitempty"`
 	// Resources defines the amount of memory and CPU to allocate to each component of the Control Plane
 	// (kube-apiserver, controller-manager, and scheduler).
 	Resources *ControlPlaneComponentsResources `json:"resources,omitempty"`
@@ -111,20 +115,31 @@ type ServiceSpec struct {
 }
 
 // AddonSpec defines the spec for every addon.
-type AddonSpec struct{}
+type AddonSpec struct {
+	ImageOverrideTrait `json:",inline"`
+}
+
+type ImageOverrideTrait struct {
+	// ImageRepository sets the container registry to pull images from.
+	// if not set, the default ImageRepository will be used instead.
+	ImageRepository string `json:"imageRepository,omitempty"`
+	// ImageTag allows to specify a tag for the image.
+	// In case this value is set, kubeadm does not change automatically the version of the above components during upgrades.
+	ImageTag string `json:"imageTag,omitempty"`
+}
 
 // KonnectivitySpec defines the spec for Konnectivity.
 type KonnectivitySpec struct {
 	// Port of Konnectivity proxy server.
 	ProxyPort int32 `json:"proxyPort"`
 	// Version for Konnectivity server and agent.
-	// +kubebuilder:default=v0.0.31
+	// +kubebuilder:default=v0.0.32
 	Version string `json:"version,omitempty"`
 	// ServerImage defines the container image for Konnectivity's server.
-	// +kubebuilder:default=us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-server
+	// +kubebuilder:default=registry.k8s.io/kas-network-proxy/proxy-server
 	ServerImage string `json:"serverImage,omitempty"`
 	// AgentImage defines the container image for Konnectivity's agent.
-	// +kubebuilder:default=us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-agent
+	// +kubebuilder:default=registry.k8s.io/kas-network-proxy/proxy-agent
 	AgentImage string `json:"agentImage,omitempty"`
 	// Resources define the amount of CPU and memory to allocate to the Konnectivity server.
 	Resources *corev1.ResourceRequirements `json:"resources,omitempty"`
@@ -132,13 +147,22 @@ type KonnectivitySpec struct {
 
 // AddonsSpec defines the enabled addons and their features.
 type AddonsSpec struct {
-	CoreDNS      *AddonSpec        `json:"coreDNS,omitempty"`
+	// Enables the DNS addon in the Tenant Cluster.
+	// The registry and the tag are configurable, the image is hard-coded to `coredns`.
+	CoreDNS *AddonSpec `json:"coreDNS,omitempty"`
+	// Enables the Konnectivity addon in the Tenant Cluster, required if the worker nodes are in a different network.
 	Konnectivity *KonnectivitySpec `json:"konnectivity,omitempty"`
-	KubeProxy    *AddonSpec        `json:"kubeProxy,omitempty"`
+	// Enables the kube-proxy addon in the Tenant Cluster.
+	// The registry and the tag are configurable, the image is hard-coded to `kube-proxy`.
+	KubeProxy *AddonSpec `json:"kubeProxy,omitempty"`
 }
 
 // TenantControlPlaneSpec defines the desired state of TenantControlPlane.
 type TenantControlPlaneSpec struct {
+	// DataStore allows to specify a DataStore that should be used to store the Kubernetes data for the given Tenant Control Plane.
+	// This parameter is optional and acts as an override over the default one which is used by the Kamaji Operator.
+	// Migration from a different DataStore to another one is not yet supported and the reconciliation will be blocked.
+	DataStore    string       `json:"dataStore,omitempty"`
 	ControlPlane ControlPlane `json:"controlPlane"`
 	// Kubernetes specification for tenant control plane
 	Kubernetes KubernetesSpec `json:"kubernetes"`
@@ -150,6 +174,7 @@ type TenantControlPlaneSpec struct {
 
 // +kubebuilder:object:root=true
 // +kubebuilder:subresource:status
+// +kubebuilder:subresource:scale:specpath=.spec.controlPlane.deployment.replicas,statuspath=.status.kubernetesResources.deployment.replicas,selectorpath=.status.kubernetesResources.deployment.selector
 // +kubebuilder:resource:shortName=tcp
 // +kubebuilder:printcolumn:name="Version",type="string",JSONPath=".spec.kubernetes.version",description="Kubernetes version"
 // +kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.kubernetesResources.version.status",description="Kubernetes version"

api/v1alpha1/zz_generated.deepcopy.go

@@ -61,6 +61,7 @@ func (in *AdditionalMetadata) DeepCopy() *AdditionalMetadata {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *AddonSpec) DeepCopyInto(out *AddonSpec) {
 	*out = *in
+	out.ImageOverrideTrait = in.ImageOverrideTrait
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AddonSpec.
@@ -156,6 +157,44 @@ func (in AdmissionControllers) DeepCopy() AdmissionControllers {
 	return *out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *BasicAuth) DeepCopyInto(out *BasicAuth) {
+	*out = *in
+	in.Username.DeepCopyInto(&out.Username)
+	in.Password.DeepCopyInto(&out.Password)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BasicAuth.
+func (in *BasicAuth) DeepCopy() *BasicAuth {
+	if in == nil {
+		return nil
+	}
+	out := new(BasicAuth)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *CertKeyPair) DeepCopyInto(out *CertKeyPair) {
+	*out = *in
+	in.Certificate.DeepCopyInto(&out.Certificate)
+	if in.PrivateKey != nil {
+		in, out := &in.PrivateKey, &out.PrivateKey
+		*out = new(ContentRef)
+		(*in).DeepCopyInto(*out)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CertKeyPair.
+func (in *CertKeyPair) DeepCopy() *CertKeyPair {
+	if in == nil {
+		return nil
+	}
+	out := new(CertKeyPair)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *CertificatePrivateKeyPairStatus) DeepCopyInto(out *CertificatePrivateKeyPairStatus) {
 	*out = *in
@@ -198,12 +237,58 @@ func (in *CertificatesStatus) DeepCopy() *CertificatesStatus {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ClientCertificate) DeepCopyInto(out *ClientCertificate) {
+	*out = *in
+	in.Certificate.DeepCopyInto(&out.Certificate)
+	in.PrivateKey.DeepCopyInto(&out.PrivateKey)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClientCertificate.
+func (in *ClientCertificate) DeepCopy() *ClientCertificate {
+	if in == nil {
+		return nil
+	}
+	out := new(ClientCertificate)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ContentRef) DeepCopyInto(out *ContentRef) {
+	*out = *in
+	if in.Content != nil {
+		in, out := &in.Content, &out.Content
+		*out = make([]byte, len(*in))
+		copy(*out, *in)
+	}
+	if in.SecretRef != nil {
+		in, out := &in.SecretRef, &out.SecretRef
+		*out = new(SecretReference)
+		**out = **in
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ContentRef.
+func (in *ContentRef) DeepCopy() *ContentRef {
+	if in == nil {
+		return nil
+	}
+	out := new(ContentRef)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ControlPlane) DeepCopyInto(out *ControlPlane) {
 	*out = *in
 	in.Deployment.DeepCopyInto(&out.Deployment)
 	in.Service.DeepCopyInto(&out.Service)
-	in.Ingress.DeepCopyInto(&out.Ingress)
+	if in.Ingress != nil {
+		in, out := &in.Ingress, &out.Ingress
+		*out = new(IngressSpec)
+		(*in).DeepCopyInto(*out)
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ControlPlane.
@@ -281,9 +366,168 @@ func (in *ControlPlaneExtraArgs) DeepCopy() *ControlPlaneExtraArgs {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *DataStore) DeepCopyInto(out *DataStore) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
+	in.Spec.DeepCopyInto(&out.Spec)
+	in.Status.DeepCopyInto(&out.Status)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataStore.
+func (in *DataStore) DeepCopy() *DataStore {
+	if in == nil {
+		return nil
+	}
+	out := new(DataStore)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *DataStore) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *DataStoreCertificateStatus) DeepCopyInto(out *DataStoreCertificateStatus) {
+	*out = *in
+	in.LastUpdate.DeepCopyInto(&out.LastUpdate)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataStoreCertificateStatus.
+func (in *DataStoreCertificateStatus) DeepCopy() *DataStoreCertificateStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(DataStoreCertificateStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *DataStoreConfigStatus) DeepCopyInto(out *DataStoreConfigStatus) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataStoreConfigStatus.
+func (in *DataStoreConfigStatus) DeepCopy() *DataStoreConfigStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(DataStoreConfigStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *DataStoreList) DeepCopyInto(out *DataStoreList) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ListMeta.DeepCopyInto(&out.ListMeta)
+	if in.Items != nil {
+		in, out := &in.Items, &out.Items
+		*out = make([]DataStore, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataStoreList.
+func (in *DataStoreList) DeepCopy() *DataStoreList {
+	if in == nil {
+		return nil
+	}
+	out := new(DataStoreList)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *DataStoreList) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *DataStoreSetupStatus) DeepCopyInto(out *DataStoreSetupStatus) {
+	*out = *in
+	in.LastUpdate.DeepCopyInto(&out.LastUpdate)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataStoreSetupStatus.
+func (in *DataStoreSetupStatus) DeepCopy() *DataStoreSetupStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(DataStoreSetupStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *DataStoreSpec) DeepCopyInto(out *DataStoreSpec) {
+	*out = *in
+	if in.Endpoints != nil {
+		in, out := &in.Endpoints, &out.Endpoints
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
+	if in.BasicAuth != nil {
+		in, out := &in.BasicAuth, &out.BasicAuth
+		*out = new(BasicAuth)
+		(*in).DeepCopyInto(*out)
+	}
+	in.TLSConfig.DeepCopyInto(&out.TLSConfig)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataStoreSpec.
+func (in *DataStoreSpec) DeepCopy() *DataStoreSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(DataStoreSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *DataStoreStatus) DeepCopyInto(out *DataStoreStatus) {
+	*out = *in
+	if in.UsedBy != nil {
+		in, out := &in.UsedBy, &out.UsedBy
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataStoreStatus.
+func (in *DataStoreStatus) DeepCopy() *DataStoreStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(DataStoreStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *DeploymentSpec) DeepCopyInto(out *DeploymentSpec) {
 	*out = *in
+	if in.TopologySpreadConstraints != nil {
+		in, out := &in.TopologySpreadConstraints, &out.TopologySpreadConstraints
+		*out = make([]v1.TopologySpreadConstraint, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
 	if in.Resources != nil {
 		in, out := &in.Resources, &out.Resources
 		*out = new(ControlPlaneComponentsResources)
@@ -340,23 +584,6 @@ func (in *ETCDCertificatesStatus) DeepCopy() *ETCDCertificatesStatus {
 	return out
 }
 
-// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *ETCDStatus) DeepCopyInto(out *ETCDStatus) {
-	*out = *in
-	in.Role.DeepCopyInto(&out.Role)
-	in.User.DeepCopyInto(&out.User)
-}
-
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ETCDStatus.
-func (in *ETCDStatus) DeepCopy() *ETCDStatus {
-	if in == nil {
-		return nil
-	}
-	out := new(ETCDStatus)
-	in.DeepCopyInto(out)
-	return out
-}
-
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ExternalKubernetesObjectStatus) DeepCopyInto(out *ExternalKubernetesObjectStatus) {
 	*out = *in
@@ -373,6 +600,21 @@ func (in *ExternalKubernetesObjectStatus) DeepCopy() *ExternalKubernetesObjectSt
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ImageOverrideTrait) DeepCopyInto(out *ImageOverrideTrait) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ImageOverrideTrait.
+func (in *ImageOverrideTrait) DeepCopy() *ImageOverrideTrait {
+	if in == nil {
+		return nil
+	}
+	out := new(ImageOverrideTrait)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *IngressSpec) DeepCopyInto(out *IngressSpec) {
 	*out = *in
@@ -390,19 +632,16 @@ func (in *IngressSpec) DeepCopy() *IngressSpec {
 }
 
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *KineMySQLStatus) DeepCopyInto(out *KineMySQLStatus) {
+func (in *KonnectivityConfigMap) DeepCopyInto(out *KonnectivityConfigMap) {
 	*out = *in
-	out.Config = in.Config
-	in.Setup.DeepCopyInto(&out.Setup)
-	in.Certificate.DeepCopyInto(&out.Certificate)
 }
 
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KineMySQLStatus.
-func (in *KineMySQLStatus) DeepCopy() *KineMySQLStatus {
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KonnectivityConfigMap.
+func (in *KonnectivityConfigMap) DeepCopy() *KonnectivityConfigMap {
 	if in == nil {
 		return nil
 	}
-	out := new(KineMySQLStatus)
+	out := new(KonnectivityConfigMap)
 	in.DeepCopyInto(out)
 	return out
 }
@@ -430,6 +669,7 @@ func (in *KonnectivitySpec) DeepCopy() *KonnectivitySpec {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *KonnectivityStatus) DeepCopyInto(out *KonnectivityStatus) {
 	*out = *in
+	out.ConfigMap = in.ConfigMap
 	in.Certificate.DeepCopyInto(&out.Certificate)
 	in.Kubeconfig.DeepCopyInto(&out.Kubeconfig)
 	in.ServiceAccount.DeepCopyInto(&out.ServiceAccount)
@@ -623,7 +863,11 @@ func (in *KubernetesStatus) DeepCopyInto(out *KubernetesStatus) {
 	in.Version.DeepCopyInto(&out.Version)
 	in.Deployment.DeepCopyInto(&out.Deployment)
 	in.Service.DeepCopyInto(&out.Service)
-	in.Ingress.DeepCopyInto(&out.Ingress)
+	if in.Ingress != nil {
+		in, out := &in.Ingress, &out.Ingress
+		*out = new(KubernetesIngressStatus)
+		(*in).DeepCopyInto(*out)
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubernetesStatus.
@@ -698,48 +942,17 @@ func (in *PublicKeyPrivateKeyPairStatus) DeepCopy() *PublicKeyPrivateKeyPairStat
 }
 
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *SQLCertificateStatus) DeepCopyInto(out *SQLCertificateStatus) {
+func (in *SecretReference) DeepCopyInto(out *SecretReference) {
 	*out = *in
-	in.LastUpdate.DeepCopyInto(&out.LastUpdate)
+	out.SecretReference = in.SecretReference
 }
 
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SQLCertificateStatus.
-func (in *SQLCertificateStatus) DeepCopy() *SQLCertificateStatus {
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecretReference.
+func (in *SecretReference) DeepCopy() *SecretReference {
 	if in == nil {
 		return nil
 	}
-	out := new(SQLCertificateStatus)
-	in.DeepCopyInto(out)
-	return out
-}
-
-// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *SQLConfigStatus) DeepCopyInto(out *SQLConfigStatus) {
-	*out = *in
-}
-
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SQLConfigStatus.
-func (in *SQLConfigStatus) DeepCopy() *SQLConfigStatus {
-	if in == nil {
-		return nil
-	}
-	out := new(SQLConfigStatus)
-	in.DeepCopyInto(out)
-	return out
-}
-
-// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *SQLSetupStatus) DeepCopyInto(out *SQLSetupStatus) {
-	*out = *in
-	in.LastUpdate.DeepCopyInto(&out.LastUpdate)
-}
-
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SQLSetupStatus.
-func (in *SQLSetupStatus) DeepCopy() *SQLSetupStatus {
-	if in == nil {
-		return nil
-	}
-	out := new(SQLSetupStatus)
+	out := new(SecretReference)
 	in.DeepCopyInto(out)
 	return out
 }
@@ -763,16 +976,9 @@ func (in *ServiceSpec) DeepCopy() *ServiceSpec {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *StorageStatus) DeepCopyInto(out *StorageStatus) {
 	*out = *in
-	if in.ETCD != nil {
-		in, out := &in.ETCD, &out.ETCD
-		*out = new(ETCDStatus)
-		(*in).DeepCopyInto(*out)
-	}
-	if in.KineMySQL != nil {
-		in, out := &in.KineMySQL, &out.KineMySQL
-		*out = new(KineMySQLStatus)
-		(*in).DeepCopyInto(*out)
-	}
+	out.Config = in.Config
+	in.Setup.DeepCopyInto(&out.Setup)
+	in.Certificate.DeepCopyInto(&out.Certificate)
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StorageStatus.
@@ -785,6 +991,23 @@ func (in *StorageStatus) DeepCopy() *StorageStatus {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *TLSConfig) DeepCopyInto(out *TLSConfig) {
+	*out = *in
+	in.CertificateAuthority.DeepCopyInto(&out.CertificateAuthority)
+	in.ClientCertificate.DeepCopyInto(&out.ClientCertificate)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TLSConfig.
+func (in *TLSConfig) DeepCopy() *TLSConfig {
+	if in == nil {
+		return nil
+	}
+	out := new(TLSConfig)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *TenantControlPlane) DeepCopyInto(out *TenantControlPlane) {
 	*out = *in

charts/kamaji/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v2
 name: kamaji
-description: A Kubernetes distribution aimed to build and operate a Managed Kubernetes service with a fraction of operational burde.
+description: Kamaji is a tool aimed to build and operate a Managed Kubernetes Service with a fraction of the operational burden. With Kamaji, you can deploy and operate hundreds of Kubernetes clusters as a hyper-scaler.
 
 # A chart can be either an 'application' or a 'library' chart.
 #
@@ -15,16 +15,16 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.1
+version: 0.9.2
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: 0.1.0
+appVersion: v0.1.0
 
-home: https://github.com/clastix/kamaji-internal/tree/master/helm/kamaji
-sources: ["https://github.com/clastix/kamaji-internal"]
+home: https://github.com/clastix/kamaji
+sources: ["https://github.com/clastix/kamaji"]
 kubeVersion: ">=1.18"
 maintainers:
 - email: iam@mendrugory.com
@@ -33,3 +33,5 @@ maintainers:
   name: Dario Tranchitella
 - email: me@maxgio.it
   name: Massimiliano Giovagnoli
+- email: me@bsctl.io
+  name: Adriano Pezzuto

charts/kamaji/README.md

@@ -0,0 +1,141 @@
+# kamaji
+
+![Version: 0.9.2](https://img.shields.io/badge/Version-0.9.2-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v0.1.0](https://img.shields.io/badge/AppVersion-v0.1.0-informational?style=flat-square)
+
+Kamaji is a tool aimed to build and operate a Managed Kubernetes Service with a fraction of the operational burden. With Kamaji, you can deploy and operate hundreds of Kubernetes clusters as a hyper-scaler.
+
+## Maintainers
+
+| Name | Email | Url |
+| ---- | ------ | --- |
+| Gonzalo Gabriel Jiménez Fuentes | <iam@mendrugory.com> |  |
+| Dario Tranchitella | <dario@tranchitella.eu> |  |
+| Massimiliano Giovagnoli | <me@maxgio.it> |  |
+| Adriano Pezzuto | <me@bsctl.io> |  |
+
+## Source Code
+
+* <https://github.com/clastix/kamaji>
+
+## Requirements
+
+Kubernetes: `>=1.18`
+
+[Kamaji](https://github.com/clastix/kamaji) requires a [multi-tenant `etcd`](https://github.com/clastix/kamaji-internal/blob/master/deploy/getting-started-with-kamaji.md#setup-internal-multi-tenant-etcd) cluster.
+This Helm Chart starting from v0.1.1 provides the installation of an internal `etcd` in order to streamline the local test. If you'd like to use an externally managed etcd instance, you can specify the overrides and by setting the value `etcd.deploy=false`.
+
+> For production use an externally managed `etcd` is highly recommended, the `etcd` addon offered by this Chart is not considered production-grade.
+
+## Install Kamaji
+
+To install the Chart with the release name `kamaji`:
+
+        helm upgrade --install --namespace kamaji-system --create-namespace clastix/kamaji
+
+Show the status:
+
+        helm status kamaji -n kamaji-system
+
+Upgrade the Chart
+
+        helm upgrade kamaji -n kamaji-system clastix/kamaji
+
+Uninstall the Chart
+
+        helm uninstall kamaji -n kamaji-system
+
+## Customize the installation
+
+There are two methods for specifying overrides of values during Chart installation: `--values` and `--set`.
+
+The `--values` option is the preferred method because it allows you to keep your overrides in a YAML file, rather than specifying them all on the command line. Create a copy of the YAML file `values.yaml` and add your overrides to it.
+
+Specify your overrides file when you install the Chart:
+
+        helm upgrade kamaji --install --namespace kamaji-system --create-namespace clastix/kamaji --values myvalues.yaml
+
+The values in your overrides file `myvalues.yaml` will override their counterparts in the Chart's values.yaml file. Any values in `values.yaml` that weren’t overridden will keep their defaults.
+
+If you only need to make minor customizations, you can specify them on the command line by using the `--set` option. For example:
+
+        helm upgrade kamaji --install --namespace kamaji-system --create-namespace clastix/kamaji --set etcd.deploy=false
+
+Here the values you can override:
+
+## Values
+
+| Key | Type | Default | Description |
+|-----|------|---------|-------------|
+| affinity | object | `{}` | Kubernetes affinity rules to apply to Kamaji controller pods |
+| configPath | string | `"./kamaji.yaml"` | Configuration file path alternative. (default "./kamaji.yaml") |
+| datastore.basicAuth.passwordSecret.keyPath | string | `nil` | The Secret key where the data is stored. |
+| datastore.basicAuth.passwordSecret.name | string | `nil` | The name of the Secret containing the password used to connect to the relational database. |
+| datastore.basicAuth.passwordSecret.namespace | string | `nil` | The namespace of the Secret containing the password used to connect to the relational database. |
+| datastore.basicAuth.usernameSecret.keyPath | string | `nil` | The Secret key where the data is stored. |
+| datastore.basicAuth.usernameSecret.name | string | `nil` | The name of the Secret containing the username used to connect to the relational database. |
+| datastore.basicAuth.usernameSecret.namespace | string | `nil` | The namespace of the Secret containing the username used to connect to the relational database. |
+| datastore.driver | string | `"etcd"` | (string) The Kamaji Datastore driver, supported: etcd, MySQL, PostgreSQL (defaults=etcd). |
+| datastore.endpoints | list | `[]` | (array) List of endpoints of the selected Datastore. When letting the Chart install the etcd datastore, this field is populated automatically. |
+| datastore.nameOverride | string | `nil` | The Datastore name override, if empty defaults to `default` |
+| datastore.tlsConfig.certificateAuthority.certificate.keyPath | string | `nil` | Key of the Secret which contains the content of the certificate. |
+| datastore.tlsConfig.certificateAuthority.certificate.name | string | `nil` | Name of the Secret containing the CA required to establish the mandatory SSL/TLS connection to the datastore. |
+| datastore.tlsConfig.certificateAuthority.certificate.namespace | string | `nil` | Namespace of the Secret containing the CA required to establish the mandatory SSL/TLS connection to the datastore. |
+| datastore.tlsConfig.certificateAuthority.privateKey.keyPath | string | `nil` | Key of the Secret which contains the content of the private key. |
+| datastore.tlsConfig.certificateAuthority.privateKey.name | string | `nil` | Name of the Secret containing the CA private key required to establish the mandatory SSL/TLS connection to the datastore. |
+| datastore.tlsConfig.certificateAuthority.privateKey.namespace | string | `nil` | Namespace of the Secret containing the CA private key required to establish the mandatory SSL/TLS connection to the datastore. |
+| datastore.tlsConfig.clientCertificate.certificate.keyPath | string | `nil` | Key of the Secret which contains the content of the certificate. |
+| datastore.tlsConfig.clientCertificate.certificate.name | string | `nil` | Name of the Secret containing the client certificate required to establish the mandatory SSL/TLS connection to the datastore. |
+| datastore.tlsConfig.clientCertificate.certificate.namespace | string | `nil` | Namespace of the Secret containing the client certificate required to establish the mandatory SSL/TLS connection to the datastore. |
+| datastore.tlsConfig.clientCertificate.privateKey.keyPath | string | `nil` | Key of the Secret which contains the content of the private key. |
+| datastore.tlsConfig.clientCertificate.privateKey.name | string | `nil` | Name of the Secret containing the client certificate private key required to establish the mandatory SSL/TLS connection to the datastore. |
+| datastore.tlsConfig.clientCertificate.privateKey.namespace | string | `nil` | Namespace of the Secret containing the client certificate private key required to establish the mandatory SSL/TLS connection to the datastore. |
+| etcd.compactionInterval | int | `0` | ETCD Compaction interval (e.g. "5m0s"). (default: "0" (disabled)) |
+| etcd.deploy | bool | `true` | Install an etcd with enabled multi-tenancy along with Kamaji |
+| etcd.image | object | `{"pullPolicy":"IfNotPresent","repository":"quay.io/coreos/etcd","tag":"v3.5.4"}` | Install specific etcd image |
+| etcd.livenessProbe | object | `{"failureThreshold":8,"httpGet":{"path":"/health?serializable=true","port":2381,"scheme":"HTTP"},"initialDelaySeconds":10,"periodSeconds":10,"timeoutSeconds":15}` | The livenessProbe for the etcd container |
+| etcd.overrides.caSecret.name | string | `"etcd-certs"` | Name of the secret which contains CA's certificate and private key. (default: "etcd-certs") |
+| etcd.overrides.caSecret.namespace | string | `"kamaji-system"` | Namespace of the secret which contains CA's certificate and private key. (default: "kamaji-system") |
+| etcd.overrides.clientSecret.name | string | `"root-client-certs"` | Name of the secret which contains ETCD client certificates. (default: "root-client-certs") |
+| etcd.overrides.clientSecret.namespace | string | `"kamaji-system"` | Name of the namespace where the secret which contains ETCD client certificates is. (default: "kamaji-system") |
+| etcd.overrides.endpoints | object | `{"etcd-0":"etcd-0.etcd.kamaji-system.svc.cluster.local","etcd-1":"etcd-1.etcd.kamaji-system.svc.cluster.local","etcd-2":"etcd-2.etcd.kamaji-system.svc.cluster.local"}` | (map) Dictionary of the endpoints for the etcd cluster's members, key is the name of the etcd server. Don't define the protocol (TLS is automatically inflected), or any port, inflected from .etcd.peerApiPort value. |
+| etcd.peerApiPort | int | `2380` | The peer API port which servers are listening to. |
+| etcd.persistence.accessModes[0] | string | `"ReadWriteOnce"` |  |
+| etcd.persistence.size | string | `"10Gi"` |  |
+| etcd.persistence.storageClass | string | `""` |  |
+| etcd.port | int | `2379` | The client request port. |
+| etcd.serviceAccount.create | bool | `true` | Create a ServiceAccount, required to install and provision the etcd backing storage (default: true) |
+| etcd.serviceAccount.name | string | `""` | Define the ServiceAccount name to use during the setup and provision of the etcd backing storage (default: "") |
+| extraArgs | list | `[]` | A list of extra arguments to add to the kamaji controller default ones |
+| fullnameOverride | string | `""` |  |
+| healthProbeBindAddress | string | `":8081"` | The address the probe endpoint binds to. (default ":8081") |
+| image.pullPolicy | string | `"Always"` |  |
+| image.repository | string | `"clastix/kamaji"` | The container image of the Kamaji controller. |
+| image.tag | string | `nil` | Overrides the image tag whose default is the chart appVersion. |
+| imagePullSecrets | list | `[]` |  |
+| livenessProbe | object | `{"httpGet":{"path":"/healthz","port":"healthcheck"},"initialDelaySeconds":15,"periodSeconds":20}` | The livenessProbe for the controller container |
+| loggingDevel.enable | bool | `false` | (string) Development Mode defaults(encoder=consoleEncoder,logLevel=Debug,stackTraceLevel=Warn). Production Mode defaults(encoder=jsonEncoder,logLevel=Info,stackTraceLevel=Error) (default false) |
+| metricsBindAddress | string | `":8080"` | (string) The address the metric endpoint binds to. (default ":8080") |
+| nameOverride | string | `""` |  |
+| nodeSelector | object | `{}` | Kubernetes node selector rules to schedule Kamaji controller |
+| podAnnotations | object | `{}` | The annotations to apply to the Kamaji controller pods. |
+| podSecurityContext | object | `{"runAsNonRoot":true}` | The securityContext to apply to the Kamaji controller pods. |
+| readinessProbe | object | `{"httpGet":{"path":"/readyz","port":"healthcheck"},"initialDelaySeconds":5,"periodSeconds":10}` | The readinessProbe for the controller container |
+| replicaCount | int | `1` | The number of the pod replicas for the Kamaji controller. |
+| resources.limits.cpu | string | `"200m"` |  |
+| resources.limits.memory | string | `"100Mi"` |  |
+| resources.requests.cpu | string | `"100m"` |  |
+| resources.requests.memory | string | `"20Mi"` |  |
+| securityContext | object | `{"allowPrivilegeEscalation":false}` | The securityContext to apply to the Kamaji controller container only. It does not apply to the Kamaji RBAC proxy container. |
+| service.port | int | `8443` |  |
+| service.type | string | `"ClusterIP"` |  |
+| serviceAccount.annotations | object | `{}` |  |
+| serviceAccount.create | bool | `true` |  |
+| serviceAccount.name | string | `"kamaji-controller-manager"` |  |
+| temporaryDirectoryPath | string | `"/tmp/kamaji"` | Directory which will be used to work with temporary files. (default "/tmp/kamaji") |
+| tolerations | list | `[]` | Kubernetes node taints that the Kamaji controller pods would tolerate |
+
+## Installing and managing etcd as DataStore
+
+Kamaji supports multiple data store, although `etcd` is the default one: thus, an initial cluster will be created upon the Chart installation.
+
+The `DataStore` resource can be configured with the proper values in case of overrides when using a different driver, otherwise all the required data will be inherited by the Chart values.

charts/kamaji/README.md.gotmpl

@@ -0,0 +1,62 @@
+{{ template "chart.header" . }}
+{{ template "chart.deprecationWarning" . }}
+
+{{ template "chart.badgesSection" . }}
+
+{{ template "chart.description" . }}
+
+{{ template "chart.maintainersSection" . }}
+
+{{ template "chart.sourcesSection" . }}
+
+{{ template "chart.requirementsSection" . }}
+
+[Kamaji](https://github.com/clastix/kamaji) requires a [multi-tenant `etcd`](https://github.com/clastix/kamaji-internal/blob/master/deploy/getting-started-with-kamaji.md#setup-internal-multi-tenant-etcd) cluster.
+This Helm Chart starting from v0.1.1 provides the installation of an internal `etcd` in order to streamline the local test. If you'd like to use an externally managed etcd instance, you can specify the overrides and by setting the value `etcd.deploy=false`.
+
+> For production use an externally managed `etcd` is highly recommended, the `etcd` addon offered by this Chart is not considered production-grade.
+
+## Install Kamaji
+
+To install the Chart with the release name `kamaji`:
+
+
+        helm upgrade --install --namespace kamaji-system --create-namespace clastix/kamaji
+
+Show the status:
+
+        helm status kamaji -n kamaji-system
+
+Upgrade the Chart
+
+        helm upgrade kamaji -n kamaji-system clastix/kamaji
+
+Uninstall the Chart
+
+        helm uninstall kamaji -n kamaji-system
+
+## Customize the installation
+
+There are two methods for specifying overrides of values during Chart installation: `--values` and `--set`.
+
+The `--values` option is the preferred method because it allows you to keep your overrides in a YAML file, rather than specifying them all on the command line. Create a copy of the YAML file `values.yaml` and add your overrides to it.
+
+Specify your overrides file when you install the Chart:
+
+        helm upgrade kamaji --install --namespace kamaji-system --create-namespace clastix/kamaji --values myvalues.yaml
+
+The values in your overrides file `myvalues.yaml` will override their counterparts in the Chart's values.yaml file. Any values in `values.yaml` that weren’t overridden will keep their defaults.
+
+If you only need to make minor customizations, you can specify them on the command line by using the `--set` option. For example:
+
+        helm upgrade kamaji --install --namespace kamaji-system --create-namespace clastix/kamaji --set etcd.deploy=false
+
+Here the values you can override:
+
+{{ template "chart.valuesSection" . }}
+
+## Installing and managing etcd as DataStore
+
+Kamaji supports multiple data store, although `etcd` is the default one: thus, an initial cluster will be created upon the Chart installation.
+
+The `DataStore` resource can be configured with the proper values in case of overrides when using a different driver, otherwise all the required data will be inherited by the Chart values.

charts/kamaji/crds/datastore.yaml

@@ -0,0 +1,268 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.9.2
+  creationTimestamp: null
+  name: datastores.kamaji.clastix.io
+spec:
+  group: kamaji.clastix.io
+  names:
+    kind: DataStore
+    listKind: DataStoreList
+    plural: datastores
+    singular: datastore
+  scope: Cluster
+  versions:
+  - additionalPrinterColumns:
+    - description: Kamaji data store driver
+      jsonPath: .spec.driver
+      name: Driver
+      type: string
+    - description: Age
+      jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1alpha1
+    schema:
+      openAPIV3Schema:
+        description: DataStore is the Schema for the datastores API.
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: DataStoreSpec defines the desired state of DataStore.
+            properties:
+              basicAuth:
+                description: In case of authentication enabled for the given data
+                  store, specifies the username and password pair. This value is optional.
+                properties:
+                  password:
+                    properties:
+                      content:
+                        description: Bare content of the file, base64 encoded. It
+                          has precedence over the SecretReference value.
+                        format: byte
+                        type: string
+                      secretReference:
+                        properties:
+                          keyPath:
+                            description: Name of the key for the given Secret reference
+                              where the content is stored. This value is mandatory.
+                            type: string
+                          name:
+                            description: name is unique within a namespace to reference
+                              a secret resource.
+                            type: string
+                          namespace:
+                            description: namespace defines the space within which
+                              the secret name must be unique.
+                            type: string
+                        required:
+                        - keyPath
+                        type: object
+                        x-kubernetes-map-type: atomic
+                    type: object
+                  username:
+                    properties:
+                      content:
+                        description: Bare content of the file, base64 encoded. It
+                          has precedence over the SecretReference value.
+                        format: byte
+                        type: string
+                      secretReference:
+                        properties:
+                          keyPath:
+                            description: Name of the key for the given Secret reference
+                              where the content is stored. This value is mandatory.
+                            type: string
+                          name:
+                            description: name is unique within a namespace to reference
+                              a secret resource.
+                            type: string
+                          namespace:
+                            description: namespace defines the space within which
+                              the secret name must be unique.
+                            type: string
+                        required:
+                        - keyPath
+                        type: object
+                        x-kubernetes-map-type: atomic
+                    type: object
+                required:
+                - password
+                - username
+                type: object
+              driver:
+                description: The driver to use to connect to the shared datastore.
+                type: string
+              endpoints:
+                description: List of the endpoints to connect to the shared datastore.
+                  No need for protocol, just bare IP/FQDN and port.
+                items:
+                  type: string
+                type: array
+              tlsConfig:
+                description: Defines the TLS/SSL configuration required to connect
+                  to the data store in a secure way.
+                properties:
+                  certificateAuthority:
+                    description: Retrieve the Certificate Authority certificate and
+                      private key, such as bare content of the file, or a SecretReference.
+                      The key reference is required since etcd authentication is based
+                      on certificates, and Kamaji is responsible in creating this.
+                    properties:
+                      certificate:
+                        properties:
+                          content:
+                            description: Bare content of the file, base64 encoded.
+                              It has precedence over the SecretReference value.
+                            format: byte
+                            type: string
+                          secretReference:
+                            properties:
+                              keyPath:
+                                description: Name of the key for the given Secret
+                                  reference where the content is stored. This value
+                                  is mandatory.
+                                type: string
+                              name:
+                                description: name is unique within a namespace to
+                                  reference a secret resource.
+                                type: string
+                              namespace:
+                                description: namespace defines the space within which
+                                  the secret name must be unique.
+                                type: string
+                            required:
+                            - keyPath
+                            type: object
+                            x-kubernetes-map-type: atomic
+                        type: object
+                      privateKey:
+                        properties:
+                          content:
+                            description: Bare content of the file, base64 encoded.
+                              It has precedence over the SecretReference value.
+                            format: byte
+                            type: string
+                          secretReference:
+                            properties:
+                              keyPath:
+                                description: Name of the key for the given Secret
+                                  reference where the content is stored. This value
+                                  is mandatory.
+                                type: string
+                              name:
+                                description: name is unique within a namespace to
+                                  reference a secret resource.
+                                type: string
+                              namespace:
+                                description: namespace defines the space within which
+                                  the secret name must be unique.
+                                type: string
+                            required:
+                            - keyPath
+                            type: object
+                            x-kubernetes-map-type: atomic
+                        type: object
+                    required:
+                    - certificate
+                    type: object
+                  clientCertificate:
+                    description: Specifies the SSL/TLS key and private key pair used
+                      to connect to the data store.
+                    properties:
+                      certificate:
+                        properties:
+                          content:
+                            description: Bare content of the file, base64 encoded.
+                              It has precedence over the SecretReference value.
+                            format: byte
+                            type: string
+                          secretReference:
+                            properties:
+                              keyPath:
+                                description: Name of the key for the given Secret
+                                  reference where the content is stored. This value
+                                  is mandatory.
+                                type: string
+                              name:
+                                description: name is unique within a namespace to
+                                  reference a secret resource.
+                                type: string
+                              namespace:
+                                description: namespace defines the space within which
+                                  the secret name must be unique.
+                                type: string
+                            required:
+                            - keyPath
+                            type: object
+                            x-kubernetes-map-type: atomic
+                        type: object
+                      privateKey:
+                        properties:
+                          content:
+                            description: Bare content of the file, base64 encoded.
+                              It has precedence over the SecretReference value.
+                            format: byte
+                            type: string
+                          secretReference:
+                            properties:
+                              keyPath:
+                                description: Name of the key for the given Secret
+                                  reference where the content is stored. This value
+                                  is mandatory.
+                                type: string
+                              name:
+                                description: name is unique within a namespace to
+                                  reference a secret resource.
+                                type: string
+                              namespace:
+                                description: namespace defines the space within which
+                                  the secret name must be unique.
+                                type: string
+                            required:
+                            - keyPath
+                            type: object
+                            x-kubernetes-map-type: atomic
+                        type: object
+                    required:
+                    - certificate
+                    - privateKey
+                    type: object
+                required:
+                - certificateAuthority
+                - clientCertificate
+                type: object
+            required:
+            - driver
+            - endpoints
+            - tlsConfig
+            type: object
+          status:
+            description: DataStoreStatus defines the observed state of DataStore.
+            properties:
+              usedBy:
+                description: List of the Tenant Control Planes, namespaced named,
+                  using this data store.
+                items:
+                  type: string
+                type: array
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}

charts/kamaji/crds/tenantcontrolplane.yaml

@@ -1,10 +1,9 @@
-
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.6.1
+    controller-gen.kubebuilder.io/version: v0.9.2
   creationTimestamp: null
   name: tenantcontrolplanes.kamaji.clastix.io
 spec:
@@ -64,13 +63,27 @@ spec:
                 description: Addons contain which addons are enabled
                 properties:
                   coreDNS:
-                    description: AddonSpec defines the spec for every addon.
+                    description: Enables the DNS addon in the Tenant Cluster. The
+                      registry and the tag are configurable, the image is hard-coded
+                      to `coredns`.
+                    properties:
+                      imageRepository:
+                        description: ImageRepository sets the container registry to
+                          pull images from. if not set, the default ImageRepository
+                          will be used instead.
+                        type: string
+                      imageTag:
+                        description: ImageTag allows to specify a tag for the image.
+                          In case this value is set, kubeadm does not change automatically
+                          the version of the above components during upgrades.
+                        type: string
                     type: object
                   konnectivity:
-                    description: KonnectivitySpec defines the spec for Konnectivity.
+                    description: Enables the Konnectivity addon in the Tenant Cluster,
+                      required if the worker nodes are in a different network.
                     properties:
                       agentImage:
-                        default: us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-agent
+                        default: registry.k8s.io/kas-network-proxy/proxy-agent
                         description: AgentImage defines the container image for Konnectivity's
                           agent.
                         type: string
@@ -107,19 +120,32 @@ spec:
                             type: object
                         type: object
                       serverImage:
-                        default: us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-server
+                        default: registry.k8s.io/kas-network-proxy/proxy-server
                         description: ServerImage defines the container image for Konnectivity's
                           server.
                         type: string
                       version:
-                        default: v0.0.31
+                        default: v0.0.32
                         description: Version for Konnectivity server and agent.
                         type: string
                     required:
                     - proxyPort
                     type: object
                   kubeProxy:
-                    description: AddonSpec defines the spec for every addon.
+                    description: Enables the kube-proxy addon in the Tenant Cluster.
+                      The registry and the tag are configurable, the image is hard-coded
+                      to `kube-proxy`.
+                    properties:
+                      imageRepository:
+                        description: ImageRepository sets the container registry to
+                          pull images from. if not set, the default ImageRepository
+                          will be used instead.
+                        type: string
+                      imageTag:
+                        description: ImageTag allows to specify a tag for the image.
+                          In case this value is set, kubeadm does not change automatically
+                          the version of the above components during upgrades.
+                        type: string
                     type: object
                 type: object
               controlPlane:
@@ -263,6 +289,194 @@ spec:
                                 type: object
                             type: object
                         type: object
+                      topologySpreadConstraints:
+                        description: TopologySpreadConstraints describes how the Tenant
+                          Control Plane pods ought to spread across topology domains.
+                          Scheduler will schedule pods in a way which abides by the
+                          constraints. In case of nil underlying LabelSelector, the
+                          Kamaji one for the given Tenant Control Plane will be used.
+                          All topologySpreadConstraints are ANDed.
+                        items:
+                          description: TopologySpreadConstraint specifies how to spread
+                            matching pods among the given topology.
+                          properties:
+                            labelSelector:
+                              description: LabelSelector is used to find matching
+                                pods. Pods that match this label selector are counted
+                                to determine the number of pods in their corresponding
+                                topology domain.
+                              properties:
+                                matchExpressions:
+                                  description: matchExpressions is a list of label
+                                    selector requirements. The requirements are ANDed.
+                                  items:
+                                    description: A label selector requirement is a
+                                      selector that contains values, a key, and an
+                                      operator that relates the key and values.
+                                    properties:
+                                      key:
+                                        description: key is the label key that the
+                                          selector applies to.
+                                        type: string
+                                      operator:
+                                        description: operator represents a key's relationship
+                                          to a set of values. Valid operators are
+                                          In, NotIn, Exists and DoesNotExist.
+                                        type: string
+                                      values:
+                                        description: values is an array of string
+                                          values. If the operator is In or NotIn,
+                                          the values array must be non-empty. If the
+                                          operator is Exists or DoesNotExist, the
+                                          values array must be empty. This array is
+                                          replaced during a strategic merge patch.
+                                        items:
+                                          type: string
+                                        type: array
+                                    required:
+                                    - key
+                                    - operator
+                                    type: object
+                                  type: array
+                                matchLabels:
+                                  additionalProperties:
+                                    type: string
+                                  description: matchLabels is a map of {key,value}
+                                    pairs. A single {key,value} in the matchLabels
+                                    map is equivalent to an element of matchExpressions,
+                                    whose key field is "key", the operator is "In",
+                                    and the values array contains only "value". The
+                                    requirements are ANDed.
+                                  type: object
+                              type: object
+                              x-kubernetes-map-type: atomic
+                            matchLabelKeys:
+                              description: MatchLabelKeys is a set of pod label keys
+                                to select the pods over which spreading will be calculated.
+                                The keys are used to lookup values from the incoming
+                                pod labels, those key-value labels are ANDed with
+                                labelSelector to select the group of existing pods
+                                over which spreading will be calculated for the incoming
+                                pod. Keys that don't exist in the incoming pod labels
+                                will be ignored. A null or empty list means only match
+                                against labelSelector.
+                              items:
+                                type: string
+                              type: array
+                              x-kubernetes-list-type: atomic
+                            maxSkew:
+                              description: 'MaxSkew describes the degree to which
+                                pods may be unevenly distributed. When `whenUnsatisfiable=DoNotSchedule`,
+                                it is the maximum permitted difference between the
+                                number of matching pods in the target topology and
+                                the global minimum. The global minimum is the minimum
+                                number of matching pods in an eligible domain or zero
+                                if the number of eligible domains is less than MinDomains.
+                                For example, in a 3-zone cluster, MaxSkew is set to
+                                1, and pods with the same labelSelector spread as
+                                2/2/1: In this case, the global minimum is 1. | zone1
+                                | zone2 | zone3 | |  P P  |  P P  |   P   | - if MaxSkew
+                                is 1, incoming pod can only be scheduled to zone3
+                                to become 2/2/2; scheduling it onto zone1(zone2) would
+                                make the ActualSkew(3-1) on zone1(zone2) violate MaxSkew(1).
+                                - if MaxSkew is 2, incoming pod can be scheduled onto
+                                any zone. When `whenUnsatisfiable=ScheduleAnyway`,
+                                it is used to give higher precedence to topologies
+                                that satisfy it. It''s a required field. Default value
+                                is 1 and 0 is not allowed.'
+                              format: int32
+                              type: integer
+                            minDomains:
+                              description: "MinDomains indicates a minimum number
+                                of eligible domains. When the number of eligible domains
+                                with matching topology keys is less than minDomains,
+                                Pod Topology Spread treats \"global minimum\" as 0,
+                                and then the calculation of Skew is performed. And
+                                when the number of eligible domains with matching
+                                topology keys equals or greater than minDomains, this
+                                value has no effect on scheduling. As a result, when
+                                the number of eligible domains is less than minDomains,
+                                scheduler won't schedule more than maxSkew Pods to
+                                those domains. If value is nil, the constraint behaves
+                                as if MinDomains is equal to 1. Valid values are integers
+                                greater than 0. When value is not nil, WhenUnsatisfiable
+                                must be DoNotSchedule. \n For example, in a 3-zone
+                                cluster, MaxSkew is set to 2, MinDomains is set to
+                                5 and pods with the same labelSelector spread as 2/2/2:
+                                | zone1 | zone2 | zone3 | |  P P  |  P P  |  P P  |
+                                The number of domains is less than 5(MinDomains),
+                                so \"global minimum\" is treated as 0. In this situation,
+                                new pod with the same labelSelector cannot be scheduled,
+                                because computed skew will be 3(3 - 0) if new Pod
+                                is scheduled to any of the three zones, it will violate
+                                MaxSkew. \n This is a beta field and requires the
+                                MinDomainsInPodTopologySpread feature gate to be enabled
+                                (enabled by default)."
+                              format: int32
+                              type: integer
+                            nodeAffinityPolicy:
+                              description: "NodeAffinityPolicy indicates how we will
+                                treat Pod's nodeAffinity/nodeSelector when calculating
+                                pod topology spread skew. Options are: - Honor: only
+                                nodes matching nodeAffinity/nodeSelector are included
+                                in the calculations. - Ignore: nodeAffinity/nodeSelector
+                                are ignored. All nodes are included in the calculations.
+                                \n If this value is nil, the behavior is equivalent
+                                to the Honor policy. This is a alpha-level feature
+                                enabled by the NodeInclusionPolicyInPodTopologySpread
+                                feature flag."
+                              type: string
+                            nodeTaintsPolicy:
+                              description: "NodeTaintsPolicy indicates how we will
+                                treat node taints when calculating pod topology spread
+                                skew. Options are: - Honor: nodes without taints,
+                                along with tainted nodes for which the incoming pod
+                                has a toleration, are included. - Ignore: node taints
+                                are ignored. All nodes are included. \n If this value
+                                is nil, the behavior is equivalent to the Ignore policy.
+                                This is a alpha-level feature enabled by the NodeInclusionPolicyInPodTopologySpread
+                                feature flag."
+                              type: string
+                            topologyKey:
+                              description: TopologyKey is the key of node labels.
+                                Nodes that have a label with this key and identical
+                                values are considered to be in the same topology.
+                                We consider each <key, value> as a "bucket", and try
+                                to put balanced number of pods into each bucket. We
+                                define a domain as a particular instance of a topology.
+                                Also, we define an eligible domain as a domain whose
+                                nodes meet the requirements of nodeAffinityPolicy
+                                and nodeTaintsPolicy. e.g. If TopologyKey is "kubernetes.io/hostname",
+                                each Node is a domain of that topology. And, if TopologyKey
+                                is "topology.kubernetes.io/zone", each zone is a domain
+                                of that topology. It's a required field.
+                              type: string
+                            whenUnsatisfiable:
+                              description: 'WhenUnsatisfiable indicates how to deal
+                                with a pod if it doesn''t satisfy the spread constraint.
+                                - DoNotSchedule (default) tells the scheduler not
+                                to schedule it. - ScheduleAnyway tells the scheduler
+                                to schedule the pod in any location, but giving higher
+                                precedence to topologies that would help reduce the
+                                skew. A constraint is considered "Unsatisfiable" for
+                                an incoming pod if and only if every possible node
+                                assignment for that pod would violate "MaxSkew" on
+                                some topology. For example, in a 3-zone cluster, MaxSkew
+                                is set to 1, and pods with the same labelSelector
+                                spread as 3/1/1: | zone1 | zone2 | zone3 | | P P P
+                                |   P   |   P   | If WhenUnsatisfiable is set to DoNotSchedule,
+                                incoming pod can only be scheduled to zone2(zone3)
+                                to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3)
+                                satisfies MaxSkew(1). In other words, the cluster
+                                can still be imbalanced, but scheduler won''t make
+                                it *more* imbalanced. It''s a required field.'
+                              type: string
+                          required:
+                          - maxSkew
+                          - topologyKey
+                          - whenUnsatisfiable
+                          type: object
+                        type: array
                     type: object
                   ingress:
                     description: Defining the options for an Optional Ingress which
@@ -282,8 +496,6 @@ spec:
                               type: string
                             type: object
                         type: object
-                      enabled:
-                        type: boolean
                       hostname:
                         description: Hostname is an optional field which will be used
                           as Ingress's Host. If it is not defined, Ingress's host
@@ -292,8 +504,6 @@ spec:
                         type: string
                       ingressClassName:
                         type: string
-                    required:
-                    - enabled
                     type: object
                   service:
                     description: Defining the options for the Tenant Control Plane
@@ -327,6 +537,14 @@ spec:
                 required:
                 - service
                 type: object
+              dataStore:
+                description: DataStore allows to specify a DataStore that should be
+                  used to store the Kubernetes data for the given Tenant Control Plane.
+                  This parameter is optional and acts as an override over the default
+                  one which is used by the Kamaji Operator. Migration from a different
+                  DataStore to another one is not yet supported and the reconciliation
+                  will be blocked.
+                type: string
               kubernetes:
                 description: Kubernetes specification for tenant control plane
                 properties:
@@ -479,6 +697,8 @@ spec:
                     properties:
                       agent:
                         properties:
+                          checksum:
+                            type: string
                           lastUpdate:
                             description: Last time when k8s object was updated
                             format: date-time
@@ -487,23 +707,22 @@ spec:
                             type: string
                           namespace:
                             type: string
-                          resourceVersion:
-                            description: Resource version of k8s object
-                            type: string
                         type: object
                       certificate:
                         description: CertificatePrivateKeyPairStatus defines the status.
                         properties:
+                          checksum:
+                            type: string
                           lastUpdate:
                             format: date-time
                             type: string
-                          resourceVersion:
-                            type: string
                           secretName:
                             type: string
                         type: object
                       clusterrolebinding:
                         properties:
+                          checksum:
+                            type: string
                           lastUpdate:
                             description: Last time when k8s object was updated
                             format: date-time
@@ -512,12 +731,14 @@ spec:
                             type: string
                           namespace:
                             type: string
-                          resourceVersion:
-                            description: Resource version of k8s object
+                        type: object
+                      configMap:
+                        properties:
+                          checksum:
+                            type: string
+                          name:
                             type: string
                         type: object
-                      egressSelectorConfiguration:
-                        type: string
                       enabled:
                         type: boolean
                       kubeconfig:
@@ -534,6 +755,8 @@ spec:
                         type: object
                       sa:
                         properties:
+                          checksum:
+                            type: string
                           lastUpdate:
                             description: Last time when k8s object was updated
                             format: date-time
@@ -542,9 +765,6 @@ spec:
                             type: string
                           namespace:
                             type: string
-                          resourceVersion:
-                            description: Resource version of k8s object
-                            type: string
                         type: object
                       service:
                         description: KubernetesServiceStatus defines the status for
@@ -556,15 +776,15 @@ spec:
                               description: "Condition contains details for one aspect
                                 of the current state of this API Resource. --- This
                                 struct is intended for direct use as an array at the
-                                field path .status.conditions.  For example, type
-                                FooStatus struct{     // Represents the observations
-                                of a foo's current state.     // Known .status.conditions.type
+                                field path .status.conditions.  For example, \n type
+                                FooStatus struct{ // Represents the observations of
+                                a foo's current state. // Known .status.conditions.type
                                 are: \"Available\", \"Progressing\", and \"Degraded\"
-                                \    // +patchMergeKey=type     // +patchStrategy=merge
-                                \    // +listType=map     // +listMapKey=type     Conditions
-                                []metav1.Condition `json:\"conditions,omitempty\"
-                                patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"`
-                                \n     // other fields }"
+                                // +patchMergeKey=type // +patchStrategy=merge //
+                                +listType=map // +listMapKey=type Conditions []metav1.Condition
+                                `json:\"conditions,omitempty\" patchStrategy:\"merge\"
+                                patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"`
+                                \n // other fields }"
                               properties:
                                 lastTransitionTime:
                                   description: lastTransitionTime is the last time
@@ -668,11 +888,10 @@ spec:
                                               the error shall comply with the following
                                               rules: - built-in error values shall
                                               be specified in this file and those
-                                              shall use   CamelCase names - cloud
-                                              provider specific error values must
-                                              have names that comply with the   format
-                                              foo.example.com/CamelCase. --- The regex
-                                              it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)'
+                                              shall use CamelCase names - cloud provider
+                                              specific error values must have names
+                                              that comply with the format foo.example.com/CamelCase.
+                                              --- The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)'
                                             maxLength: 316
                                             pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
                                             type: string
@@ -738,33 +957,33 @@ spec:
                   apiServer:
                     description: CertificatePrivateKeyPairStatus defines the status.
                     properties:
+                      checksum:
+                        type: string
                       lastUpdate:
                         format: date-time
                         type: string
-                      resourceVersion:
-                        type: string
                       secretName:
                         type: string
                     type: object
                   apiServerKubeletClient:
                     description: CertificatePrivateKeyPairStatus defines the status.
                     properties:
+                      checksum:
+                        type: string
                       lastUpdate:
                         format: date-time
                         type: string
-                      resourceVersion:
-                        type: string
                       secretName:
                         type: string
                     type: object
                   ca:
                     description: CertificatePrivateKeyPairStatus defines the status.
                     properties:
+                      checksum:
+                        type: string
                       lastUpdate:
                         format: date-time
                         type: string
-                      resourceVersion:
-                        type: string
                       secretName:
                         type: string
                     type: object
@@ -773,9 +992,11 @@ spec:
                       of ETCD Certificate for API server.
                     properties:
                       apiServer:
-                        description: ETCDCertificateStatus defines the observed state
-                          of ETCD Certificate for API server.
+                        description: APIServerCertificatesStatus defines the observed
+                          state of ETCD Certificate for API server.
                         properties:
+                          checksum:
+                            type: string
                           lastUpdate:
                             format: date-time
                             type: string
@@ -786,6 +1007,8 @@ spec:
                         description: ETCDCertificateStatus defines the observed state
                           of ETCD Certificate for API server.
                         properties:
+                          checksum:
+                            type: string
                           lastUpdate:
                             format: date-time
                             type: string
@@ -796,28 +1019,30 @@ spec:
                   frontProxyCA:
                     description: CertificatePrivateKeyPairStatus defines the status.
                     properties:
+                      checksum:
+                        type: string
                       lastUpdate:
                         format: date-time
                         type: string
-                      resourceVersion:
-                        type: string
                       secretName:
                         type: string
                     type: object
                   frontProxyClient:
                     description: CertificatePrivateKeyPairStatus defines the status.
                     properties:
+                      checksum:
+                        type: string
                       lastUpdate:
                         format: date-time
                         type: string
-                      resourceVersion:
-                        type: string
                       secretName:
                         type: string
                     type: object
                   sa:
                     description: PublicKeyPrivateKeyPairStatus defines the status.
                     properties:
+                      checksum:
+                        type: string
                       lastUpdate:
                         format: date-time
                         type: string
@@ -1002,6 +1227,10 @@ spec:
                           by this deployment (their labels match the selector).
                         format: int32
                         type: integer
+                      selector:
+                        description: Selector is the label selector used to group
+                          the Tenant Control Plane Pods used by the scale subresource.
+                        type: string
                       unavailableReplicas:
                         description: Total number of unavailable pods targeted by
                           this deployment. This is the total number of pods that are
@@ -1018,6 +1247,7 @@ spec:
                     required:
                     - name
                     - namespace
+                    - selector
                     type: object
                   ingress:
                     description: KubernetesIngressStatus defines the status for the
@@ -1056,9 +1286,9 @@ spec:
                                           with the service port The format of the
                                           error shall comply with the following rules:
                                           - built-in error values shall be specified
-                                          in this file and those shall use   CamelCase
+                                          in this file and those shall use CamelCase
                                           names - cloud provider specific error values
-                                          must have names that comply with the   format
+                                          must have names that comply with the format
                                           foo.example.com/CamelCase. --- The regex
                                           it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)'
                                         maxLength: 316
@@ -1107,14 +1337,14 @@ spec:
                           description: "Condition contains details for one aspect
                             of the current state of this API Resource. --- This struct
                             is intended for direct use as an array at the field path
-                            .status.conditions.  For example, type FooStatus struct{
-                            \    // Represents the observations of a foo's current
-                            state.     // Known .status.conditions.type are: \"Available\",
-                            \"Progressing\", and \"Degraded\"     // +patchMergeKey=type
-                            \    // +patchStrategy=merge     // +listType=map     //
-                            +listMapKey=type     Conditions []metav1.Condition `json:\"conditions,omitempty\"
-                            patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"`
-                            \n     // other fields }"
+                            .status.conditions.  For example, \n type FooStatus struct{
+                            // Represents the observations of a foo's current state.
+                            // Known .status.conditions.type are: \"Available\", \"Progressing\",
+                            and \"Degraded\" // +patchMergeKey=type // +patchStrategy=merge
+                            // +listType=map // +listMapKey=type Conditions []metav1.Condition
+                            `json:\"conditions,omitempty\" patchStrategy:\"merge\"
+                            patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"`
+                            \n // other fields }"
                           properties:
                             lastTransitionTime:
                               description: lastTransitionTime is the last time the
@@ -1213,9 +1443,9 @@ spec:
                                           with the service port The format of the
                                           error shall comply with the following rules:
                                           - built-in error values shall be specified
-                                          in this file and those shall use   CamelCase
+                                          in this file and those shall use CamelCase
                                           names - cloud provider specific error values
-                                          must have names that comply with the   format
+                                          must have names that comply with the format
                                           foo.example.com/CamelCase. --- The regex
                                           it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)'
                                         maxLength: 316
@@ -1277,84 +1507,44 @@ spec:
                         description: Version is the running Kubernetes version of
                           the Tenant Control Plane.
                         type: string
-                    required:
-                    - status
                     type: object
                 type: object
               storage:
                 description: Storage Status contains information about Kubernetes
                   storage system
                 properties:
-                  etcd:
-                    description: ETCDStatus defines the observed state of ETCDStatus.
+                  certificate:
                     properties:
-                      role:
-                        properties:
-                          exists:
-                            type: boolean
-                          name:
-                            type: string
-                          permissions:
-                            items:
-                              properties:
-                                key:
-                                  type: string
-                                rangeEnd:
-                                  type: string
-                                type:
-                                  type: integer
-                              type: object
-                            type: array
-                        required:
-                        - exists
-                        - name
-                        type: object
-                      user:
-                        properties:
-                          exists:
-                            type: boolean
-                          name:
-                            type: string
-                          roles:
-                            items:
-                              type: string
-                            type: array
-                        required:
-                        - exists
-                        - name
-                        type: object
+                      checksum:
+                        type: string
+                      lastUpdate:
+                        format: date-time
+                        type: string
+                      secretName:
+                        type: string
                     type: object
-                  kineMySQL:
+                  config:
                     properties:
-                      certificate:
-                        properties:
-                          lastUpdate:
-                            format: date-time
-                            type: string
-                          resourceVersion:
-                            type: string
-                          secretName:
-                            type: string
-                        type: object
-                      config:
-                        properties:
-                          resourceVersion:
-                            type: string
-                          secretName:
-                            type: string
-                        type: object
-                      setup:
-                        properties:
-                          lastUpdate:
-                            format: date-time
-                            type: string
-                          schema:
-                            type: string
-                          sqlConfigResourceVersion:
-                            type: string
-                          user:
-                            type: string
-                        type: object
+                      checksum:
+                        type: string
+                      secretName:
+                        type: string
+                    type: object
+                  dataStoreName:
+                    type: string
+                  driver:
+                    type: string
+                  setup:
+                    properties:
+                      checksum:
+                        type: string
+                      lastUpdate:
+                        format: date-time
+                        type: string
+                      schema:
+                        type: string
+                      user:
+                        type: string
                     type: object
                 type: object
             type: object
@@ -1362,10 +1552,8 @@ spec:
     served: true
     storage: true
     subresources:
+      scale:
+        labelSelectorPath: .status.kubernetesResources.deployment.selector
+        specReplicasPath: .spec.controlPlane.deployment.replicas
+        statusReplicasPath: .status.kubernetesResources.deployment.replicas
       status: {}
-status:
-  acceptedNames:
-    kind: ""
-    plural: ""
-  conditions: []
-  storedVersions: []

charts/kamaji/templates/_helpers_datastore.tpl

@@ -0,0 +1,90 @@
+{{/*
+Create a default fully qualified datastore name.
+*/}}
+{{- define "datastore.fullname" -}}
+{{- default "default" .Values.datastore.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "datastore.labels" -}}
+kamaji.clastix.io/datastore: {{ .Values.datastore.driver }}
+helm.sh/chart: {{ include "kamaji.chart" . }}
+{{ include "kamaji.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Datastore endpoints, in case of ETCD, retrieving the one provided by the chart.
+*/}}
+{{- define "datastore.endpoints" -}}
+{{- if eq .Values.datastore.driver "etcd" }}
+{{ include "etcd.endpoints" . }}
+{{- else }}
+{{ .Values.datastore.endpoints }}
+{{- end }}
+{{- end }}
+
+{{/*
+The Certificate Authority section for the DataSource object.
+*/}}
+{{- define "datastore.certificateAuthority" -}}
+{{- if eq .Values.datastore.driver "etcd" }}
+certificate:
+  secretReference:
+    name: {{ include "etcd.caSecretName" . }}
+    namespace: {{ include "etcd.caSecretNamespace" . }}
+    keyPath: ca.crt
+privateKey:
+  secretReference:
+    name: {{ include "etcd.caSecretName" . }}
+    namespace: {{ include "etcd.caSecretNamespace" . }}
+    keyPath: ca.key
+{{- else }}
+certificate:
+  secretReference:
+    name: {{ .Values.datastore.tlsConfig.certificateAuthority.certificate.name }}
+    namespace: {{ .Values.datastore.tlsConfig.certificateAuthority.certificate.namespace }}
+    keyPath: {{ .Values.datastore.tlsConfig.certificateAuthority.certificate.keyPath }}
+{{- if .Values.datastore.tlsConfig.certificateAuthority.privateKey.name }}
+privateKey:
+  secretReference:
+    name: {{ .Values.datastore.tlsConfig.certificateAuthority.privateKey.name }}
+    namespace: {{ .Values.datastore.tlsConfig.certificateAuthority.privateKey.namespace }}
+    keyPath: {{ .Values.datastore.tlsConfig.certificateAuthority.privateKey.keyPath }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+The Client Certificate section for the DataSource object.
+*/}}
+{{- define "datastore.clientCertificate" -}}
+{{- if eq .Values.datastore.driver "etcd" }}
+certificate:
+    secretReference:
+      name: {{ include "etcd.clientSecretName" . }}
+      namespace: {{ include "etcd.clientSecretNamespace" . }}
+      keyPath: tls.crt
+privateKey:
+    secretReference:
+      name: {{ include "etcd.clientSecretName" . }}
+      namespace: {{ include "etcd.clientSecretNamespace" . }}
+      keyPath: tls.key
+{{- else }}
+certificate:
+  secretReference:
+    name: {{ .Values.datastore.tlsConfig.clientCertificate.certificate.name }}
+    namespace: {{ .Values.datastore.tlsConfig.clientCertificate.certificate.namespace }}
+    keyPath: {{ .Values.datastore.tlsConfig.clientCertificate.certificate.keyPath }}
+privateKey:
+  secretReference:
+    name: {{ .Values.datastore.tlsConfig.clientCertificate.privateKey.name }}
+    namespace: {{ .Values.datastore.tlsConfig.clientCertificate.privateKey.namespace }}
+    keyPath: {{ .Values.datastore.tlsConfig.clientCertificate.privateKey.keyPath }}
+{{- end }}
+{{- end }}

charts/kamaji/templates/_helpers_etcd.tpl

@@ -93,19 +93,41 @@ Namespace of the etcd root-client secret.
 {{- end }}
 
 {{/*
-List the declared etcd endpoints, using the overrides in case of unmanaged etcd.
+Comma separated list of etcd endpoints, using the overrides in case of unmanaged etcd.
 */}}
 {{- define "etcd.endpoints" }}
+{{- $list := list -}}
 {{- if .Values.etcd.deploy }}
-{{- range $count := until 3 -}}
-    {{- printf "https://%s-%d.%s.%s.svc.cluster.local:2379" "etcd" $count ( include "etcd.serviceName" . ) $.Release.Namespace -}}
-    {{- if lt $count  ( sub 3 1 ) -}}
-      {{- printf "," -}}
+    {{- range $count := until 3 -}}
+        {{- $list = append $list (printf "%s-%d.%s.%s.svc.cluster.local:%d" "etcd" $count ( include "etcd.serviceName" . ) $.Release.Namespace (int $.Values.etcd.port) ) -}}
+    {{- end }}
+{{- else if .Values.etcd.overrides.endpoints }}
+    {{- range $v := .Values.etcd.overrides.endpoints -}}
+        {{- $list = append $list (printf "%s:%d" $v (int $.Values.etcd.port) ) -}}
     {{- end -}}
+{{- else if not .Values.etcd.overrides.endpoints }}
+    {{- fail "A valid .Values.etcd.overrides.endpoints required!" }}
 {{- end }}
-{{- else }}
-{{- required "A valid .Values.etcd.overrides.endpoints required!" .Values.etcd.overrides.endpoints }}
+{{- $list | toYaml }}
 {{- end }}
+
+{{/*
+Key-value of the etcd peers, using the overrides in case of unmanaged etcd.
+*/}}
+{{- define "etcd.initialCluster" }}
+{{- $list := list -}}
+{{- if .Values.etcd.deploy }}
+    {{- range $i, $count := until 3 -}}
+        {{- $list = append $list ( printf "etcd-%d=https://%s-%d.%s.%s.svc.cluster.local:%d" $i "etcd" $count ( include "etcd.serviceName" . ) $.Release.Namespace (int $.Values.etcd.peerApiPort) ) -}}
+    {{- end }}
+{{- else if .Values.etcd.overrides.endpoints }}
+    {{- range $k, $v := .Values.etcd.overrides.endpoints -}}
+        {{- $list = append $list ( printf "%s=%s:%d" $k $v (int $.Values.etcd.peerApiPort) ) -}}
+    {{- end -}}
+{{- else if not .Values.etcd.overrides.endpoints }}
+    {{- fail "A valid .Values.etcd.overrides.endpoints required!" }}
+{{- end }}
+{{- join "," $list -}}
 {{- end }}
 
 {{/*

charts/kamaji/templates/controller.yaml

@@ -40,16 +40,11 @@ spec:
           protocol: TCP
       - args:
         - --config-file={{ .Values.configPath }}
-        - --etcd-ca-secret-name={{ include "etcd.caSecretName" . }}
-        - --etcd-ca-secret-namespace={{ include "etcd.caSecretNamespace" . }}
-        - --etcd-client-secret-name={{ include "etcd.clientSecretName" . }}
-        - --etcd-client-secret-namespace={{ include "etcd.clientSecretNamespace" . }}
-        - --etcd-compaction-interval={{ .Values.etcd.compactionInterval }}
-        - --etcd-endpoints={{ include "etcd.endpoints" . }}
         - --health-probe-bind-address={{ .Values.healthProbeBindAddress }}
         - --leader-elect
         - --metrics-bind-address={{ .Values.metricsBindAddress }}
         - --tmp-directory={{ .Values.temporaryDirectoryPath }}
+        - --datastore={{ include "datastore.fullname" . }}
         {{- if .Values.loggingDevel.enable }}
         - --zap-devel
         {{- end }}

charts/kamaji/templates/datastore.yaml

@@ -0,0 +1,19 @@
+apiVersion: kamaji.clastix.io/v1alpha1
+kind: DataStore
+metadata:
+  name: {{ include "datastore.fullname" . }}
+  labels:
+    {{- include "datastore.labels" . | nindent 4 }}
+spec:
+  driver: {{ .Values.datastore.driver }}
+  endpoints:
+    {{- include "datastore.endpoints" . | indent 4 }}
+{{- if (and .Values.datastore.basicAuth.usernameSecret.name .Values.datastore.basicAuth.passwordSecret.name) }}
+  basicAuth:
+    {{- .Values.datastore.basicAuth | toYaml | nindent 4 }}
+{{- end }}
+  tlsConfig:
+    certificateAuthority:
+      {{- include "datastore.certificateAuthority" . | indent 6 }}
+    clientCertificate:
+      {{- include "datastore.clientCertificate" . | indent 6 }}

charts/kamaji/templates/etcd_job_postdelete.yaml

@@ -7,7 +7,7 @@ metadata:
   annotations:
     "helm.sh/hook": pre-delete
     "helm.sh/hook-weight": "-5"
-    "helm.sh/hook-delete-policy": hook-succeeded
+    "helm.sh/hook-delete-policy": "hook-succeeded,hook-failed"
   name: "{{ .Release.Name }}-etcd-teardown"
   namespace: {{ .Release.Namespace }}
 spec:

charts/kamaji/templates/etcd_job_postinstall.yaml

@@ -7,7 +7,7 @@ metadata:
   annotations:
     "helm.sh/hook": post-install
     "helm.sh/hook-weight": "-5"
-    "helm.sh/hook-delete-policy": hook-succeeded
+    "helm.sh/hook-delete-policy": "hook-succeeded,hook-failed"
   name: "{{ .Release.Name }}-etcd-setup"
   namespace: {{ .Release.Namespace }}
 spec:
@@ -43,7 +43,7 @@ spec:
               kubectl --namespace={{ .Release.Namespace }} delete secret --ignore-not-found=true {{ include "etcd.caSecretName" . }} {{ include "etcd.clientSecretName" . }} &&
               kubectl --namespace={{ .Release.Namespace }} create secret generic {{ include "etcd.caSecretName" . }} --from-file=/certs/ca.crt --from-file=/certs/ca.key --from-file=/certs/peer-key.pem --from-file=/certs/peer.pem --from-file=/certs/server-key.pem --from-file=/certs/server.pem &&
               kubectl --namespace={{ .Release.Namespace }} create secret tls {{ include "etcd.clientSecretName" . }} --key=/certs/root-client-key.pem --cert=/certs/root-client.pem &&
-              kubectl --namespace={{ .Release.Namespace }} rollout status sts/etcd --timeout=120s
+              kubectl --namespace={{ .Release.Namespace }} rollout status sts/etcd --timeout=300s
           volumeMounts:
             - mountPath: /certs
               name: certs

charts/kamaji/templates/etcd_service.yaml

@@ -9,9 +9,9 @@ metadata:
 spec:
   clusterIP: None
   ports:
-    - port: 2379
+    - port: {{ .Values.etcd.port }}
       name: client
-    - port: 2380
+    - port: {{ .Values.etcd.peerApiPort }}
       name: peer
   selector:
     {{- include "etcd.selectorLabels" . | nindent 4 }}

charts/kamaji/templates/etcd_sts.yaml

@@ -24,7 +24,8 @@ spec:
             secretName: {{ include "etcd.caSecretName" . }}
       containers:
         - name: etcd
-          image: quay.io/coreos/etcd:v3.5.1
+          image: {{ .Values.etcd.image.repository }}:{{ .Values.etcd.image.tag | default "v3.5.4" }}
+          imagePullPolicy: {{ .Values.etcd.image.pullPolicy }}
           ports:
             - containerPort: 2379
               name: client
@@ -40,22 +41,25 @@ spec:
             - --data-dir=/var/run/etcd
             - --name=$(POD_NAME)
             - --initial-cluster-state=new
-            - --initial-cluster=etcd-0=https://etcd-0.etcd.$(POD_NAMESPACE).svc.cluster.local:2380,etcd-1=https://etcd-1.etcd.$(POD_NAMESPACE).svc.cluster.local:2380,etcd-2=https://etcd-2.etcd.$(POD_NAMESPACE).svc.cluster.local:2380
+            - --initial-cluster={{ include "etcd.initialCluster" . }}
             - --initial-advertise-peer-urls=https://$(POD_NAME).etcd.$(POD_NAMESPACE).svc.cluster.local:2380
+            - --advertise-client-urls=https://$(POD_NAME).etcd.$(POD_NAMESPACE).svc.cluster.local:2379
             - --initial-cluster-token=kamaji
             - --listen-client-urls=https://0.0.0.0:2379
-            - --advertise-client-urls={{ include "etcd.endpoints" . }}
+            - --listen-metrics-urls=http://0.0.0.0:2381
+            - --listen-peer-urls=https://0.0.0.0:2380
             - --client-cert-auth=true
+            - --peer-client-cert-auth=true
             - --trusted-ca-file=/etc/etcd/pki/ca.crt
             - --cert-file=/etc/etcd/pki/server.pem
             - --key-file=/etc/etcd/pki/server-key.pem
-            - --listen-peer-urls=https://0.0.0.0:2380
-            - --peer-client-cert-auth=true
             - --peer-trusted-ca-file=/etc/etcd/pki/ca.crt
             - --peer-cert-file=/etc/etcd/pki/peer.pem
             - --peer-key-file=/etc/etcd/pki/peer-key.pem
             - --auto-compaction-mode=periodic
             - --auto-compaction-retention=5m
+            - --snapshot-count=10000
+            - --quota-backend-bytes=8589934592
             - --v=8
           env:
             - name: POD_NAME
@@ -66,32 +70,24 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.namespace
+          {{- with .Values.etcd.livenessProbe }}
           livenessProbe:
-            failureThreshold: 8
-            httpGet:
-              host: 127.0.0.1
-              path: /health
-              port: 2381
-              scheme: HTTP
-            initialDelaySeconds: 10
-            periodSeconds: 10
-            timeoutSeconds: 15
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.etcd.startupProbe }}
           startupProbe:
-            failureThreshold: 24
-            httpGet:
-              host: 127.0.0.1
-              path: /health
-              port: 2381
-              scheme: HTTP
-            initialDelaySeconds: 10
-            periodSeconds: 10
-            timeoutSeconds: 15
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
   volumeClaimTemplates:
-    - metadata:
-        name: data
-      spec:
-        accessModes: ["ReadWriteOnce"]
-        resources:
-          requests:
-            storage: 8Gi
+  - metadata:
+      name: data
+    spec:
+      storageClassName: {{ .Values.etcd.persistence.storageClassName }}
+      accessModes:
+      {{- range .Values.etcd.persistence.accessModes }}
+      - {{ . | quote }}
+      {{- end }}
+      resources:
+        requests:
+          storage: {{ .Values.etcd.persistence.size }}
 {{- end }}

charts/kamaji/templates/rbac.yaml

@@ -102,6 +102,32 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+    - kamaji.clastix.io
+  resources:
+    - datastores
+  verbs:
+    - create
+    - delete
+    - get
+    - list
+    - patch
+    - update
+    - watch
+- apiGroups:
+    - kamaji.clastix.io
+  resources:
+    - datastores/finalizers
+  verbs:
+    - update
+- apiGroups:
+    - kamaji.clastix.io
+  resources:
+    - datastores/status
+  verbs:
+    - get
+    - patch
+    - update
 - apiGroups:
   - kamaji.clastix.io
   resources:

charts/kamaji/values.yaml

@@ -0,0 +1,210 @@
+# Default values for kamaji.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+# -- The number of the pod replicas for the Kamaji controller.
+replicaCount: 1
+
+image:
+  # -- The container image of the Kamaji controller.
+  repository: clastix/kamaji
+  pullPolicy: Always
+  # -- Overrides the image tag whose default is the chart appVersion.
+  tag:
+
+# -- A list of extra arguments to add to the kamaji controller default ones
+extraArgs: []
+
+# -- Configuration file path alternative. (default "./kamaji.yaml")
+configPath: "./kamaji.yaml"
+
+etcd:
+  # -- Install an etcd with enabled multi-tenancy along with Kamaji
+  deploy: true
+
+  # -- The peer API port which servers are listening to.
+  peerApiPort: 2380
+
+  # -- The client request port.
+  port: 2379
+
+  # -- Install specific etcd image
+  image:
+    repository: quay.io/coreos/etcd
+    tag: "v3.5.4"
+    pullPolicy: IfNotPresent
+
+  # -- The livenessProbe for the etcd container
+  livenessProbe:
+    failureThreshold: 8
+    httpGet:
+      path: /health?serializable=true
+      port: 2381
+      scheme: HTTP
+    initialDelaySeconds: 10
+    periodSeconds: 10
+    timeoutSeconds: 15
+
+  serviceAccount:
+    # -- Create a ServiceAccount, required to install and provision the etcd backing storage (default: true)
+    create: true
+    # -- Define the ServiceAccount name to use during the setup and provision of the etcd backing storage (default: "")
+    name: ""
+  persistence:
+    size: 10Gi
+    storageClass: ""
+    accessModes:
+    - ReadWriteOnce
+
+  overrides:
+    caSecret:
+      # -- Name of the secret which contains CA's certificate and private key. (default: "etcd-certs")
+      name: etcd-certs
+      # -- Namespace of the secret which contains CA's certificate and private key. (default: "kamaji-system")
+      namespace: kamaji-system
+    clientSecret:
+      # -- Name of the secret which contains ETCD client certificates. (default: "root-client-certs")
+      name: root-client-certs
+      # -- Name of the namespace where the secret which contains ETCD client certificates is. (default: "kamaji-system")
+      namespace: kamaji-system
+    # -- (map) Dictionary of the endpoints for the etcd cluster's members, key is the name of the etcd server. Don't define the protocol (TLS is automatically inflected), or any port, inflected from .etcd.peerApiPort value.
+    endpoints:
+      etcd-0: etcd-0.etcd.kamaji-system.svc.cluster.local
+      etcd-1: etcd-1.etcd.kamaji-system.svc.cluster.local
+      etcd-2: etcd-2.etcd.kamaji-system.svc.cluster.local
+  # -- ETCD Compaction interval (e.g. "5m0s"). (default: "0" (disabled))
+  compactionInterval: 0
+
+# -- The address the probe endpoint binds to. (default ":8081")
+healthProbeBindAddress: ":8081"
+
+# -- The livenessProbe for the controller container
+livenessProbe:
+  httpGet:
+    path: /healthz
+    port: healthcheck
+  initialDelaySeconds: 15
+  periodSeconds: 20
+
+# -- The readinessProbe for the controller container
+readinessProbe:
+  httpGet:
+    path: /readyz
+    port: healthcheck
+  initialDelaySeconds: 5
+  periodSeconds: 10
+
+# -- (string) The address the metric endpoint binds to. (default ":8080")
+metricsBindAddress: ":8080"
+
+imagePullSecrets: []
+nameOverride: ""
+fullnameOverride: ""
+
+serviceAccount:
+  # Specifies whether a service account should be created
+  create: true
+  # Annotations to add to the service account
+  annotations: {}
+  # The name of the service account to use.
+  # If not set and create is true, a name is generated using the fullname template
+  name: kamaji-controller-manager
+
+# -- The annotations to apply to the Kamaji controller pods.
+podAnnotations: {}
+
+# -- The securityContext to apply to the Kamaji controller pods.
+podSecurityContext:
+  runAsNonRoot: true
+
+# -- The securityContext to apply to the Kamaji controller container only. It does not apply to the Kamaji RBAC proxy container.
+securityContext:
+  allowPrivilegeEscalation: false
+  # capabilities:
+  #   drop:
+  #   - ALL
+  # readOnlyRootFilesystem: true
+  # runAsNonRoot: true
+  # runAsUser: 1000
+
+service:
+  type: ClusterIP
+  port: 8443
+
+resources:
+  limits:
+    cpu: 200m
+    memory: 100Mi
+  requests:
+    cpu: 100m
+    memory: 20Mi
+
+# -- Kubernetes node selector rules to schedule Kamaji controller
+nodeSelector: {}
+
+# -- Kubernetes node taints that the Kamaji controller pods would tolerate
+tolerations: []
+
+# -- Kubernetes affinity rules to apply to Kamaji controller pods
+affinity: {}
+
+# -- Directory which will be used to work with temporary files. (default "/tmp/kamaji")
+temporaryDirectoryPath: "/tmp/kamaji"
+
+loggingDevel:
+  # -- (string) Development Mode defaults(encoder=consoleEncoder,logLevel=Debug,stackTraceLevel=Warn). Production Mode defaults(encoder=jsonEncoder,logLevel=Info,stackTraceLevel=Error) (default false)
+  enable: false
+
+datastore:
+  # -- (string) The Datastore name override, if empty defaults to `default`
+  nameOverride:
+  # -- (string) The Kamaji Datastore driver, supported: etcd, MySQL, PostgreSQL (defaults=etcd).
+  driver: etcd
+  # -- (array) List of endpoints of the selected Datastore. When letting the Chart install the etcd datastore, this field is populated automatically.
+  endpoints: []
+  basicAuth:
+    usernameSecret:
+      # -- The name of the Secret containing the username used to connect to the relational database.
+      name:
+      # -- The namespace of the Secret containing the username used to connect to the relational database.
+      namespace:
+      # -- The Secret key where the data is stored.
+      keyPath:
+    passwordSecret:
+      # -- The name of the Secret containing the password used to connect to the relational database.
+      name:
+      # -- The namespace of the Secret containing the password used to connect to the relational database.
+      namespace:
+      # -- The Secret key where the data is stored.
+      keyPath:
+  tlsConfig:
+    certificateAuthority:
+      certificate:
+        # -- Name of the Secret containing the CA required to establish the mandatory SSL/TLS connection to the datastore.
+        name:
+        # -- Namespace of the Secret containing the CA required to establish the mandatory SSL/TLS connection to the datastore.
+        namespace:
+        # -- Key of the Secret which contains the content of the certificate.
+        keyPath:
+      privateKey:
+        # -- Name of the Secret containing the CA private key required to establish the mandatory SSL/TLS connection to the datastore.
+        name:
+        # -- Namespace of the Secret containing the CA private key required to establish the mandatory SSL/TLS connection to the datastore.
+        namespace:
+        # -- Key of the Secret which contains the content of the private key.
+        keyPath:
+    clientCertificate:
+      certificate:
+        # -- Name of the Secret containing the client certificate required to establish the mandatory SSL/TLS connection to the datastore.
+        name:
+        # -- Namespace of the Secret containing the client certificate required to establish the mandatory SSL/TLS connection to the datastore.
+        namespace:
+        # -- Key of the Secret which contains the content of the certificate.
+        keyPath:
+      privateKey:
+        # -- Name of the Secret containing the client certificate private key required to establish the mandatory SSL/TLS connection to the datastore.
+        name:
+        # -- Namespace of the Secret containing the client certificate private key required to establish the mandatory SSL/TLS connection to the datastore.
+        namespace:
+        # -- Key of the Secret which contains the content of the private key.
+        keyPath:

config/crd/bases/kamaji.clastix.io_datastores.yaml

@@ -0,0 +1,268 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.9.2
+  creationTimestamp: null
+  name: datastores.kamaji.clastix.io
+spec:
+  group: kamaji.clastix.io
+  names:
+    kind: DataStore
+    listKind: DataStoreList
+    plural: datastores
+    singular: datastore
+  scope: Cluster
+  versions:
+  - additionalPrinterColumns:
+    - description: Kamaji data store driver
+      jsonPath: .spec.driver
+      name: Driver
+      type: string
+    - description: Age
+      jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1alpha1
+    schema:
+      openAPIV3Schema:
+        description: DataStore is the Schema for the datastores API.
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: DataStoreSpec defines the desired state of DataStore.
+            properties:
+              basicAuth:
+                description: In case of authentication enabled for the given data
+                  store, specifies the username and password pair. This value is optional.
+                properties:
+                  password:
+                    properties:
+                      content:
+                        description: Bare content of the file, base64 encoded. It
+                          has precedence over the SecretReference value.
+                        format: byte
+                        type: string
+                      secretReference:
+                        properties:
+                          keyPath:
+                            description: Name of the key for the given Secret reference
+                              where the content is stored. This value is mandatory.
+                            type: string
+                          name:
+                            description: name is unique within a namespace to reference
+                              a secret resource.
+                            type: string
+                          namespace:
+                            description: namespace defines the space within which
+                              the secret name must be unique.
+                            type: string
+                        required:
+                        - keyPath
+                        type: object
+                        x-kubernetes-map-type: atomic
+                    type: object
+                  username:
+                    properties:
+                      content:
+                        description: Bare content of the file, base64 encoded. It
+                          has precedence over the SecretReference value.
+                        format: byte
+                        type: string
+                      secretReference:
+                        properties:
+                          keyPath:
+                            description: Name of the key for the given Secret reference
+                              where the content is stored. This value is mandatory.
+                            type: string
+                          name:
+                            description: name is unique within a namespace to reference
+                              a secret resource.
+                            type: string
+                          namespace:
+                            description: namespace defines the space within which
+                              the secret name must be unique.
+                            type: string
+                        required:
+                        - keyPath
+                        type: object
+                        x-kubernetes-map-type: atomic
+                    type: object
+                required:
+                - password
+                - username
+                type: object
+              driver:
+                description: The driver to use to connect to the shared datastore.
+                type: string
+              endpoints:
+                description: List of the endpoints to connect to the shared datastore.
+                  No need for protocol, just bare IP/FQDN and port.
+                items:
+                  type: string
+                type: array
+              tlsConfig:
+                description: Defines the TLS/SSL configuration required to connect
+                  to the data store in a secure way.
+                properties:
+                  certificateAuthority:
+                    description: Retrieve the Certificate Authority certificate and
+                      private key, such as bare content of the file, or a SecretReference.
+                      The key reference is required since etcd authentication is based
+                      on certificates, and Kamaji is responsible in creating this.
+                    properties:
+                      certificate:
+                        properties:
+                          content:
+                            description: Bare content of the file, base64 encoded.
+                              It has precedence over the SecretReference value.
+                            format: byte
+                            type: string
+                          secretReference:
+                            properties:
+                              keyPath:
+                                description: Name of the key for the given Secret
+                                  reference where the content is stored. This value
+                                  is mandatory.
+                                type: string
+                              name:
+                                description: name is unique within a namespace to
+                                  reference a secret resource.
+                                type: string
+                              namespace:
+                                description: namespace defines the space within which
+                                  the secret name must be unique.
+                                type: string
+                            required:
+                            - keyPath
+                            type: object
+                            x-kubernetes-map-type: atomic
+                        type: object
+                      privateKey:
+                        properties:
+                          content:
+                            description: Bare content of the file, base64 encoded.
+                              It has precedence over the SecretReference value.
+                            format: byte
+                            type: string
+                          secretReference:
+                            properties:
+                              keyPath:
+                                description: Name of the key for the given Secret
+                                  reference where the content is stored. This value
+                                  is mandatory.
+                                type: string
+                              name:
+                                description: name is unique within a namespace to
+                                  reference a secret resource.
+                                type: string
+                              namespace:
+                                description: namespace defines the space within which
+                                  the secret name must be unique.
+                                type: string
+                            required:
+                            - keyPath
+                            type: object
+                            x-kubernetes-map-type: atomic
+                        type: object
+                    required:
+                    - certificate
+                    type: object
+                  clientCertificate:
+                    description: Specifies the SSL/TLS key and private key pair used
+                      to connect to the data store.
+                    properties:
+                      certificate:
+                        properties:
+                          content:
+                            description: Bare content of the file, base64 encoded.
+                              It has precedence over the SecretReference value.
+                            format: byte
+                            type: string
+                          secretReference:
+                            properties:
+                              keyPath:
+                                description: Name of the key for the given Secret
+                                  reference where the content is stored. This value
+                                  is mandatory.
+                                type: string
+                              name:
+                                description: name is unique within a namespace to
+                                  reference a secret resource.
+                                type: string
+                              namespace:
+                                description: namespace defines the space within which
+                                  the secret name must be unique.
+                                type: string
+                            required:
+                            - keyPath
+                            type: object
+                            x-kubernetes-map-type: atomic
+                        type: object
+                      privateKey:
+                        properties:
+                          content:
+                            description: Bare content of the file, base64 encoded.
+                              It has precedence over the SecretReference value.
+                            format: byte
+                            type: string
+                          secretReference:
+                            properties:
+                              keyPath:
+                                description: Name of the key for the given Secret
+                                  reference where the content is stored. This value
+                                  is mandatory.
+                                type: string
+                              name:
+                                description: name is unique within a namespace to
+                                  reference a secret resource.
+                                type: string
+                              namespace:
+                                description: namespace defines the space within which
+                                  the secret name must be unique.
+                                type: string
+                            required:
+                            - keyPath
+                            type: object
+                            x-kubernetes-map-type: atomic
+                        type: object
+                    required:
+                    - certificate
+                    - privateKey
+                    type: object
+                required:
+                - certificateAuthority
+                - clientCertificate
+                type: object
+            required:
+            - driver
+            - endpoints
+            - tlsConfig
+            type: object
+          status:
+            description: DataStoreStatus defines the observed state of DataStore.
+            properties:
+              usedBy:
+                description: List of the Tenant Control Planes, namespaced named,
+                  using this data store.
+                items:
+                  type: string
+                type: array
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}

config/crd/bases/kamaji.clastix.io_tenantcontrolplanes.yaml

@@ -1,10 +1,9 @@
-
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.6.1
+    controller-gen.kubebuilder.io/version: v0.9.2
   creationTimestamp: null
   name: tenantcontrolplanes.kamaji.clastix.io
 spec:
@@ -64,13 +63,27 @@ spec:
                 description: Addons contain which addons are enabled
                 properties:
                   coreDNS:
-                    description: AddonSpec defines the spec for every addon.
+                    description: Enables the DNS addon in the Tenant Cluster. The
+                      registry and the tag are configurable, the image is hard-coded
+                      to `coredns`.
+                    properties:
+                      imageRepository:
+                        description: ImageRepository sets the container registry to
+                          pull images from. if not set, the default ImageRepository
+                          will be used instead.
+                        type: string
+                      imageTag:
+                        description: ImageTag allows to specify a tag for the image.
+                          In case this value is set, kubeadm does not change automatically
+                          the version of the above components during upgrades.
+                        type: string
                     type: object
                   konnectivity:
-                    description: KonnectivitySpec defines the spec for Konnectivity.
+                    description: Enables the Konnectivity addon in the Tenant Cluster,
+                      required if the worker nodes are in a different network.
                     properties:
                       agentImage:
-                        default: us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-agent
+                        default: registry.k8s.io/kas-network-proxy/proxy-agent
                         description: AgentImage defines the container image for Konnectivity's
                           agent.
                         type: string
@@ -107,19 +120,32 @@ spec:
                             type: object
                         type: object
                       serverImage:
-                        default: us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-server
+                        default: registry.k8s.io/kas-network-proxy/proxy-server
                         description: ServerImage defines the container image for Konnectivity's
                           server.
                         type: string
                       version:
-                        default: v0.0.31
+                        default: v0.0.32
                         description: Version for Konnectivity server and agent.
                         type: string
                     required:
                     - proxyPort
                     type: object
                   kubeProxy:
-                    description: AddonSpec defines the spec for every addon.
+                    description: Enables the kube-proxy addon in the Tenant Cluster.
+                      The registry and the tag are configurable, the image is hard-coded
+                      to `kube-proxy`.
+                    properties:
+                      imageRepository:
+                        description: ImageRepository sets the container registry to
+                          pull images from. if not set, the default ImageRepository
+                          will be used instead.
+                        type: string
+                      imageTag:
+                        description: ImageTag allows to specify a tag for the image.
+                          In case this value is set, kubeadm does not change automatically
+                          the version of the above components during upgrades.
+                        type: string
                     type: object
                 type: object
               controlPlane:
@@ -263,6 +289,194 @@ spec:
                                 type: object
                             type: object
                         type: object
+                      topologySpreadConstraints:
+                        description: TopologySpreadConstraints describes how the Tenant
+                          Control Plane pods ought to spread across topology domains.
+                          Scheduler will schedule pods in a way which abides by the
+                          constraints. In case of nil underlying LabelSelector, the
+                          Kamaji one for the given Tenant Control Plane will be used.
+                          All topologySpreadConstraints are ANDed.
+                        items:
+                          description: TopologySpreadConstraint specifies how to spread
+                            matching pods among the given topology.
+                          properties:
+                            labelSelector:
+                              description: LabelSelector is used to find matching
+                                pods. Pods that match this label selector are counted
+                                to determine the number of pods in their corresponding
+                                topology domain.
+                              properties:
+                                matchExpressions:
+                                  description: matchExpressions is a list of label
+                                    selector requirements. The requirements are ANDed.
+                                  items:
+                                    description: A label selector requirement is a
+                                      selector that contains values, a key, and an
+                                      operator that relates the key and values.
+                                    properties:
+                                      key:
+                                        description: key is the label key that the
+                                          selector applies to.
+                                        type: string
+                                      operator:
+                                        description: operator represents a key's relationship
+                                          to a set of values. Valid operators are
+                                          In, NotIn, Exists and DoesNotExist.
+                                        type: string
+                                      values:
+                                        description: values is an array of string
+                                          values. If the operator is In or NotIn,
+                                          the values array must be non-empty. If the
+                                          operator is Exists or DoesNotExist, the
+                                          values array must be empty. This array is
+                                          replaced during a strategic merge patch.
+                                        items:
+                                          type: string
+                                        type: array
+                                    required:
+                                    - key
+                                    - operator
+                                    type: object
+                                  type: array
+                                matchLabels:
+                                  additionalProperties:
+                                    type: string
+                                  description: matchLabels is a map of {key,value}
+                                    pairs. A single {key,value} in the matchLabels
+                                    map is equivalent to an element of matchExpressions,
+                                    whose key field is "key", the operator is "In",
+                                    and the values array contains only "value". The
+                                    requirements are ANDed.
+                                  type: object
+                              type: object
+                              x-kubernetes-map-type: atomic
+                            matchLabelKeys:
+                              description: MatchLabelKeys is a set of pod label keys
+                                to select the pods over which spreading will be calculated.
+                                The keys are used to lookup values from the incoming
+                                pod labels, those key-value labels are ANDed with
+                                labelSelector to select the group of existing pods
+                                over which spreading will be calculated for the incoming
+                                pod. Keys that don't exist in the incoming pod labels
+                                will be ignored. A null or empty list means only match
+                                against labelSelector.
+                              items:
+                                type: string
+                              type: array
+                              x-kubernetes-list-type: atomic
+                            maxSkew:
+                              description: 'MaxSkew describes the degree to which
+                                pods may be unevenly distributed. When `whenUnsatisfiable=DoNotSchedule`,
+                                it is the maximum permitted difference between the
+                                number of matching pods in the target topology and
+                                the global minimum. The global minimum is the minimum
+                                number of matching pods in an eligible domain or zero
+                                if the number of eligible domains is less than MinDomains.
+                                For example, in a 3-zone cluster, MaxSkew is set to
+                                1, and pods with the same labelSelector spread as
+                                2/2/1: In this case, the global minimum is 1. | zone1
+                                | zone2 | zone3 | |  P P  |  P P  |   P   | - if MaxSkew
+                                is 1, incoming pod can only be scheduled to zone3
+                                to become 2/2/2; scheduling it onto zone1(zone2) would
+                                make the ActualSkew(3-1) on zone1(zone2) violate MaxSkew(1).
+                                - if MaxSkew is 2, incoming pod can be scheduled onto
+                                any zone. When `whenUnsatisfiable=ScheduleAnyway`,
+                                it is used to give higher precedence to topologies
+                                that satisfy it. It''s a required field. Default value
+                                is 1 and 0 is not allowed.'
+                              format: int32
+                              type: integer
+                            minDomains:
+                              description: "MinDomains indicates a minimum number
+                                of eligible domains. When the number of eligible domains
+                                with matching topology keys is less than minDomains,
+                                Pod Topology Spread treats \"global minimum\" as 0,
+                                and then the calculation of Skew is performed. And
+                                when the number of eligible domains with matching
+                                topology keys equals or greater than minDomains, this
+                                value has no effect on scheduling. As a result, when
+                                the number of eligible domains is less than minDomains,
+                                scheduler won't schedule more than maxSkew Pods to
+                                those domains. If value is nil, the constraint behaves
+                                as if MinDomains is equal to 1. Valid values are integers
+                                greater than 0. When value is not nil, WhenUnsatisfiable
+                                must be DoNotSchedule. \n For example, in a 3-zone
+                                cluster, MaxSkew is set to 2, MinDomains is set to
+                                5 and pods with the same labelSelector spread as 2/2/2:
+                                | zone1 | zone2 | zone3 | |  P P  |  P P  |  P P  |
+                                The number of domains is less than 5(MinDomains),
+                                so \"global minimum\" is treated as 0. In this situation,
+                                new pod with the same labelSelector cannot be scheduled,
+                                because computed skew will be 3(3 - 0) if new Pod
+                                is scheduled to any of the three zones, it will violate
+                                MaxSkew. \n This is a beta field and requires the
+                                MinDomainsInPodTopologySpread feature gate to be enabled
+                                (enabled by default)."
+                              format: int32
+                              type: integer
+                            nodeAffinityPolicy:
+                              description: "NodeAffinityPolicy indicates how we will
+                                treat Pod's nodeAffinity/nodeSelector when calculating
+                                pod topology spread skew. Options are: - Honor: only
+                                nodes matching nodeAffinity/nodeSelector are included
+                                in the calculations. - Ignore: nodeAffinity/nodeSelector
+                                are ignored. All nodes are included in the calculations.
+                                \n If this value is nil, the behavior is equivalent
+                                to the Honor policy. This is a alpha-level feature
+                                enabled by the NodeInclusionPolicyInPodTopologySpread
+                                feature flag."
+                              type: string
+                            nodeTaintsPolicy:
+                              description: "NodeTaintsPolicy indicates how we will
+                                treat node taints when calculating pod topology spread
+                                skew. Options are: - Honor: nodes without taints,
+                                along with tainted nodes for which the incoming pod
+                                has a toleration, are included. - Ignore: node taints
+                                are ignored. All nodes are included. \n If this value
+                                is nil, the behavior is equivalent to the Ignore policy.
+                                This is a alpha-level feature enabled by the NodeInclusionPolicyInPodTopologySpread
+                                feature flag."
+                              type: string
+                            topologyKey:
+                              description: TopologyKey is the key of node labels.
+                                Nodes that have a label with this key and identical
+                                values are considered to be in the same topology.
+                                We consider each <key, value> as a "bucket", and try
+                                to put balanced number of pods into each bucket. We
+                                define a domain as a particular instance of a topology.
+                                Also, we define an eligible domain as a domain whose
+                                nodes meet the requirements of nodeAffinityPolicy
+                                and nodeTaintsPolicy. e.g. If TopologyKey is "kubernetes.io/hostname",
+                                each Node is a domain of that topology. And, if TopologyKey
+                                is "topology.kubernetes.io/zone", each zone is a domain
+                                of that topology. It's a required field.
+                              type: string
+                            whenUnsatisfiable:
+                              description: 'WhenUnsatisfiable indicates how to deal
+                                with a pod if it doesn''t satisfy the spread constraint.
+                                - DoNotSchedule (default) tells the scheduler not
+                                to schedule it. - ScheduleAnyway tells the scheduler
+                                to schedule the pod in any location, but giving higher
+                                precedence to topologies that would help reduce the
+                                skew. A constraint is considered "Unsatisfiable" for
+                                an incoming pod if and only if every possible node
+                                assignment for that pod would violate "MaxSkew" on
+                                some topology. For example, in a 3-zone cluster, MaxSkew
+                                is set to 1, and pods with the same labelSelector
+                                spread as 3/1/1: | zone1 | zone2 | zone3 | | P P P
+                                |   P   |   P   | If WhenUnsatisfiable is set to DoNotSchedule,
+                                incoming pod can only be scheduled to zone2(zone3)
+                                to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3)
+                                satisfies MaxSkew(1). In other words, the cluster
+                                can still be imbalanced, but scheduler won''t make
+                                it *more* imbalanced. It''s a required field.'
+                              type: string
+                          required:
+                          - maxSkew
+                          - topologyKey
+                          - whenUnsatisfiable
+                          type: object
+                        type: array
                     type: object
                   ingress:
                     description: Defining the options for an Optional Ingress which
@@ -282,8 +496,6 @@ spec:
                               type: string
                             type: object
                         type: object
-                      enabled:
-                        type: boolean
                       hostname:
                         description: Hostname is an optional field which will be used
                           as Ingress's Host. If it is not defined, Ingress's host
@@ -292,8 +504,6 @@ spec:
                         type: string
                       ingressClassName:
                         type: string
-                    required:
-                    - enabled
                     type: object
                   service:
                     description: Defining the options for the Tenant Control Plane
@@ -327,6 +537,14 @@ spec:
                 required:
                 - service
                 type: object
+              dataStore:
+                description: DataStore allows to specify a DataStore that should be
+                  used to store the Kubernetes data for the given Tenant Control Plane.
+                  This parameter is optional and acts as an override over the default
+                  one which is used by the Kamaji Operator. Migration from a different
+                  DataStore to another one is not yet supported and the reconciliation
+                  will be blocked.
+                type: string
               kubernetes:
                 description: Kubernetes specification for tenant control plane
                 properties:
@@ -479,6 +697,8 @@ spec:
                     properties:
                       agent:
                         properties:
+                          checksum:
+                            type: string
                           lastUpdate:
                             description: Last time when k8s object was updated
                             format: date-time
@@ -487,23 +707,22 @@ spec:
                             type: string
                           namespace:
                             type: string
-                          resourceVersion:
-                            description: Resource version of k8s object
-                            type: string
                         type: object
                       certificate:
                         description: CertificatePrivateKeyPairStatus defines the status.
                         properties:
+                          checksum:
+                            type: string
                           lastUpdate:
                             format: date-time
                             type: string
-                          resourceVersion:
-                            type: string
                           secretName:
                             type: string
                         type: object
                       clusterrolebinding:
                         properties:
+                          checksum:
+                            type: string
                           lastUpdate:
                             description: Last time when k8s object was updated
                             format: date-time
@@ -512,12 +731,14 @@ spec:
                             type: string
                           namespace:
                             type: string
-                          resourceVersion:
-                            description: Resource version of k8s object
+                        type: object
+                      configMap:
+                        properties:
+                          checksum:
+                            type: string
+                          name:
                             type: string
                         type: object
-                      egressSelectorConfiguration:
-                        type: string
                       enabled:
                         type: boolean
                       kubeconfig:
@@ -534,6 +755,8 @@ spec:
                         type: object
                       sa:
                         properties:
+                          checksum:
+                            type: string
                           lastUpdate:
                             description: Last time when k8s object was updated
                             format: date-time
@@ -542,9 +765,6 @@ spec:
                             type: string
                           namespace:
                             type: string
-                          resourceVersion:
-                            description: Resource version of k8s object
-                            type: string
                         type: object
                       service:
                         description: KubernetesServiceStatus defines the status for
@@ -556,15 +776,15 @@ spec:
                               description: "Condition contains details for one aspect
                                 of the current state of this API Resource. --- This
                                 struct is intended for direct use as an array at the
-                                field path .status.conditions.  For example, type
-                                FooStatus struct{     // Represents the observations
-                                of a foo's current state.     // Known .status.conditions.type
+                                field path .status.conditions.  For example, \n type
+                                FooStatus struct{ // Represents the observations of
+                                a foo's current state. // Known .status.conditions.type
                                 are: \"Available\", \"Progressing\", and \"Degraded\"
-                                \    // +patchMergeKey=type     // +patchStrategy=merge
-                                \    // +listType=map     // +listMapKey=type     Conditions
-                                []metav1.Condition `json:\"conditions,omitempty\"
-                                patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"`
-                                \n     // other fields }"
+                                // +patchMergeKey=type // +patchStrategy=merge //
+                                +listType=map // +listMapKey=type Conditions []metav1.Condition
+                                `json:\"conditions,omitempty\" patchStrategy:\"merge\"
+                                patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"`
+                                \n // other fields }"
                               properties:
                                 lastTransitionTime:
                                   description: lastTransitionTime is the last time
@@ -668,11 +888,10 @@ spec:
                                               the error shall comply with the following
                                               rules: - built-in error values shall
                                               be specified in this file and those
-                                              shall use   CamelCase names - cloud
-                                              provider specific error values must
-                                              have names that comply with the   format
-                                              foo.example.com/CamelCase. --- The regex
-                                              it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)'
+                                              shall use CamelCase names - cloud provider
+                                              specific error values must have names
+                                              that comply with the format foo.example.com/CamelCase.
+                                              --- The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)'
                                             maxLength: 316
                                             pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
                                             type: string
@@ -738,33 +957,33 @@ spec:
                   apiServer:
                     description: CertificatePrivateKeyPairStatus defines the status.
                     properties:
+                      checksum:
+                        type: string
                       lastUpdate:
                         format: date-time
                         type: string
-                      resourceVersion:
-                        type: string
                       secretName:
                         type: string
                     type: object
                   apiServerKubeletClient:
                     description: CertificatePrivateKeyPairStatus defines the status.
                     properties:
+                      checksum:
+                        type: string
                       lastUpdate:
                         format: date-time
                         type: string
-                      resourceVersion:
-                        type: string
                       secretName:
                         type: string
                     type: object
                   ca:
                     description: CertificatePrivateKeyPairStatus defines the status.
                     properties:
+                      checksum:
+                        type: string
                       lastUpdate:
                         format: date-time
                         type: string
-                      resourceVersion:
-                        type: string
                       secretName:
                         type: string
                     type: object
@@ -773,9 +992,11 @@ spec:
                       of ETCD Certificate for API server.
                     properties:
                       apiServer:
-                        description: ETCDCertificateStatus defines the observed state
-                          of ETCD Certificate for API server.
+                        description: APIServerCertificatesStatus defines the observed
+                          state of ETCD Certificate for API server.
                         properties:
+                          checksum:
+                            type: string
                           lastUpdate:
                             format: date-time
                             type: string
@@ -786,6 +1007,8 @@ spec:
                         description: ETCDCertificateStatus defines the observed state
                           of ETCD Certificate for API server.
                         properties:
+                          checksum:
+                            type: string
                           lastUpdate:
                             format: date-time
                             type: string
@@ -796,28 +1019,30 @@ spec:
                   frontProxyCA:
                     description: CertificatePrivateKeyPairStatus defines the status.
                     properties:
+                      checksum:
+                        type: string
                       lastUpdate:
                         format: date-time
                         type: string
-                      resourceVersion:
-                        type: string
                       secretName:
                         type: string
                     type: object
                   frontProxyClient:
                     description: CertificatePrivateKeyPairStatus defines the status.
                     properties:
+                      checksum:
+                        type: string
                       lastUpdate:
                         format: date-time
                         type: string
-                      resourceVersion:
-                        type: string
                       secretName:
                         type: string
                     type: object
                   sa:
                     description: PublicKeyPrivateKeyPairStatus defines the status.
                     properties:
+                      checksum:
+                        type: string
                       lastUpdate:
                         format: date-time
                         type: string
@@ -1002,6 +1227,10 @@ spec:
                           by this deployment (their labels match the selector).
                         format: int32
                         type: integer
+                      selector:
+                        description: Selector is the label selector used to group
+                          the Tenant Control Plane Pods used by the scale subresource.
+                        type: string
                       unavailableReplicas:
                         description: Total number of unavailable pods targeted by
                           this deployment. This is the total number of pods that are
@@ -1018,6 +1247,7 @@ spec:
                     required:
                     - name
                     - namespace
+                    - selector
                     type: object
                   ingress:
                     description: KubernetesIngressStatus defines the status for the
@@ -1056,9 +1286,9 @@ spec:
                                           with the service port The format of the
                                           error shall comply with the following rules:
                                           - built-in error values shall be specified
-                                          in this file and those shall use   CamelCase
+                                          in this file and those shall use CamelCase
                                           names - cloud provider specific error values
-                                          must have names that comply with the   format
+                                          must have names that comply with the format
                                           foo.example.com/CamelCase. --- The regex
                                           it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)'
                                         maxLength: 316
@@ -1107,14 +1337,14 @@ spec:
                           description: "Condition contains details for one aspect
                             of the current state of this API Resource. --- This struct
                             is intended for direct use as an array at the field path
-                            .status.conditions.  For example, type FooStatus struct{
-                            \    // Represents the observations of a foo's current
-                            state.     // Known .status.conditions.type are: \"Available\",
-                            \"Progressing\", and \"Degraded\"     // +patchMergeKey=type
-                            \    // +patchStrategy=merge     // +listType=map     //
-                            +listMapKey=type     Conditions []metav1.Condition `json:\"conditions,omitempty\"
-                            patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"`
-                            \n     // other fields }"
+                            .status.conditions.  For example, \n type FooStatus struct{
+                            // Represents the observations of a foo's current state.
+                            // Known .status.conditions.type are: \"Available\", \"Progressing\",
+                            and \"Degraded\" // +patchMergeKey=type // +patchStrategy=merge
+                            // +listType=map // +listMapKey=type Conditions []metav1.Condition
+                            `json:\"conditions,omitempty\" patchStrategy:\"merge\"
+                            patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"`
+                            \n // other fields }"
                           properties:
                             lastTransitionTime:
                               description: lastTransitionTime is the last time the
@@ -1213,9 +1443,9 @@ spec:
                                           with the service port The format of the
                                           error shall comply with the following rules:
                                           - built-in error values shall be specified
-                                          in this file and those shall use   CamelCase
+                                          in this file and those shall use CamelCase
                                           names - cloud provider specific error values
-                                          must have names that comply with the   format
+                                          must have names that comply with the format
                                           foo.example.com/CamelCase. --- The regex
                                           it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)'
                                         maxLength: 316
@@ -1277,84 +1507,44 @@ spec:
                         description: Version is the running Kubernetes version of
                           the Tenant Control Plane.
                         type: string
-                    required:
-                    - status
                     type: object
                 type: object
               storage:
                 description: Storage Status contains information about Kubernetes
                   storage system
                 properties:
-                  etcd:
-                    description: ETCDStatus defines the observed state of ETCDStatus.
+                  certificate:
                     properties:
-                      role:
-                        properties:
-                          exists:
-                            type: boolean
-                          name:
-                            type: string
-                          permissions:
-                            items:
-                              properties:
-                                key:
-                                  type: string
-                                rangeEnd:
-                                  type: string
-                                type:
-                                  type: integer
-                              type: object
-                            type: array
-                        required:
-                        - exists
-                        - name
-                        type: object
-                      user:
-                        properties:
-                          exists:
-                            type: boolean
-                          name:
-                            type: string
-                          roles:
-                            items:
-                              type: string
-                            type: array
-                        required:
-                        - exists
-                        - name
-                        type: object
+                      checksum:
+                        type: string
+                      lastUpdate:
+                        format: date-time
+                        type: string
+                      secretName:
+                        type: string
                     type: object
-                  kineMySQL:
+                  config:
                     properties:
-                      certificate:
-                        properties:
-                          lastUpdate:
-                            format: date-time
-                            type: string
-                          resourceVersion:
-                            type: string
-                          secretName:
-                            type: string
-                        type: object
-                      config:
-                        properties:
-                          resourceVersion:
-                            type: string
-                          secretName:
-                            type: string
-                        type: object
-                      setup:
-                        properties:
-                          lastUpdate:
-                            format: date-time
-                            type: string
-                          schema:
-                            type: string
-                          sqlConfigResourceVersion:
-                            type: string
-                          user:
-                            type: string
-                        type: object
+                      checksum:
+                        type: string
+                      secretName:
+                        type: string
+                    type: object
+                  dataStoreName:
+                    type: string
+                  driver:
+                    type: string
+                  setup:
+                    properties:
+                      checksum:
+                        type: string
+                      lastUpdate:
+                        format: date-time
+                        type: string
+                      schema:
+                        type: string
+                      user:
+                        type: string
                     type: object
                 type: object
             type: object
@@ -1362,10 +1552,8 @@ spec:
     served: true
     storage: true
     subresources:
+      scale:
+        labelSelectorPath: .status.kubernetesResources.deployment.selector
+        specReplicasPath: .spec.controlPlane.deployment.replicas
+        statusReplicasPath: .status.kubernetesResources.deployment.replicas
       status: {}
-status:
-  acceptedNames:
-    kind: ""
-    plural: ""
-  conditions: []
-  storedVersions: []

config/crd/kustomization.yaml

@@ -3,6 +3,7 @@
 # It should be run by config/default
 resources:
 - bases/kamaji.clastix.io_tenantcontrolplanes.yaml
+- bases/kamaji.clastix.io_datastores.yaml
 #+kubebuilder:scaffold:crdkustomizeresource
 
 patchesStrategicMerge:

config/crd/patches/cainjection_in_datastores.yaml

@@ -0,0 +1,7 @@
+# The following patch adds a directive for certmanager to inject CA into the CRD
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME)
+  name: datastores.kamaji.clastix.io

config/crd/patches/webhook_in_datastores.yaml

@@ -0,0 +1,16 @@
+# The following patch enables a conversion webhook for the CRD
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: datastores.kamaji.clastix.io
+spec:
+  conversion:
+    strategy: Webhook
+    webhook:
+      clientConfig:
+        service:
+          namespace: system
+          name: webhook-service
+          path: /convert
+      conversionReviewVersions:
+      - v1

config/default/kustomization.yaml

@@ -16,6 +16,7 @@ bases:
 - ../crd
 - ../rbac
 - ../manager
+- ../samples
 # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
 # crd/kustomization.yaml
 #- ../webhook

config/default/manager_auth_proxy_patch.yaml

@@ -25,3 +25,4 @@ spec:
         - "--health-probe-bind-address=:8081"
         - "--metrics-bind-address=127.0.0.1:8080"
         - "--leader-elect"
+        - "--datastore=kamaji-etcd"

config/install.yaml

@@ -9,7 +9,235 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.6.1
+    controller-gen.kubebuilder.io/version: v0.9.2
+  creationTimestamp: null
+  name: datastores.kamaji.clastix.io
+spec:
+  group: kamaji.clastix.io
+  names:
+    kind: DataStore
+    listKind: DataStoreList
+    plural: datastores
+    singular: datastore
+  scope: Cluster
+  versions:
+  - additionalPrinterColumns:
+    - description: Kamaji data store driver
+      jsonPath: .spec.driver
+      name: Driver
+      type: string
+    - description: Age
+      jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1alpha1
+    schema:
+      openAPIV3Schema:
+        description: DataStore is the Schema for the datastores API.
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: DataStoreSpec defines the desired state of DataStore.
+            properties:
+              basicAuth:
+                description: In case of authentication enabled for the given data store, specifies the username and password pair. This value is optional.
+                properties:
+                  password:
+                    properties:
+                      content:
+                        description: Bare content of the file, base64 encoded. It has precedence over the SecretReference value.
+                        format: byte
+                        type: string
+                      secretReference:
+                        properties:
+                          keyPath:
+                            description: Name of the key for the given Secret reference where the content is stored. This value is mandatory.
+                            type: string
+                          name:
+                            description: name is unique within a namespace to reference a secret resource.
+                            type: string
+                          namespace:
+                            description: namespace defines the space within which the secret name must be unique.
+                            type: string
+                        required:
+                        - keyPath
+                        type: object
+                        x-kubernetes-map-type: atomic
+                    type: object
+                  username:
+                    properties:
+                      content:
+                        description: Bare content of the file, base64 encoded. It has precedence over the SecretReference value.
+                        format: byte
+                        type: string
+                      secretReference:
+                        properties:
+                          keyPath:
+                            description: Name of the key for the given Secret reference where the content is stored. This value is mandatory.
+                            type: string
+                          name:
+                            description: name is unique within a namespace to reference a secret resource.
+                            type: string
+                          namespace:
+                            description: namespace defines the space within which the secret name must be unique.
+                            type: string
+                        required:
+                        - keyPath
+                        type: object
+                        x-kubernetes-map-type: atomic
+                    type: object
+                required:
+                - password
+                - username
+                type: object
+              driver:
+                description: The driver to use to connect to the shared datastore.
+                type: string
+              endpoints:
+                description: List of the endpoints to connect to the shared datastore. No need for protocol, just bare IP/FQDN and port.
+                items:
+                  type: string
+                type: array
+              tlsConfig:
+                description: Defines the TLS/SSL configuration required to connect to the data store in a secure way.
+                properties:
+                  certificateAuthority:
+                    description: Retrieve the Certificate Authority certificate and private key, such as bare content of the file, or a SecretReference. The key reference is required since etcd authentication is based on certificates, and Kamaji is responsible in creating this.
+                    properties:
+                      certificate:
+                        properties:
+                          content:
+                            description: Bare content of the file, base64 encoded. It has precedence over the SecretReference value.
+                            format: byte
+                            type: string
+                          secretReference:
+                            properties:
+                              keyPath:
+                                description: Name of the key for the given Secret reference where the content is stored. This value is mandatory.
+                                type: string
+                              name:
+                                description: name is unique within a namespace to reference a secret resource.
+                                type: string
+                              namespace:
+                                description: namespace defines the space within which the secret name must be unique.
+                                type: string
+                            required:
+                            - keyPath
+                            type: object
+                            x-kubernetes-map-type: atomic
+                        type: object
+                      privateKey:
+                        properties:
+                          content:
+                            description: Bare content of the file, base64 encoded. It has precedence over the SecretReference value.
+                            format: byte
+                            type: string
+                          secretReference:
+                            properties:
+                              keyPath:
+                                description: Name of the key for the given Secret reference where the content is stored. This value is mandatory.
+                                type: string
+                              name:
+                                description: name is unique within a namespace to reference a secret resource.
+                                type: string
+                              namespace:
+                                description: namespace defines the space within which the secret name must be unique.
+                                type: string
+                            required:
+                            - keyPath
+                            type: object
+                            x-kubernetes-map-type: atomic
+                        type: object
+                    required:
+                    - certificate
+                    type: object
+                  clientCertificate:
+                    description: Specifies the SSL/TLS key and private key pair used to connect to the data store.
+                    properties:
+                      certificate:
+                        properties:
+                          content:
+                            description: Bare content of the file, base64 encoded. It has precedence over the SecretReference value.
+                            format: byte
+                            type: string
+                          secretReference:
+                            properties:
+                              keyPath:
+                                description: Name of the key for the given Secret reference where the content is stored. This value is mandatory.
+                                type: string
+                              name:
+                                description: name is unique within a namespace to reference a secret resource.
+                                type: string
+                              namespace:
+                                description: namespace defines the space within which the secret name must be unique.
+                                type: string
+                            required:
+                            - keyPath
+                            type: object
+                            x-kubernetes-map-type: atomic
+                        type: object
+                      privateKey:
+                        properties:
+                          content:
+                            description: Bare content of the file, base64 encoded. It has precedence over the SecretReference value.
+                            format: byte
+                            type: string
+                          secretReference:
+                            properties:
+                              keyPath:
+                                description: Name of the key for the given Secret reference where the content is stored. This value is mandatory.
+                                type: string
+                              name:
+                                description: name is unique within a namespace to reference a secret resource.
+                                type: string
+                              namespace:
+                                description: namespace defines the space within which the secret name must be unique.
+                                type: string
+                            required:
+                            - keyPath
+                            type: object
+                            x-kubernetes-map-type: atomic
+                        type: object
+                    required:
+                    - certificate
+                    - privateKey
+                    type: object
+                required:
+                - certificateAuthority
+                - clientCertificate
+                type: object
+            required:
+            - driver
+            - endpoints
+            - tlsConfig
+            type: object
+          status:
+            description: DataStoreStatus defines the observed state of DataStore.
+            properties:
+              usedBy:
+                description: List of the Tenant Control Planes, namespaced named, using this data store.
+                items:
+                  type: string
+                type: array
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.9.2
   creationTimestamp: null
   name: tenantcontrolplanes.kamaji.clastix.io
 spec:
@@ -64,13 +292,20 @@ spec:
                 description: Addons contain which addons are enabled
                 properties:
                   coreDNS:
-                    description: AddonSpec defines the spec for every addon.
+                    description: Enables the DNS addon in the Tenant Cluster. The registry and the tag are configurable, the image is hard-coded to `coredns`.
+                    properties:
+                      imageRepository:
+                        description: ImageRepository sets the container registry to pull images from. if not set, the default ImageRepository will be used instead.
+                        type: string
+                      imageTag:
+                        description: ImageTag allows to specify a tag for the image. In case this value is set, kubeadm does not change automatically the version of the above components during upgrades.
+                        type: string
                     type: object
                   konnectivity:
-                    description: KonnectivitySpec defines the spec for Konnectivity.
+                    description: Enables the Konnectivity addon in the Tenant Cluster, required if the worker nodes are in a different network.
                     properties:
                       agentImage:
-                        default: us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-agent
+                        default: registry.k8s.io/kas-network-proxy/proxy-agent
                         description: AgentImage defines the container image for Konnectivity's agent.
                         type: string
                       proxyPort:
@@ -100,18 +335,25 @@ spec:
                             type: object
                         type: object
                       serverImage:
-                        default: us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-server
+                        default: registry.k8s.io/kas-network-proxy/proxy-server
                         description: ServerImage defines the container image for Konnectivity's server.
                         type: string
                       version:
-                        default: v0.0.31
+                        default: v0.0.32
                         description: Version for Konnectivity server and agent.
                         type: string
                     required:
                     - proxyPort
                     type: object
                   kubeProxy:
-                    description: AddonSpec defines the spec for every addon.
+                    description: Enables the kube-proxy addon in the Tenant Cluster. The registry and the tag are configurable, the image is hard-coded to `kube-proxy`.
+                    properties:
+                      imageRepository:
+                        description: ImageRepository sets the container registry to pull images from. if not set, the default ImageRepository will be used instead.
+                        type: string
+                      imageTag:
+                        description: ImageTag allows to specify a tag for the image. In case this value is set, kubeadm does not change automatically the version of the above components during upgrades.
+                        type: string
                     type: object
                 type: object
               controlPlane:
@@ -227,6 +469,74 @@ spec:
                                 type: object
                             type: object
                         type: object
+                      topologySpreadConstraints:
+                        description: TopologySpreadConstraints describes how the Tenant Control Plane pods ought to spread across topology domains. Scheduler will schedule pods in a way which abides by the constraints. In case of nil underlying LabelSelector, the Kamaji one for the given Tenant Control Plane will be used. All topologySpreadConstraints are ANDed.
+                        items:
+                          description: TopologySpreadConstraint specifies how to spread matching pods among the given topology.
+                          properties:
+                            labelSelector:
+                              description: LabelSelector is used to find matching pods. Pods that match this label selector are counted to determine the number of pods in their corresponding topology domain.
+                              properties:
+                                matchExpressions:
+                                  description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
+                                  items:
+                                    description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.
+                                    properties:
+                                      key:
+                                        description: key is the label key that the selector applies to.
+                                        type: string
+                                      operator:
+                                        description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.
+                                        type: string
+                                      values:
+                                        description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.
+                                        items:
+                                          type: string
+                                        type: array
+                                    required:
+                                    - key
+                                    - operator
+                                    type: object
+                                  type: array
+                                matchLabels:
+                                  additionalProperties:
+                                    type: string
+                                  description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed.
+                                  type: object
+                              type: object
+                              x-kubernetes-map-type: atomic
+                            matchLabelKeys:
+                              description: MatchLabelKeys is a set of pod label keys to select the pods over which spreading will be calculated. The keys are used to lookup values from the incoming pod labels, those key-value labels are ANDed with labelSelector to select the group of existing pods over which spreading will be calculated for the incoming pod. Keys that don't exist in the incoming pod labels will be ignored. A null or empty list means only match against labelSelector.
+                              items:
+                                type: string
+                              type: array
+                              x-kubernetes-list-type: atomic
+                            maxSkew:
+                              description: 'MaxSkew describes the degree to which pods may be unevenly distributed. When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference between the number of matching pods in the target topology and the global minimum. The global minimum is the minimum number of matching pods in an eligible domain or zero if the number of eligible domains is less than MinDomains. For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same labelSelector spread as 2/2/1: In this case, the global minimum is 1. | zone1 | zone2 | zone3 | |  P P  |  P P  |   P   | - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) violate MaxSkew(1). - if MaxSkew is 2, incoming pod can be scheduled onto any zone. When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence to topologies that satisfy it. It''s a required field. Default value is 1 and 0 is not allowed.'
+                              format: int32
+                              type: integer
+                            minDomains:
+                              description: "MinDomains indicates a minimum number of eligible domains. When the number of eligible domains with matching topology keys is less than minDomains, Pod Topology Spread treats \"global minimum\" as 0, and then the calculation of Skew is performed. And when the number of eligible domains with matching topology keys equals or greater than minDomains, this value has no effect on scheduling. As a result, when the number of eligible domains is less than minDomains, scheduler won't schedule more than maxSkew Pods to those domains. If value is nil, the constraint behaves as if MinDomains is equal to 1. Valid values are integers greater than 0. When value is not nil, WhenUnsatisfiable must be DoNotSchedule. \n For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same labelSelector spread as 2/2/2: | zone1 | zone2 | zone3 | |  P P  |  P P  |  P P  | The number of domains is less than 5(MinDomains), so \"global minimum\" is treated as 0. In this situation, new pod with the same labelSelector cannot be scheduled, because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, it will violate MaxSkew. \n This is a beta field and requires the MinDomainsInPodTopologySpread feature gate to be enabled (enabled by default)."
+                              format: int32
+                              type: integer
+                            nodeAffinityPolicy:
+                              description: "NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector when calculating pod topology spread skew. Options are: - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. \n If this value is nil, the behavior is equivalent to the Honor policy. This is a alpha-level feature enabled by the NodeInclusionPolicyInPodTopologySpread feature flag."
+                              type: string
+                            nodeTaintsPolicy:
+                              description: "NodeTaintsPolicy indicates how we will treat node taints when calculating pod topology spread skew. Options are: - Honor: nodes without taints, along with tainted nodes for which the incoming pod has a toleration, are included. - Ignore: node taints are ignored. All nodes are included. \n If this value is nil, the behavior is equivalent to the Ignore policy. This is a alpha-level feature enabled by the NodeInclusionPolicyInPodTopologySpread feature flag."
+                              type: string
+                            topologyKey:
+                              description: TopologyKey is the key of node labels. Nodes that have a label with this key and identical values are considered to be in the same topology. We consider each <key, value> as a "bucket", and try to put balanced number of pods into each bucket. We define a domain as a particular instance of a topology. Also, we define an eligible domain as a domain whose nodes meet the requirements of nodeAffinityPolicy and nodeTaintsPolicy. e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. It's a required field.
+                              type: string
+                            whenUnsatisfiable:
+                              description: 'WhenUnsatisfiable indicates how to deal with a pod if it doesn''t satisfy the spread constraint. - DoNotSchedule (default) tells the scheduler not to schedule it. - ScheduleAnyway tells the scheduler to schedule the pod in any location, but giving higher precedence to topologies that would help reduce the skew. A constraint is considered "Unsatisfiable" for an incoming pod if and only if every possible node assignment for that pod would violate "MaxSkew" on some topology. For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same labelSelector spread as 3/1/1: | zone1 | zone2 | zone3 | | P P P |   P   |   P   | If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler won''t make it *more* imbalanced. It''s a required field.'
+                              type: string
+                          required:
+                          - maxSkew
+                          - topologyKey
+                          - whenUnsatisfiable
+                          type: object
+                        type: array
                     type: object
                   ingress:
                     description: Defining the options for an Optional Ingress which will expose API Server of the Tenant Control Plane
@@ -243,15 +553,11 @@ spec:
                               type: string
                             type: object
                         type: object
-                      enabled:
-                        type: boolean
                       hostname:
                         description: Hostname is an optional field which will be used as Ingress's Host. If it is not defined, Ingress's host will be "<tenant>.<namespace>.<domain>", where domain is specified under NetworkProfileSpec
                         type: string
                       ingressClassName:
                         type: string
-                    required:
-                    - enabled
                     type: object
                   service:
                     description: Defining the options for the Tenant Control Plane Service resource.
@@ -281,6 +587,9 @@ spec:
                 required:
                 - service
                 type: object
+              dataStore:
+                description: DataStore allows to specify a DataStore that should be used to store the Kubernetes data for the given Tenant Control Plane. This parameter is optional and acts as an override over the default one which is used by the Kamaji Operator. Migration from a different DataStore to another one is not yet supported and the reconciliation will be blocked.
+                type: string
               kubernetes:
                 description: Kubernetes specification for tenant control plane
                 properties:
@@ -423,6 +732,8 @@ spec:
                     properties:
                       agent:
                         properties:
+                          checksum:
+                            type: string
                           lastUpdate:
                             description: Last time when k8s object was updated
                             format: date-time
@@ -431,23 +742,22 @@ spec:
                             type: string
                           namespace:
                             type: string
-                          resourceVersion:
-                            description: Resource version of k8s object
-                            type: string
                         type: object
                       certificate:
                         description: CertificatePrivateKeyPairStatus defines the status.
                         properties:
+                          checksum:
+                            type: string
                           lastUpdate:
                             format: date-time
                             type: string
-                          resourceVersion:
-                            type: string
                           secretName:
                             type: string
                         type: object
                       clusterrolebinding:
                         properties:
+                          checksum:
+                            type: string
                           lastUpdate:
                             description: Last time when k8s object was updated
                             format: date-time
@@ -456,12 +766,14 @@ spec:
                             type: string
                           namespace:
                             type: string
-                          resourceVersion:
-                            description: Resource version of k8s object
+                        type: object
+                      configMap:
+                        properties:
+                          checksum:
+                            type: string
+                          name:
                             type: string
                         type: object
-                      egressSelectorConfiguration:
-                        type: string
                       enabled:
                         type: boolean
                       kubeconfig:
@@ -477,6 +789,8 @@ spec:
                         type: object
                       sa:
                         properties:
+                          checksum:
+                            type: string
                           lastUpdate:
                             description: Last time when k8s object was updated
                             format: date-time
@@ -485,9 +799,6 @@ spec:
                             type: string
                           namespace:
                             type: string
-                          resourceVersion:
-                            description: Resource version of k8s object
-                            type: string
                         type: object
                       service:
                         description: KubernetesServiceStatus defines the status for the Tenant Control Plane Service in the management cluster.
@@ -495,7 +806,7 @@ spec:
                           conditions:
                             description: Current service state
                             items:
-                              description: "Condition contains details for one aspect of the current state of this API Resource. --- This struct is intended for direct use as an array at the field path .status.conditions.  For example, type FooStatus struct{     // Represents the observations of a foo's current state.     // Known .status.conditions.type are: \"Available\", \"Progressing\", and \"Degraded\"     // +patchMergeKey=type     // +patchStrategy=merge     // +listType=map     // +listMapKey=type     Conditions []metav1.Condition `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"` \n     // other fields }"
+                              description: "Condition contains details for one aspect of the current state of this API Resource. --- This struct is intended for direct use as an array at the field path .status.conditions.  For example, \n type FooStatus struct{ // Represents the observations of a foo's current state. // Known .status.conditions.type are: \"Available\", \"Progressing\", and \"Degraded\" // +patchMergeKey=type // +patchStrategy=merge // +listType=map // +listMapKey=type Conditions []metav1.Condition `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"` \n // other fields }"
                               properties:
                                 lastTransitionTime:
                                   description: lastTransitionTime is the last time the condition transitioned from one status to another. This should be when the underlying condition changed.  If that is not known, then using the time when the API field changed is acceptable.
@@ -558,7 +869,7 @@ spec:
                                       items:
                                         properties:
                                           error:
-                                            description: 'Error is to record the problem with the service port The format of the error shall comply with the following rules: - built-in error values shall be specified in this file and those shall use   CamelCase names - cloud provider specific error values must have names that comply with the   format foo.example.com/CamelCase. --- The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)'
+                                            description: 'Error is to record the problem with the service port The format of the error shall comply with the following rules: - built-in error values shall be specified in this file and those shall use CamelCase names - cloud provider specific error values must have names that comply with the format foo.example.com/CamelCase. --- The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)'
                                             maxLength: 316
                                             pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
                                             type: string
@@ -617,33 +928,33 @@ spec:
                   apiServer:
                     description: CertificatePrivateKeyPairStatus defines the status.
                     properties:
+                      checksum:
+                        type: string
                       lastUpdate:
                         format: date-time
                         type: string
-                      resourceVersion:
-                        type: string
                       secretName:
                         type: string
                     type: object
                   apiServerKubeletClient:
                     description: CertificatePrivateKeyPairStatus defines the status.
                     properties:
+                      checksum:
+                        type: string
                       lastUpdate:
                         format: date-time
                         type: string
-                      resourceVersion:
-                        type: string
                       secretName:
                         type: string
                     type: object
                   ca:
                     description: CertificatePrivateKeyPairStatus defines the status.
                     properties:
+                      checksum:
+                        type: string
                       lastUpdate:
                         format: date-time
                         type: string
-                      resourceVersion:
-                        type: string
                       secretName:
                         type: string
                     type: object
@@ -651,8 +962,10 @@ spec:
                     description: ETCDCertificatesStatus defines the observed state of ETCD Certificate for API server.
                     properties:
                       apiServer:
-                        description: ETCDCertificateStatus defines the observed state of ETCD Certificate for API server.
+                        description: APIServerCertificatesStatus defines the observed state of ETCD Certificate for API server.
                         properties:
+                          checksum:
+                            type: string
                           lastUpdate:
                             format: date-time
                             type: string
@@ -662,6 +975,8 @@ spec:
                       ca:
                         description: ETCDCertificateStatus defines the observed state of ETCD Certificate for API server.
                         properties:
+                          checksum:
+                            type: string
                           lastUpdate:
                             format: date-time
                             type: string
@@ -672,28 +987,30 @@ spec:
                   frontProxyCA:
                     description: CertificatePrivateKeyPairStatus defines the status.
                     properties:
+                      checksum:
+                        type: string
                       lastUpdate:
                         format: date-time
                         type: string
-                      resourceVersion:
-                        type: string
                       secretName:
                         type: string
                     type: object
                   frontProxyClient:
                     description: CertificatePrivateKeyPairStatus defines the status.
                     properties:
+                      checksum:
+                        type: string
                       lastUpdate:
                         format: date-time
                         type: string
-                      resourceVersion:
-                        type: string
                       secretName:
                         type: string
                     type: object
                   sa:
                     description: PublicKeyPrivateKeyPairStatus defines the status.
                     properties:
+                      checksum:
+                        type: string
                       lastUpdate:
                         format: date-time
                         type: string
@@ -854,6 +1171,9 @@ spec:
                         description: Total number of non-terminated pods targeted by this deployment (their labels match the selector).
                         format: int32
                         type: integer
+                      selector:
+                        description: Selector is the label selector used to group the Tenant Control Plane Pods used by the scale subresource.
+                        type: string
                       unavailableReplicas:
                         description: Total number of unavailable pods targeted by this deployment. This is the total number of pods that are still required for the deployment to have 100% available capacity. They may either be pods that are running but not yet available or pods that still have not been created.
                         format: int32
@@ -865,6 +1185,7 @@ spec:
                     required:
                     - name
                     - namespace
+                    - selector
                     type: object
                   ingress:
                     description: KubernetesIngressStatus defines the status for the Tenant Control Plane Ingress in the management cluster.
@@ -888,7 +1209,7 @@ spec:
                                   items:
                                     properties:
                                       error:
-                                        description: 'Error is to record the problem with the service port The format of the error shall comply with the following rules: - built-in error values shall be specified in this file and those shall use   CamelCase names - cloud provider specific error values must have names that comply with the   format foo.example.com/CamelCase. --- The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)'
+                                        description: 'Error is to record the problem with the service port The format of the error shall comply with the following rules: - built-in error values shall be specified in this file and those shall use CamelCase names - cloud provider specific error values must have names that comply with the format foo.example.com/CamelCase. --- The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)'
                                         maxLength: 316
                                         pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
                                         type: string
@@ -925,7 +1246,7 @@ spec:
                       conditions:
                         description: Current service state
                         items:
-                          description: "Condition contains details for one aspect of the current state of this API Resource. --- This struct is intended for direct use as an array at the field path .status.conditions.  For example, type FooStatus struct{     // Represents the observations of a foo's current state.     // Known .status.conditions.type are: \"Available\", \"Progressing\", and \"Degraded\"     // +patchMergeKey=type     // +patchStrategy=merge     // +listType=map     // +listMapKey=type     Conditions []metav1.Condition `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"` \n     // other fields }"
+                          description: "Condition contains details for one aspect of the current state of this API Resource. --- This struct is intended for direct use as an array at the field path .status.conditions.  For example, \n type FooStatus struct{ // Represents the observations of a foo's current state. // Known .status.conditions.type are: \"Available\", \"Progressing\", and \"Degraded\" // +patchMergeKey=type // +patchStrategy=merge // +listType=map // +listMapKey=type Conditions []metav1.Condition `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"` \n // other fields }"
                           properties:
                             lastTransitionTime:
                               description: lastTransitionTime is the last time the condition transitioned from one status to another. This should be when the underlying condition changed.  If that is not known, then using the time when the API field changed is acceptable.
@@ -988,7 +1309,7 @@ spec:
                                   items:
                                     properties:
                                       error:
-                                        description: 'Error is to record the problem with the service port The format of the error shall comply with the following rules: - built-in error values shall be specified in this file and those shall use   CamelCase names - cloud provider specific error values must have names that comply with the   format foo.example.com/CamelCase. --- The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)'
+                                        description: 'Error is to record the problem with the service port The format of the error shall comply with the following rules: - built-in error values shall be specified in this file and those shall use CamelCase names - cloud provider specific error values must have names that comply with the format foo.example.com/CamelCase. --- The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)'
                                         maxLength: 316
                                         pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
                                         type: string
@@ -1039,83 +1360,43 @@ spec:
                       version:
                         description: Version is the running Kubernetes version of the Tenant Control Plane.
                         type: string
-                    required:
-                    - status
                     type: object
                 type: object
               storage:
                 description: Storage Status contains information about Kubernetes storage system
                 properties:
-                  etcd:
-                    description: ETCDStatus defines the observed state of ETCDStatus.
+                  certificate:
                     properties:
-                      role:
-                        properties:
-                          exists:
-                            type: boolean
-                          name:
-                            type: string
-                          permissions:
-                            items:
-                              properties:
-                                key:
-                                  type: string
-                                rangeEnd:
-                                  type: string
-                                type:
-                                  type: integer
-                              type: object
-                            type: array
-                        required:
-                        - exists
-                        - name
-                        type: object
-                      user:
-                        properties:
-                          exists:
-                            type: boolean
-                          name:
-                            type: string
-                          roles:
-                            items:
-                              type: string
-                            type: array
-                        required:
-                        - exists
-                        - name
-                        type: object
+                      checksum:
+                        type: string
+                      lastUpdate:
+                        format: date-time
+                        type: string
+                      secretName:
+                        type: string
                     type: object
-                  kineMySQL:
+                  config:
                     properties:
-                      certificate:
-                        properties:
-                          lastUpdate:
-                            format: date-time
-                            type: string
-                          resourceVersion:
-                            type: string
-                          secretName:
-                            type: string
-                        type: object
-                      config:
-                        properties:
-                          resourceVersion:
-                            type: string
-                          secretName:
-                            type: string
-                        type: object
-                      setup:
-                        properties:
-                          lastUpdate:
-                            format: date-time
-                            type: string
-                          schema:
-                            type: string
-                          sqlConfigResourceVersion:
-                            type: string
-                          user:
-                            type: string
-                        type: object
+                      checksum:
+                        type: string
+                      secretName:
+                        type: string
+                    type: object
+                  dataStoreName:
+                    type: string
+                  driver:
+                    type: string
+                  setup:
+                    properties:
+                      checksum:
+                        type: string
+                      lastUpdate:
+                        format: date-time
+                        type: string
+                      schema:
+                        type: string
+                      user:
+                        type: string
                     type: object
                 type: object
             type: object
@@ -1123,13 +1404,11 @@ spec:
     served: true
     storage: true
     subresources:
+      scale:
+        labelSelectorPath: .status.kubernetesResources.deployment.selector
+        specReplicasPath: .spec.controlPlane.deployment.replicas
+        statusReplicasPath: .status.kubernetesResources.deployment.replicas
       status: {}
-status:
-  acceptedNames:
-    kind: ""
-    plural: ""
-  conditions: []
-  storedVersions: []
 ---
 apiVersion: v1
 kind: ServiceAccount
@@ -1229,6 +1508,26 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - kamaji.clastix.io
+  resources:
+  - datastores
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - kamaji.clastix.io
+  resources:
+  - datastores/status
+  verbs:
+  - get
+  - patch
+  - update
 - apiGroups:
   - kamaji.clastix.io
   resources:
@@ -1404,9 +1703,10 @@ spec:
         - --health-probe-bind-address=:8081
         - --metrics-bind-address=127.0.0.1:8080
         - --leader-elect
+        - --datastore=kamaji-etcd
         command:
         - /manager
-        image: clastix/kamaji:latest
+        image: clastix/kamaji:v0.1.0
         imagePullPolicy: Always
         livenessProbe:
           httpGet:
@@ -1434,3 +1734,39 @@ spec:
         runAsNonRoot: true
       serviceAccountName: kamaji-controller-manager
       terminationGracePeriodSeconds: 10
+---
+apiVersion: kamaji.clastix.io/v1alpha1
+kind: DataStore
+metadata:
+  name: kamaji-etcd
+  namespace: kamaji-system
+spec:
+  basicAuth: null
+  driver: etcd
+  endpoints:
+  - etcd-0.etcd.kamaji-system.svc.cluster.local:2379
+  - etcd-1.etcd.kamaji-system.svc.cluster.local:2379
+  - etcd-2.etcd.kamaji-system.svc.cluster.local:2379
+  tlsConfig:
+    certificateAuthority:
+      certificate:
+        secretReference:
+          keyPath: ca.crt
+          name: etcd-certs
+          namespace: kamaji-system
+      privateKey:
+        secretReference:
+          keyPath: ca.key
+          name: etcd-certs
+          namespace: kamaji-system
+    clientCertificate:
+      certificate:
+        secretReference:
+          keyPath: tls.crt
+          name: root-client-certs
+          namespace: kamaji-system
+      privateKey:
+        secretReference:
+          keyPath: tls.key
+          name: root-client-certs
+          namespace: kamaji-system

config/manager/kustomization.yaml

@@ -13,4 +13,4 @@ kind: Kustomization
 images:
 - name: controller
   newName: clastix/kamaji
-  newTag: latest
+  newTag: v0.1.0

config/rbac/datastore_editor_role.yaml

@@ -0,0 +1,24 @@
+# permissions for end users to edit datastores.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: datastore-editor-role
+rules:
+- apiGroups:
+  - kamaji.clastix.io
+  resources:
+  - datastores
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - kamaji.clastix.io
+  resources:
+  - datastores/status
+  verbs:
+  - get

config/rbac/datastore_viewer_role.yaml

@@ -0,0 +1,20 @@
+# permissions for end users to view datastores.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: datastore-viewer-role
+rules:
+- apiGroups:
+  - kamaji.clastix.io
+  resources:
+  - datastores
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - kamaji.clastix.io
+  resources:
+  - datastores/status
+  verbs:
+  - get

config/rbac/role.yaml

@@ -1,4 +1,3 @@
-
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
@@ -54,6 +53,26 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - kamaji.clastix.io
+  resources:
+  - datastores
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - kamaji.clastix.io
+  resources:
+  - datastores/status
+  verbs:
+  - get
+  - patch
+  - update
 - apiGroups:
   - kamaji.clastix.io
   resources:

config/samples/kamaji_v1alpha1_datastore_etcd.yaml

@@ -0,0 +1,34 @@
+apiVersion: kamaji.clastix.io/v1alpha1
+kind: DataStore
+metadata:
+  name: etcd
+spec:
+  driver: etcd
+  endpoints:
+    - etcd-0.etcd.kamaji-system.svc.cluster.local:2379
+    - etcd-1.etcd.kamaji-system.svc.cluster.local:2379
+    - etcd-2.etcd.kamaji-system.svc.cluster.local:2379
+  basicAuth: null
+  tlsConfig:
+    certificateAuthority:
+      certificate:
+        secretReference:
+          name: etcd-certs
+          namespace: kamaji-system
+          keyPath: "ca.crt"
+      privateKey:
+        secretReference:
+          name: etcd-certs
+          namespace: kamaji-system
+          keyPath: "ca.key"
+    clientCertificate:
+      certificate:
+        secretReference:
+          name: root-client-certs
+          namespace: kamaji-system
+          keyPath: "tls.crt"
+      privateKey:
+        secretReference:
+          name: root-client-certs
+          namespace: kamaji-system
+          keyPath: "tls.key"

config/samples/kamaji_v1alpha1_datastore_mysql.yaml

@@ -0,0 +1,34 @@
+apiVersion: kamaji.clastix.io/v1alpha1
+kind: DataStore
+metadata:
+  name: mysql
+spec:
+  driver: MySQL
+  endpoints:
+    - mariadb.kamaji-system.svc:3306
+  basicAuth:
+    username:
+      content: cm9vdA==
+    password:
+      secretReference:
+        name: mysql-config
+        namespace: kamaji-system
+        keyPath: MYSQL_ROOT_PASSWORD
+  tlsConfig:
+    certificateAuthority:
+      certificate:
+        secretReference:
+          name: mysql-config
+          namespace: kamaji-system
+          keyPath: "ca.crt"
+    clientCertificate:
+      certificate:
+        secretReference:
+          name: mysql-config
+          namespace: kamaji-system
+          keyPath: "server.crt"
+      privateKey:
+        secretReference:
+          name: mysql-config
+          namespace: kamaji-system
+          keyPath: "server.key"

config/samples/kamaji_v1alpha1_datastore_postgresql.yaml

@@ -0,0 +1,37 @@
+apiVersion: kamaji.clastix.io/v1alpha1
+kind: DataStore
+metadata:
+  name: postgresql
+spec:
+  driver: PostgreSQL
+  endpoints:
+    - postgresql-rw.kamaji-system.svc:5432
+  basicAuth:
+    username:
+      secretReference:
+        name: postgresql-superuser
+        namespace: kamaji-system
+        keyPath: username
+    password:
+      secretReference:
+        name: postgresql-superuser
+        namespace: kamaji-system
+        keyPath: password
+  tlsConfig:
+    certificateAuthority:
+      certificate:
+        secretReference:
+          name: postgresql-ca
+          namespace: kamaji-system
+          keyPath: ca.crt
+    clientCertificate:
+      certificate:
+        secretReference:
+          name: postgres-root-cert
+          namespace: kamaji-system
+          keyPath: tls.crt
+      privateKey:
+        secretReference:
+          name: postgres-root-cert
+          namespace: kamaji-system
+          keyPath: tls.key

config/samples/kamaji_v1alpha1_tenantcontrolplane.yaml

@@ -5,48 +5,18 @@ metadata:
 spec:
   controlPlane:
     deployment:
-      replicas: 2
-      additionalMetadata:
-        annotations:
-          environment.clastix.io: test
-          tier.clastix.io: "0"
-        labels:
-          tenant.clastix.io: test
-          kind.clastix.io: deployment
+      replicas: 1
     service:
-      additionalMetadata:
-        annotations:
-          environment.clastix.io: test
-          tier.clastix.io: "0"
-        labels:
-          tenant.clastix.io: test
-          kind.clastix.io: service
       serviceType: LoadBalancer
-    ingress:
-      enabled: true
-      hostname: kamaji.local
-      ingressClassName: nginx
-      additionalMetadata:
-        annotations:
-          kubernetes.io/ingress.allow-http: "false"
-          nginx.ingress.kubernetes.io/secure-backends: "true"
-          nginx.ingress.kubernetes.io/ssl-passthrough: "true"
   kubernetes:
     version: "v1.23.1"
     kubelet:
-      cgroupfs: systemd
+      cgroupfs: cgroupfs
     admissionControllers:
       - ResourceQuota
       - LimitRanger
   networkProfile:
-    address: "127.0.0.1"
     port: 6443
-    certSANs:
-      - "test.clastix.labs"
-    serviceCidr: "10.96.0.0/16"
-    podCidr: "10.244.0.0/16"
-    dnsServiceIPs:
-      - "10.96.0.10"
   addons:
     coreDNS: {}
     kubeProxy: {}

config/samples/kustomization.yaml

@@ -1,4 +1,4 @@
 ## Append samples you want in your CSV to this file as resources ##
 resources:
-- kamaji_v1alpha1_tenantcontrolplane.yaml
+- kamaji_v1alpha1_datastore_etcd.yaml
 #+kubebuilder:scaffold:manifestskustomizesamples

controllers/datastore_controller.go

@@ -0,0 +1,180 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package controllers
+
+import (
+	"context"
+	"fmt"
+
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/fields"
+	k8stypes "k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/client-go/util/workqueue"
+	controllerruntime "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/builder"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+	"sigs.k8s.io/controller-runtime/pkg/event"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
+	"sigs.k8s.io/controller-runtime/pkg/log"
+	"sigs.k8s.io/controller-runtime/pkg/predicate"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+	"sigs.k8s.io/controller-runtime/pkg/source"
+
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+	"github.com/clastix/kamaji/indexers"
+)
+
+const (
+	dataStoreFinalizer = "finalizer.kamaji.clastix.io/datastore"
+)
+
+type DataStore struct {
+	client client.Client
+	// TenantControlPlaneTrigger is the channel used to communicate across the controllers:
+	// if a Data Source is updated we have to be sure that the reconciliation of the certificates content
+	// for each Tenant Control Plane is put in place properly.
+	TenantControlPlaneTrigger TenantControlPlaneChannel
+}
+
+//+kubebuilder:rbac:groups=kamaji.clastix.io,resources=datastores,verbs=get;list;watch;create;update;patch;delete
+//+kubebuilder:rbac:groups=kamaji.clastix.io,resources=datastores/status,verbs=get;update;patch
+
+func (r *DataStore) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) {
+	log := log.FromContext(ctx)
+
+	ds := &kamajiv1alpha1.DataStore{}
+	if err := r.client.Get(ctx, request.NamespacedName, ds); err != nil {
+		if k8serrors.IsNotFound(err) {
+			return reconcile.Result{}, nil
+		}
+
+		log.Error(err, "unable to retrieve the request")
+
+		return reconcile.Result{}, err
+	}
+	// Managing the finalizer, required to don't drop a DataSource if this is still used by a Tenant Control Plane.
+	switch {
+	case ds.DeletionTimestamp != nil && controllerutil.ContainsFinalizer(ds, dataStoreFinalizer):
+		log.Info("marked for deletion, checking conditions")
+
+		if len(ds.Status.UsedBy) == 0 {
+			log.Info("resource is no more used by any Tenant Control Plane")
+
+			controllerutil.RemoveFinalizer(ds, dataStoreFinalizer)
+
+			return reconcile.Result{}, r.client.Update(ctx, ds)
+		}
+
+		log.Info("DataStore is still used by some Tenant Control Planes, cannot be removed")
+	case ds.DeletionTimestamp == nil && !controllerutil.ContainsFinalizer(ds, dataStoreFinalizer):
+		log.Info("the resource is missing the required finalizer, adding it")
+
+		controllerutil.AddFinalizer(ds, dataStoreFinalizer)
+
+		return reconcile.Result{}, r.client.Update(ctx, ds)
+	}
+	// A Data Source can trigger several Tenant Control Planes and requires a minimum validation:
+	// we have to ensure the data provided by the Data Source is valid and referencing an existing Secret object.
+	if _, err := ds.Spec.TLSConfig.CertificateAuthority.Certificate.GetContent(ctx, r.client); err != nil {
+		log.Error(err, "invalid Certificate Authority data")
+
+		return reconcile.Result{}, err
+	}
+
+	if ds.Spec.Driver == kamajiv1alpha1.EtcdDriver {
+		if ds.Spec.TLSConfig.CertificateAuthority.PrivateKey == nil {
+			err := fmt.Errorf("a valid private key is required for the etcd driver")
+
+			log.Error(err, "missing Certificate Authority private key data")
+
+			return reconcile.Result{}, err
+		}
+		if _, err := ds.Spec.TLSConfig.CertificateAuthority.PrivateKey.GetContent(ctx, r.client); err != nil {
+			log.Error(err, "invalid Certificate Authority private key data")
+
+			return reconcile.Result{}, err
+		}
+	}
+
+	if _, err := ds.Spec.TLSConfig.ClientCertificate.Certificate.GetContent(ctx, r.client); err != nil {
+		log.Error(err, "invalid Client Certificate data")
+
+		return reconcile.Result{}, err
+	}
+
+	if _, err := ds.Spec.TLSConfig.ClientCertificate.PrivateKey.GetContent(ctx, r.client); err != nil {
+		log.Error(err, "invalid Client Certificate private key data")
+
+		return reconcile.Result{}, err
+	}
+
+	tcpList := kamajiv1alpha1.TenantControlPlaneList{}
+
+	if err := r.client.List(ctx, &tcpList, client.MatchingFieldsSelector{
+		Selector: fields.OneTermEqualSelector(indexers.TenantControlPlaneUsedDataStoreKey, ds.GetName()),
+	}); err != nil {
+		log.Error(err, "cannot retrieve list of the Tenant Control Plane using the following instance")
+
+		return reconcile.Result{}, err
+	}
+	// Updating the status with the list of Tenant Control Plane using the following Data Source
+	tcpSets := sets.NewString()
+	for _, tcp := range tcpList.Items {
+		tcpSets.Insert(getNamespacedName(tcp.GetNamespace(), tcp.GetName()).String())
+	}
+
+	ds.Status.UsedBy = tcpSets.List()
+
+	if err := r.client.Status().Update(ctx, ds); err != nil {
+		log.Error(err, "cannot update the status for the given instance")
+
+		return reconcile.Result{}, err
+	}
+	// Triggering the reconciliation of the Tenant Control Plane upon a Secret change
+	for _, i := range tcpList.Items {
+		tcp := i
+
+		r.TenantControlPlaneTrigger <- event.GenericEvent{Object: &tcp}
+	}
+
+	return reconcile.Result{}, nil
+}
+
+func (r *DataStore) InjectClient(client client.Client) error {
+	r.client = client
+
+	return nil
+}
+
+func (r *DataStore) SetupWithManager(mgr controllerruntime.Manager) error {
+	enqueueFn := func(tcp *kamajiv1alpha1.TenantControlPlane, limitingInterface workqueue.RateLimitingInterface) {
+		if dataStoreName := tcp.Status.Storage.DataStoreName; len(dataStoreName) > 0 {
+			limitingInterface.AddRateLimited(reconcile.Request{
+				NamespacedName: k8stypes.NamespacedName{
+					Name: dataStoreName,
+				},
+			})
+		}
+	}
+	//nolint:forcetypeassert
+	return controllerruntime.NewControllerManagedBy(mgr).
+		For(&kamajiv1alpha1.DataStore{}, builder.WithPredicates(
+			predicate.ResourceVersionChangedPredicate{},
+		)).
+		Watches(&source.Kind{Type: &kamajiv1alpha1.TenantControlPlane{}}, handler.Funcs{
+			CreateFunc: func(createEvent event.CreateEvent, limitingInterface workqueue.RateLimitingInterface) {
+				enqueueFn(createEvent.Object.(*kamajiv1alpha1.TenantControlPlane), limitingInterface)
+			},
+			UpdateFunc: func(updateEvent event.UpdateEvent, limitingInterface workqueue.RateLimitingInterface) {
+				enqueueFn(updateEvent.ObjectOld.(*kamajiv1alpha1.TenantControlPlane), limitingInterface)
+				enqueueFn(updateEvent.ObjectNew.(*kamajiv1alpha1.TenantControlPlane), limitingInterface)
+			},
+			DeleteFunc: func(deleteEvent event.DeleteEvent, limitingInterface workqueue.RateLimitingInterface) {
+				enqueueFn(deleteEvent.Object.(*kamajiv1alpha1.TenantControlPlane), limitingInterface)
+			},
+		}).
+		Complete(r)
+}

controllers/resources.go

@@ -1,10 +1,10 @@
 // Copyright 2022 Clastix Labs
 // SPDX-License-Identifier: Apache-2.0
+
 package controllers
 
 import (
 	"fmt"
-	"strings"
 
 	"github.com/go-logr/logr"
 	"github.com/google/uuid"
@@ -12,14 +12,10 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+	"github.com/clastix/kamaji/internal/datastore"
 	"github.com/clastix/kamaji/internal/resources"
+	ds "github.com/clastix/kamaji/internal/resources/datastore"
 	"github.com/clastix/kamaji/internal/resources/konnectivity"
-	"github.com/clastix/kamaji/internal/sql"
-	"github.com/clastix/kamaji/internal/types"
-)
-
-const (
-	separator = ","
 )
 
 type GroupResourceBuilderConfiguration struct {
@@ -27,7 +23,8 @@ type GroupResourceBuilderConfiguration struct {
 	log                 logr.Logger
 	tcpReconcilerConfig TenantControlPlaneReconcilerConfig
 	tenantControlPlane  kamajiv1alpha1.TenantControlPlane
-	DBConnection        sql.DBConnection
+	Connection          datastore.Connection
+	DataStore           kamajiv1alpha1.DataStore
 }
 
 type GroupDeleteableResourceBuilderConfiguration struct {
@@ -35,7 +32,7 @@ type GroupDeleteableResourceBuilderConfiguration struct {
 	log                 logr.Logger
 	tcpReconcilerConfig TenantControlPlaneReconcilerConfig
 	tenantControlPlane  kamajiv1alpha1.TenantControlPlane
-	DBConnection        sql.DBConnection
+	connection          datastore.Connection
 }
 
 // GetResources returns a list of resources that will be used to provide tenant control planes
@@ -53,57 +50,39 @@ func GetDeletableResources(config GroupDeleteableResourceBuilderConfiguration) [
 }
 
 func getDefaultResources(config GroupResourceBuilderConfiguration) []resources.Resource {
-	resources := append(getUpgradeResources(config.client, config.tenantControlPlane), getKubernetesServiceResources(config.client, config.tenantControlPlane)...)
-	resources = append(resources, getKubeadmConfigResources(config.client, config.tcpReconcilerConfig, config.tenantControlPlane)...)
-	resources = append(resources, getKubernetesCertificatesResources(config.client, config.log, config.tcpReconcilerConfig, config.tenantControlPlane)...)
-	resources = append(resources, getKubeconfigResources(config.client, config.log, config.tcpReconcilerConfig, config.tenantControlPlane)...)
-	resources = append(resources, getKubernetesStorageResources(config.client, config.log, config.tcpReconcilerConfig, config.DBConnection, config.tenantControlPlane)...)
-	resources = append(resources, getInternalKonnectivityResources(config.client, config.log, config.tcpReconcilerConfig, config.tenantControlPlane)...)
-	resources = append(resources, getKubernetesDeploymentResources(config.client, config.tcpReconcilerConfig, config.tenantControlPlane)...)
-	resources = append(resources, getKubernetesIngressResources(config.client, config.tenantControlPlane)...)
-	resources = append(resources, getKubeadmPhaseResources(config.client, config.log, config.tenantControlPlane)...)
-	resources = append(resources, getKubeadmAddonResources(config.client, config.log, config.tenantControlPlane)...)
-	resources = append(resources, getExternalKonnectivityResources(config.client, config.log, config.tcpReconcilerConfig, config.tenantControlPlane)...)
+	resources := append(getUpgradeResources(config.client), getKubernetesServiceResources(config.client)...)
+	resources = append(resources, getKubeadmConfigResources(config.client, getTmpDirectory(config.tcpReconcilerConfig.TmpBaseDirectory, config.tenantControlPlane), config.DataStore)...)
+	resources = append(resources, getKubernetesCertificatesResources(config.client, config.tcpReconcilerConfig, config.tenantControlPlane)...)
+	resources = append(resources, getKubeconfigResources(config.client, config.tcpReconcilerConfig, config.tenantControlPlane)...)
+	resources = append(resources, getKubernetesStorageResources(config.client, config.Connection, config.DataStore)...)
+	resources = append(resources, getInternalKonnectivityResources(config.client)...)
+	resources = append(resources, getKubernetesDeploymentResources(config.client, config.tcpReconcilerConfig, config.DataStore)...)
+	resources = append(resources, getKubernetesIngressResources(config.client)...)
+	resources = append(resources, getKubeadmPhaseResources(config.client)...)
+	resources = append(resources, getKubeadmAddonResources(config.client)...)
+	resources = append(resources, getExternalKonnectivityResources(config.client)...)
 
 	return resources
 }
 
 func getDefaultDeleteableResources(config GroupDeleteableResourceBuilderConfiguration) []resources.DeleteableResource {
-	switch config.tcpReconcilerConfig.ETCDStorageType {
-	case types.ETCD:
-		return []resources.DeleteableResource{
-			&resources.ETCDSetupResource{
-				Name:                  "etcd-setup",
-				Client:                config.client,
-				Log:                   config.log,
-				ETCDClientCertsSecret: getNamespacedName(config.tcpReconcilerConfig.ETCDClientSecretNamespace, config.tcpReconcilerConfig.ETCDClientSecretName),
-				ETCDCACertsSecret:     getNamespacedName(config.tcpReconcilerConfig.ETCDCASecretNamespace, config.tcpReconcilerConfig.ETCDCASecretName),
-				Endpoints:             getArrayFromString(config.tcpReconcilerConfig.ETCDEndpoints),
-			},
-		}
-	case types.KineMySQL:
-		return []resources.DeleteableResource{
-			&resources.SQLSetup{
-				Client:       config.client,
-				Name:         "sql-setup",
-				DBConnection: config.DBConnection,
-			},
-		}
-	default:
-		return []resources.DeleteableResource{}
+	return []resources.DeleteableResource{
+		&ds.Setup{
+			Client:     config.client,
+			Connection: config.connection,
+		},
 	}
 }
 
-func getUpgradeResources(c client.Client, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource {
+func getUpgradeResources(c client.Client) []resources.Resource {
 	return []resources.Resource{
 		&resources.KubernetesUpgrade{
-			Name:   "upgrade",
 			Client: c,
 		},
 	}
 }
 
-func getKubernetesServiceResources(c client.Client, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource {
+func getKubernetesServiceResources(c client.Client) []resources.Resource {
 	return []resources.Resource{
 		&resources.KubernetesServiceResource{
 			Client: c,
@@ -111,149 +90,107 @@ func getKubernetesServiceResources(c client.Client, tenantControlPlane kamajiv1a
 	}
 }
 
-func getKubeadmConfigResources(c client.Client, tcpReconcilerConfig TenantControlPlaneReconcilerConfig, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource {
+func getKubeadmConfigResources(c client.Client, tmpDirectory string, dataStore kamajiv1alpha1.DataStore) []resources.Resource {
+	var endpoints []string
+
+	switch dataStore.Spec.Driver {
+	case kamajiv1alpha1.EtcdDriver:
+		endpoints = dataStore.Spec.Endpoints
+	default:
+		endpoints = []string{"127.0.0.1:2379"}
+	}
+
 	return []resources.Resource{
 		&resources.KubeadmConfigResource{
-			Name:                   "kubeadmconfig",
-			ETCDs:                  getArrayFromString(tcpReconcilerConfig.ETCDEndpoints),
-			ETCDCompactionInterval: tcpReconcilerConfig.ETCDCompactionInterval,
-			Client:                 c,
-			TmpDirectory:           getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
+			ETCDs:        endpoints,
+			Client:       c,
+			TmpDirectory: tmpDirectory,
 		},
 	}
 }
 
-func getKubernetesCertificatesResources(c client.Client, log logr.Logger, tcpReconcilerConfig TenantControlPlaneReconcilerConfig, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource {
+func getKubernetesCertificatesResources(c client.Client, tcpReconcilerConfig TenantControlPlaneReconcilerConfig, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource {
 	return []resources.Resource{
 		&resources.CACertificate{
-			Name:         "ca",
 			Client:       c,
-			Log:          log,
 			TmpDirectory: getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
 		},
 		&resources.FrontProxyCACertificate{
-			Name:         "front-proxy-ca-certificate",
 			Client:       c,
-			Log:          log,
 			TmpDirectory: getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
 		},
 		&resources.SACertificate{
-			Name:         "sa-certificate",
 			Client:       c,
-			Log:          log,
 			TmpDirectory: getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
 		},
 		&resources.APIServerCertificate{
-			Name:         "api-server-certificate",
 			Client:       c,
-			Log:          log,
 			TmpDirectory: getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
 		},
 		&resources.APIServerKubeletClientCertificate{
-			Name:         "api-server-kubelet-client-certificate",
 			Client:       c,
-			Log:          log,
 			TmpDirectory: getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
 		},
 		&resources.FrontProxyClientCertificate{
-			Name:         "front-proxy-client-certificate",
 			Client:       c,
-			Log:          log,
 			TmpDirectory: getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
 		},
 	}
 }
 
-func getKubeconfigResources(c client.Client, log logr.Logger, tcpReconcilerConfig TenantControlPlaneReconcilerConfig, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource {
+func getKubeconfigResources(c client.Client, tcpReconcilerConfig TenantControlPlaneReconcilerConfig, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource {
 	return []resources.Resource{
 		&resources.KubeconfigResource{
 			Name:               "admin-kubeconfig",
 			Client:             c,
-			Log:                log,
 			KubeConfigFileName: resources.AdminKubeConfigFileName,
 			TmpDirectory:       getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
 		},
 		&resources.KubeconfigResource{
 			Name:               "controller-manager-kubeconfig",
 			Client:             c,
-			Log:                log,
 			KubeConfigFileName: resources.ControllerManagerKubeConfigFileName,
 			TmpDirectory:       getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
 		},
 		&resources.KubeconfigResource{
 			Name:               "scheduler-kubeconfig",
 			Client:             c,
-			Log:                log,
 			KubeConfigFileName: resources.SchedulerKubeConfigFileName,
 			TmpDirectory:       getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
 		},
 	}
 }
 
-func getKubernetesStorageResources(c client.Client, log logr.Logger, tcpReconcilerConfig TenantControlPlaneReconcilerConfig, dbConnection sql.DBConnection, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource {
-	switch tcpReconcilerConfig.ETCDStorageType {
-	case types.ETCD:
-		return []resources.Resource{
-			&resources.ETCDCACertificatesResource{
-				Name:                  "etcd-ca-certificates",
-				Client:                c,
-				Log:                   log,
-				ETCDCASecretName:      tcpReconcilerConfig.ETCDCASecretName,
-				ETCDCASecretNamespace: tcpReconcilerConfig.ETCDCASecretNamespace,
-			},
-			&resources.ETCDCertificatesResource{
-				Name:   "etcd-certificates",
-				Client: c,
-				Log:    log,
-			},
-			&resources.ETCDSetupResource{
-				Name:                  "etcd-setup",
-				Client:                c,
-				Log:                   log,
-				ETCDClientCertsSecret: getNamespacedName(tcpReconcilerConfig.ETCDClientSecretNamespace, tcpReconcilerConfig.ETCDClientSecretName),
-				ETCDCACertsSecret:     getNamespacedName(tcpReconcilerConfig.ETCDCASecretNamespace, tcpReconcilerConfig.ETCDCASecretName),
-				Endpoints:             getArrayFromString(tcpReconcilerConfig.ETCDEndpoints),
-			},
-		}
-	case types.KineMySQL:
-		return []resources.Resource{
-			&resources.SQLStorageConfig{
-				Client: c,
-				Name:   "sql-config",
-				Host:   dbConnection.GetHost(),
-				Port:   dbConnection.GetPort(),
-			},
-			&resources.SQLSetup{
-				Client:       c,
-				Name:         "sql-setup",
-				DBConnection: dbConnection,
-			},
-			&resources.SQLCertificate{
-				Client:                   c,
-				Name:                     "sql-certificate",
-				StorageType:              tcpReconcilerConfig.ETCDStorageType,
-				SQLConfigSecretName:      tcpReconcilerConfig.KineMySQLSecretName,
-				SQLConfigSecretNamespace: tcpReconcilerConfig.KineMySQLSecretNamespace,
-			},
-		}
-	default:
-		return []resources.Resource{}
-	}
-}
-
-func getKubernetesDeploymentResources(c client.Client, tcpReconcilerConfig TenantControlPlaneReconcilerConfig, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource {
+func getKubernetesStorageResources(c client.Client, dbConnection datastore.Connection, datastore kamajiv1alpha1.DataStore) []resources.Resource {
 	return []resources.Resource{
-		&resources.KubernetesDeploymentResource{
-			Client:                 c,
-			ETCDEndpoints:          getArrayFromString(tcpReconcilerConfig.ETCDEndpoints),
-			ETCDCompactionInterval: tcpReconcilerConfig.ETCDCompactionInterval,
-			ETCDStorageType:        tcpReconcilerConfig.ETCDStorageType,
-			KineContainerImage:     tcpReconcilerConfig.KineContainerImage,
+		&ds.Config{
+			Client:     c,
+			ConnString: dbConnection.GetConnectionString(),
+			DataStore:  datastore,
+		},
+		&ds.Setup{
+			Client:     c,
+			Connection: dbConnection,
+			DataStore:  datastore,
+		},
+		&ds.Certificate{
+			Client:    c,
+			DataStore: datastore,
 		},
 	}
 }
 
-func getKubernetesIngressResources(c client.Client, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource {
+func getKubernetesDeploymentResources(c client.Client, tcpReconcilerConfig TenantControlPlaneReconcilerConfig, dataStore kamajiv1alpha1.DataStore) []resources.Resource {
+	return []resources.Resource{
+		&resources.KubernetesDeploymentResource{
+			Client:             c,
+			DataStore:          dataStore,
+			KineContainerImage: tcpReconcilerConfig.KineContainerImage,
+		},
+	}
+}
+
+func getKubernetesIngressResources(c client.Client) []resources.Resource {
 	return []resources.Resource{
 		&resources.KubernetesIngressResource{
 			Client: c,
@@ -261,96 +198,59 @@ func getKubernetesIngressResources(c client.Client, tenantControlPlane kamajiv1a
 	}
 }
 
-func getKubeadmPhaseResources(c client.Client, log logr.Logger, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource {
+func getKubeadmPhaseResources(c client.Client) []resources.Resource {
 	return []resources.Resource{
 		&resources.KubeadmPhase{
 			Name:   "upload-config-kubeadm",
 			Client: c,
-			Log:    log,
 			Phase:  resources.PhaseUploadConfigKubeadm,
 		},
 		&resources.KubeadmPhase{
 			Name:   "upload-config-kubelet",
 			Client: c,
-			Log:    log,
 			Phase:  resources.PhaseUploadConfigKubelet,
 		},
 		&resources.KubeadmPhase{
 			Name:   "bootstrap-token",
 			Client: c,
-			Log:    log,
 			Phase:  resources.PhaseBootstrapToken,
 		},
 	}
 }
 
-func getKubeadmAddonResources(c client.Client, log logr.Logger, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource {
+func getKubeadmAddonResources(c client.Client) []resources.Resource {
 	return []resources.Resource{
 		&resources.KubeadmAddonResource{
 			Name:         "coredns",
 			Client:       c,
-			Log:          log,
 			KubeadmAddon: resources.AddonCoreDNS,
 		},
 		&resources.KubeadmAddonResource{
 			Name:         "kubeproxy",
 			Client:       c,
-			Log:          log,
 			KubeadmAddon: resources.AddonKubeProxy,
 		},
 	}
 }
 
-func getExternalKonnectivityResources(c client.Client, log logr.Logger, tcpReconcilerConfig TenantControlPlaneReconcilerConfig, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource {
+func getExternalKonnectivityResources(c client.Client) []resources.Resource {
 	return []resources.Resource{
-		&konnectivity.ServiceAccountResource{
-			Client: c,
-			Name:   "konnectivity-sa",
-		},
-		&konnectivity.ClusterRoleBindingResource{
-			Client: c,
-			Name:   "konnectivity-clusterrolebinding",
-		},
-		&konnectivity.KubernetesDeploymentResource{
-			Client: c,
-			Name:   "konnectivity-deployment",
-		},
-		&konnectivity.ServiceResource{
-			Client: c,
-			Name:   "konnectivity-service",
-		},
-		&konnectivity.Agent{
-			Client: c,
-			Name:   "konnectivity-agent",
-		},
+		&konnectivity.ServiceAccountResource{Client: c},
+		&konnectivity.ClusterRoleBindingResource{Client: c},
+		&konnectivity.KubernetesDeploymentResource{Client: c},
+		&konnectivity.ServiceResource{Client: c},
+		&konnectivity.Agent{Client: c},
 	}
 }
 
-func getInternalKonnectivityResources(c client.Client, log logr.Logger, tcpReconcilerConfig TenantControlPlaneReconcilerConfig, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource {
+func getInternalKonnectivityResources(c client.Client) []resources.Resource {
 	return []resources.Resource{
-		&konnectivity.EgressSelectorConfigurationResource{
-			Client: c,
-			Name:   "konnectivity-egress-selector-configuration",
-		},
-		&konnectivity.CertificateResource{
-			Client: c,
-			Log:    log,
-			Name:   "konnectivity-certificate",
-		},
-		&konnectivity.KubeconfigResource{
-			Client: c,
-			Name:   "konnectivity-kubeconfig",
-		},
+		&konnectivity.EgressSelectorConfigurationResource{Client: c},
+		&konnectivity.CertificateResource{Client: c},
+		&konnectivity.KubeconfigResource{Client: c},
 	}
 }
 
-func getArrayFromString(s string) []string {
-	var a []string
-	a = append(a, strings.Split(s, separator)...)
-
-	return a
-}
-
 func getNamespacedName(namespace string, name string) k8stypes.NamespacedName {
 	return k8stypes.NamespacedName{Namespace: namespace, Name: name}
 }

controllers/storage.go

@@ -8,50 +8,97 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"fmt"
+	"net"
+	"strconv"
 
-	corev1 "k8s.io/api/core/v1"
-	k8stypes "k8s.io/apimachinery/pkg/types"
+	"github.com/pkg/errors"
 
-	"github.com/clastix/kamaji/internal/sql"
-	"github.com/clastix/kamaji/internal/types"
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+	"github.com/clastix/kamaji/internal/datastore"
 )
 
-func (r *TenantControlPlaneReconciler) getStorageConnection(ctx context.Context) (sql.DBConnection, error) {
-	// TODO: https://github.com/clastix/kamaji/issues/67
-	switch r.Config.ETCDStorageType {
-	case types.KineMySQL:
-		secret := &corev1.Secret{}
-		namespacedName := k8stypes.NamespacedName{Namespace: r.Config.KineMySQLSecretNamespace, Name: r.Config.KineMySQLSecretName}
-		if err := r.Client.Get(ctx, namespacedName, secret); err != nil {
-			return nil, err
-		}
+func (r *TenantControlPlaneReconciler) getStorageConnection(ctx context.Context, ds kamajiv1alpha1.DataStore) (datastore.Connection, error) {
+	ca, err := ds.Spec.TLSConfig.CertificateAuthority.Certificate.GetContent(ctx, r.Client)
+	if err != nil {
+		return nil, err
+	}
 
-		rootCAs := x509.NewCertPool()
-		if ok := rootCAs.AppendCertsFromPEM(secret.Data["ca.crt"]); !ok {
-			return nil, fmt.Errorf("error creating root ca for mysql db connector")
-		}
+	crt, err := ds.Spec.TLSConfig.ClientCertificate.Certificate.GetContent(ctx, r.Client)
+	if err != nil {
+		return nil, err
+	}
 
-		certificate, err := tls.X509KeyPair(secret.Data["server.crt"], secret.Data["server.key"])
+	key, err := ds.Spec.TLSConfig.ClientCertificate.PrivateKey.GetContent(ctx, r.Client)
+	if err != nil {
+		return nil, err
+	}
+
+	rootCAs := x509.NewCertPool()
+	if ok := rootCAs.AppendCertsFromPEM(ca); !ok {
+		return nil, fmt.Errorf("error create root CA for the DB connector")
+	}
+
+	certificate, err := tls.X509KeyPair(crt, key)
+	if err != nil {
+		return nil, errors.Wrap(err, "cannot retrieve x.509 key pair from the Kine Secret")
+	}
+
+	var user, password string
+	if auth := ds.Spec.BasicAuth; auth != nil {
+		u, err := auth.Username.GetContent(ctx, r.Client)
 		if err != nil {
 			return nil, err
 		}
+		user = string(u)
 
-		return sql.GetDBConnection(
-			sql.ConnectionConfig{
-				SQLDriver: sql.MySQL,
-				User:      "root",
-				Password:  string(secret.Data["MYSQL_ROOT_PASSWORD"]),
-				Host:      r.Config.KineMySQLHost,
-				Port:      r.Config.KineMySQLPort,
-				DBName:    "mysql",
-				TLSConfig: &tls.Config{
-					ServerName:   r.Config.KineMySQLHost,
-					RootCAs:      rootCAs,
-					Certificates: []tls.Certificate{certificate},
-				},
-			},
-		)
+		p, err := auth.Password.GetContent(ctx, r.Client)
+		if err != nil {
+			return nil, err
+		}
+		password = string(p)
+	}
+
+	eps := make([]datastore.ConnectionEndpoint, 0, len(ds.Spec.Endpoints))
+
+	for _, ep := range ds.Spec.Endpoints {
+		host, stringPort, err := net.SplitHostPort(ep)
+		if err != nil {
+			return nil, errors.Wrap(err, "cannot retrieve host-port pair from DataStore endpoints")
+		}
+
+		port, err := strconv.Atoi(stringPort)
+		if err != nil {
+			return nil, errors.Wrap(err, "cannot convert port from string for the given DataStore")
+		}
+
+		eps = append(eps, datastore.ConnectionEndpoint{
+			Host: host,
+			Port: port,
+		})
+	}
+
+	cc := datastore.ConnectionConfig{
+		User:      user,
+		Password:  password,
+		Endpoints: eps,
+		TLSConfig: &tls.Config{
+			RootCAs:      rootCAs,
+			Certificates: []tls.Certificate{certificate},
+		},
+	}
+
+	switch ds.Spec.Driver {
+	case kamajiv1alpha1.KineMySQLDriver:
+		cc.TLSConfig.ServerName = cc.Endpoints[0].Host
+
+		return datastore.NewMySQLConnection(cc)
+	case kamajiv1alpha1.KinePostgreSQLDriver:
+		cc.TLSConfig.ServerName = cc.Endpoints[0].Host
+		//nolint:contextcheck
+		return datastore.NewPostgreSQLConnection(cc)
+	case kamajiv1alpha1.EtcdDriver:
+		return datastore.NewETCDConnection(cc)
 	default:
-		return nil, nil
+		return nil, fmt.Errorf("%s is not a valid driver", ds.Spec.Driver)
 	}
 }

controllers/tcp_channel.go

@@ -0,0 +1,8 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package controllers
+
+import "sigs.k8s.io/controller-runtime/pkg/event"
+
+type TenantControlPlaneChannel chan event.GenericEvent

controllers/tenantcontrolplane_controller.go

@@ -7,61 +7,54 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/pkg/errors"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	networkingv1 "k8s.io/api/networking/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime"
 	k8stypes "k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/util/workqueue"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+	"sigs.k8s.io/controller-runtime/pkg/event"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
 	"sigs.k8s.io/controller-runtime/pkg/log"
+	"sigs.k8s.io/controller-runtime/pkg/source"
 
 	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
 	kamajierrors "github.com/clastix/kamaji/internal/errors"
 	"github.com/clastix/kamaji/internal/resources"
-	"github.com/clastix/kamaji/internal/sql"
-	"github.com/clastix/kamaji/internal/types"
 )
 
 const (
-	finalizer = "finalizer.kamaji.clastix.io"
+	tenantControlPlaneFinalizer = "finalizer.kamaji.clastix.io"
 )
 
 // TenantControlPlaneReconciler reconciles a TenantControlPlane object.
 type TenantControlPlaneReconciler struct {
 	client.Client
-	Scheme *runtime.Scheme
-	Config TenantControlPlaneReconcilerConfig
+	Scheme      *runtime.Scheme
+	Config      TenantControlPlaneReconcilerConfig
+	TriggerChan TenantControlPlaneChannel
 }
 
 // TenantControlPlaneReconcilerConfig gives the necessary configuration for TenantControlPlaneReconciler.
 type TenantControlPlaneReconcilerConfig struct {
-	ETCDStorageType           types.ETCDStorageType
-	ETCDCASecretName          string
-	ETCDCASecretNamespace     string
-	ETCDClientSecretName      string
-	ETCDClientSecretNamespace string
-	ETCDEndpoints             string
-	ETCDCompactionInterval    string
-	TmpBaseDirectory          string
-	DBConnection              sql.DBConnection
-	KineMySQLSecretName       string
-	KineMySQLSecretNamespace  string
-	KineMySQLHost             string
-	KineMySQLPort             int
-	KineContainerImage        string
+	DefaultDataStoreName string
+	KineContainerImage   string
+	TmpBaseDirectory     string
 }
 
 //+kubebuilder:rbac:groups=kamaji.clastix.io,resources=tenantcontrolplanes,verbs=get;list;watch;create;update;patch;delete
 //+kubebuilder:rbac:groups=kamaji.clastix.io,resources=tenantcontrolplanes/status,verbs=get;update;patch
 //+kubebuilder:rbac:groups=kamaji.clastix.io,resources=tenantcontrolplanes/finalizers,verbs=update
-// +kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch;create;update;patch;delete
-// +kubebuilder:rbac:groups=core,resources=configmaps,verbs=get;list;watch;create;update;patch;delete
-// +kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;update;patch;delete
-// +kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete
-// +kubebuilder:rbac:groups=networking.k8s.io,resources=ingresses,verbs=get;list;watch;create;update;patch;delete
+//+kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch;create;update;patch;delete
+//+kubebuilder:rbac:groups=core,resources=configmaps,verbs=get;list;watch;create;update;patch;delete
+//+kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;update;patch;delete
+//+kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete
+//+kubebuilder:rbac:groups=networking.k8s.io,resources=ingresses,verbs=get;list;watch;create;update;patch;delete
 
 func (r *TenantControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
 	log := log.FromContext(ctx)
@@ -69,6 +62,8 @@ func (r *TenantControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R
 	tenantControlPlane := &kamajiv1alpha1.TenantControlPlane{}
 	isTenantControlPlane, err := r.getTenantControlPlane(ctx, req.NamespacedName, tenantControlPlane)
 	if err != nil {
+		log.Error(err, "cannot retrieve the required instance")
+
 		return ctrl.Result{}, err
 	}
 	if !isTenantControlPlane {
@@ -76,23 +71,26 @@ func (r *TenantControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R
 	}
 
 	markedToBeDeleted := tenantControlPlane.GetDeletionTimestamp() != nil
-	hasFinalizer := hasFinalizer(*tenantControlPlane)
+	hasFinalizer := controllerutil.ContainsFinalizer(tenantControlPlane, tenantControlPlaneFinalizer)
 
 	if markedToBeDeleted && !hasFinalizer {
 		return ctrl.Result{}, nil
 	}
-
-	dbConnection, err := r.getStorageConnection(ctx)
+	// Retrieving the DataStore to use for the current reconciliation
+	ds, err := r.dataStore(ctx, tenantControlPlane)
 	if err != nil {
+		log.Error(err, "cannot retrieve the DataStore for the given instance")
+
 		return ctrl.Result{}, err
 	}
-	defer func() {
-		// TODO: Currently, etcd is not accessed using this dbConnection. For that reason we need this check
-		// Check: https://github.com/clastix/kamaji/issues/67
-		if dbConnection != nil {
-			dbConnection.Close()
-		}
-	}()
+
+	dsConnection, err := r.getStorageConnection(ctx, *ds)
+	if err != nil {
+		log.Error(err, "cannot generate the DataStore connection for the given instance")
+
+		return ctrl.Result{}, err
+	}
+	defer dsConnection.Close()
 
 	if markedToBeDeleted {
 		log.Info("marked for deletion, performing clean-up")
@@ -102,12 +100,14 @@ func (r *TenantControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R
 			log:                 log,
 			tcpReconcilerConfig: r.Config,
 			tenantControlPlane:  *tenantControlPlane,
-			DBConnection:        dbConnection,
+			connection:          dsConnection,
 		}
 		registeredDeletableResources := GetDeletableResources(groupDeleteableResourceBuilderConfiguration)
 
 		for _, resource := range registeredDeletableResources {
-			if err := resources.HandleDeletion(ctx, resource, tenantControlPlane); err != nil {
+			if err = resources.HandleDeletion(ctx, resource, tenantControlPlane); err != nil {
+				log.Error(err, "resource deletion failed", "resource", resource.GetName())
+
 				return ctrl.Result{}, err
 			}
 		}
@@ -115,7 +115,9 @@ func (r *TenantControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R
 		if hasFinalizer {
 			log.Info("removing finalizer")
 
-			if err := r.RemoveFinalizer(ctx, tenantControlPlane); err != nil {
+			if err = r.RemoveFinalizer(ctx, tenantControlPlane); err != nil {
+				log.Error(err, "cannot remove the finalizer for the given resource")
+
 				return ctrl.Result{}, err
 			}
 		}
@@ -134,7 +136,8 @@ func (r *TenantControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R
 		log:                 log,
 		tcpReconcilerConfig: r.Config,
 		tenantControlPlane:  *tenantControlPlane,
-		DBConnection:        dbConnection,
+		DataStore:           *ds,
+		Connection:          dsConnection,
 	}
 	registeredResources := GetResources(groupResourceBuilderConfiguration)
 
@@ -147,6 +150,8 @@ func (r *TenantControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R
 				return ctrl.Result{Requeue: true}, nil
 			}
 
+			log.Error(err, "handling of resource failed", "resource", resource.GetName())
+
 			return ctrl.Result{}, err
 		}
 
@@ -155,6 +160,8 @@ func (r *TenantControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R
 		}
 
 		if err := r.updateStatus(ctx, req.NamespacedName, resource); err != nil {
+			log.Error(err, "update of the resource failed", "resource", resource.GetName())
+
 			return ctrl.Result{}, err
 		}
 
@@ -171,6 +178,14 @@ func (r *TenantControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R
 // SetupWithManager sets up the controller with the Manager.
 func (r *TenantControlPlaneReconciler) SetupWithManager(mgr ctrl.Manager) error {
 	return ctrl.NewControllerManagedBy(mgr).
+		Watches(&source.Channel{Source: r.TriggerChan}, handler.Funcs{GenericFunc: func(genericEvent event.GenericEvent, limitingInterface workqueue.RateLimitingInterface) {
+			limitingInterface.AddRateLimited(ctrl.Request{
+				NamespacedName: k8stypes.NamespacedName{
+					Namespace: genericEvent.Object.GetNamespace(),
+					Name:      genericEvent.Object.GetName(),
+				},
+			})
+		}}).
 		For(&kamajiv1alpha1.TenantControlPlane{}).
 		Owns(&corev1.Secret{}).
 		Owns(&corev1.ConfigMap{}).
@@ -214,24 +229,34 @@ func (r *TenantControlPlaneReconciler) updateStatus(ctx context.Context, namespa
 	return nil
 }
 
-func hasFinalizer(tenantControlPlane kamajiv1alpha1.TenantControlPlane) bool {
-	for _, f := range tenantControlPlane.GetFinalizers() {
-		if f == finalizer {
-			return true
-		}
-	}
-
-	return false
-}
-
 func (r *TenantControlPlaneReconciler) AddFinalizer(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
-	controllerutil.AddFinalizer(tenantControlPlane, finalizer)
+	controllerutil.AddFinalizer(tenantControlPlane, tenantControlPlaneFinalizer)
 
 	return r.Update(ctx, tenantControlPlane)
 }
 
 func (r *TenantControlPlaneReconciler) RemoveFinalizer(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
-	controllerutil.RemoveFinalizer(tenantControlPlane, finalizer)
+	controllerutil.RemoveFinalizer(tenantControlPlane, tenantControlPlaneFinalizer)
 
 	return r.Update(ctx, tenantControlPlane)
 }
+
+// dataStore retrieves the override DataStore for the given Tenant Control Plane if specified,
+// otherwise fallback to the default one specified in the Kamaji setup.
+func (r *TenantControlPlaneReconciler) dataStore(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (*kamajiv1alpha1.DataStore, error) {
+	dataStoreName := tenantControlPlane.Spec.DataStore
+	if len(dataStoreName) == 0 {
+		dataStoreName = r.Config.DefaultDataStoreName
+	}
+
+	if statusDataStore := tenantControlPlane.Status.Storage.DataStoreName; len(statusDataStore) > 0 && dataStoreName != statusDataStore {
+		return nil, fmt.Errorf("migration from a DataStore (current: %s) to another one (desired: %s) is not yet supported", statusDataStore, dataStoreName)
+	}
+
+	ds := &kamajiv1alpha1.DataStore{}
+	if err := r.Client.Get(ctx, k8stypes.NamespacedName{Name: dataStoreName}, ds); err != nil {
+		return nil, errors.Wrap(err, "cannot retrieve *kamajiv1alpha.DataStore object")
+	}
+
+	return ds, nil
+}

deploy/Makefile

@@ -1,16 +0,0 @@
-include etcd/Makefile
-
-deploy_path := $(patsubst %/,%,$(dir $(abspath $(lastword $(MAKEFILE_LIST)))))
-
-.DEFAULT_GOAL := kamaji
-
-.PHONY: etcd-cluster
-reqs: etcd-cluster
-
-.PHONY: kamaji
-kamaji: reqs
-	@kubectl apply -f $(deploy_path)/../../config/install.yaml
-
-.PHONY: destroy
-destroy: etcd-certificates/cleanup
-	@kubectl delete -f $(deploy_path)/../../config/install.yaml

deploy/README.md

@@ -1,21 +0,0 @@
-# Deploy Kamaji
-
-## Quickstart with KinD
-
-```sh
-make -C kind
-```
-
-## Multi-tenant etcd cluster
-
-> This assumes you already have a running Kubernetes cluster and kubeconfig.
-
-```sh
-make -C etcd
-```
-
-## Multi-tenant MySQL-MariaDB cluster
-
-> This assumes you already have a running Kubernetes cluster and kubeconfig.
-
-Read [this](./mysql/README.md) in order to know more about.

deploy/calico-cni/calico-azure.yaml

@@ -1,680 +0,0 @@
----
-# Source: calico/templates/calico-config.yaml
-# This ConfigMap is used to configure a self-hosted Calico installation.
-kind: ConfigMap
-apiVersion: v1
-metadata:
-  name: calico-config
-  namespace: kube-system
-data:
-  # Typha is disabled.
-  typha_service_name: "none"
-  # Configure the backend to use.
-  calico_backend: "vxlan"
-
-  # Configure the MTU to use for workload interfaces and tunnels.
-  # By default, MTU is auto-detected, and explicitly setting this field should not be required.
-  # You can override auto-detection by providing a non-zero value.
-  veth_mtu: "0"
-
-  # The CNI network configuration to install on each node. The special
-  # values in this config will be automatically populated.
-  cni_network_config: |-
-    {
-      "name": "k8s-pod-network",
-      "cniVersion": "0.3.1",
-      "plugins": [
-        {
-          "type": "calico",
-          "log_level": "info",
-          "log_file_path": "/var/log/calico/cni/cni.log",
-          "datastore_type": "kubernetes",
-          "nodename": "__KUBERNETES_NODE_NAME__",
-          "mtu": __CNI_MTU__,
-          "ipam": {
-              "type": "calico-ipam"
-          },
-          "policy": {
-              "type": "k8s"
-          },
-          "kubernetes": {
-              "kubeconfig": "__KUBECONFIG_FILEPATH__"
-          }
-        },
-        {
-          "type": "portmap",
-          "snat": true,
-          "capabilities": {"portMappings": true}
-        },
-        {
-          "type": "bandwidth",
-          "capabilities": {"bandwidth": true}
-        }
-      ]
-    }
-
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: calico-node
-rules:
-- apiGroups:
-  - ""
-  resources:
-  - pods
-  - nodes
-  - namespaces
-  verbs:
-  - get
-- apiGroups:
-  - discovery.k8s.io
-  resources:
-  - endpointslices
-  verbs:
-  - watch
-  - list
-- apiGroups:
-  - ""
-  resources:
-  - endpoints
-  - services
-  verbs:
-  - watch
-  - list
-  - get
-- apiGroups:
-  - ""
-  resources:
-  - configmaps
-  verbs:
-  - get
-- apiGroups:
-  - ""
-  resources:
-  - nodes/status
-  verbs:
-  - patch
-  - update
-- apiGroups:
-  - networking.k8s.io
-  resources:
-  - networkpolicies
-  verbs:
-  - watch
-  - list
-- apiGroups:
-  - ""
-  resources:
-  - pods
-  - namespaces
-  - serviceaccounts
-  verbs:
-  - list
-  - watch
-- apiGroups:
-  - ""
-  resources:
-  - pods/status
-  verbs:
-  - patch
-- apiGroups:
-  - crd.projectcalico.org
-  resources:
-  - globalfelixconfigs
-  - felixconfigurations
-  - bgppeers
-  - globalbgpconfigs
-  - bgpconfigurations
-  - ippools
-  - ipamblocks
-  - globalnetworkpolicies
-  - globalnetworksets
-  - networkpolicies
-  - networksets
-  - clusterinformations
-  - hostendpoints
-  - blockaffinities
-  verbs:
-  - get
-  - list
-  - watch
-- apiGroups:
-  - crd.projectcalico.org
-  resources:
-  - ippools
-  - felixconfigurations
-  - clusterinformations
-  verbs:
-  - create
-  - update
-- apiGroups:
-  - ""
-  resources:
-  - nodes
-  verbs:
-  - get
-  - list
-  - watch
-- apiGroups:
-  - crd.projectcalico.org
-  resources:
-  - bgpconfigurations
-  - bgppeers
-  verbs:
-  - create
-  - update
-- apiGroups:
-  - crd.projectcalico.org
-  resources:
-  - blockaffinities
-  - ipamblocks
-  - ipamhandles
-  verbs:
-  - get
-  - list
-  - create
-  - update
-  - delete
-- apiGroups:
-  - crd.projectcalico.org
-  resources:
-  - ipamconfigs
-  verbs:
-  - get
-- apiGroups:
-  - crd.projectcalico.org
-  resources:
-  - blockaffinities
-  verbs:
-  - watch
-- apiGroups:
-  - apps
-  resources:
-  - daemonsets
-  verbs:
-  - get
-
----
-
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: calico-kube-controllers
-rules:
-- apiGroups:
-  - ""
-  resources:
-  - nodes
-  verbs:
-  - watch
-  - list
-  - get
-- apiGroups:
-  - ""
-  resources:
-  - pods
-  verbs:
-  - get
-  - list
-  - watch
-- apiGroups:
-  - crd.projectcalico.org
-  resources:
-  - ippools
-  verbs:
-  - list
-- apiGroups:
-  - crd.projectcalico.org
-  resources:
-  - blockaffinities
-  - ipamblocks
-  - ipamhandles
-  verbs:
-  - get
-  - list
-  - create
-  - update
-  - delete
-  - watch
-- apiGroups:
-  - crd.projectcalico.org
-  resources:
-  - hostendpoints
-  verbs:
-  - get
-  - list
-  - create
-  - update
-  - delete
-- apiGroups:
-  - crd.projectcalico.org
-  resources:
-  - clusterinformations
-  verbs:
-  - get
-  - create
-  - update
-- apiGroups:
-  - crd.projectcalico.org
-  resources:
-  - kubecontrollersconfigurations
-  verbs:
-  - get
-  - create
-  - update
-  - watch
-
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: calico-node
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: calico-node
-subjects:
-- kind: ServiceAccount
-  name: calico-node
-  namespace: kube-system
-
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: calico-kube-controllers
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: calico-kube-controllers
-subjects:
-- kind: ServiceAccount
-  name: calico-kube-controllers
-  namespace: kube-system
----
-# Source: calico/templates/calico-node.yaml
-# This manifest installs the calico-node container, as well
-# as the CNI plugins and network config on
-# each master and worker node in a Kubernetes cluster.
-kind: DaemonSet
-apiVersion: apps/v1
-metadata:
-  name: calico-node
-  namespace: kube-system
-  labels:
-    k8s-app: calico-node
-spec:
-  selector:
-    matchLabels:
-      k8s-app: calico-node
-  updateStrategy:
-    type: RollingUpdate
-    rollingUpdate:
-      maxUnavailable: 1
-  template:
-    metadata:
-      labels:
-        k8s-app: calico-node
-    spec:
-      nodeSelector:
-        kubernetes.io/os: linux
-      hostNetwork: true
-      tolerations:
-        # Make sure calico-node gets scheduled on all nodes.
-        - effect: NoSchedule
-          operator: Exists
-        # Mark the pod as a critical add-on for rescheduling.
-        - key: CriticalAddonsOnly
-          operator: Exists
-        - effect: NoExecute
-          operator: Exists
-      serviceAccountName: calico-node
-      # Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force
-      # deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods.
-      terminationGracePeriodSeconds: 0
-      priorityClassName: system-node-critical
-      initContainers:
-        # This container performs upgrade from host-local IPAM to calico-ipam.
-        # It can be deleted if this is a fresh installation, or if you have already
-        # upgraded to use calico-ipam.
-        - name: upgrade-ipam
-          image: docker.io/calico/cni:v3.20.0
-          command: ["/opt/cni/bin/calico-ipam", "-upgrade"]
-          envFrom:
-          - configMapRef:
-              # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode.
-              name: kubernetes-services-endpoint
-              optional: true
-          env:
-            - name: KUBERNETES_NODE_NAME
-              valueFrom:
-                fieldRef:
-                  fieldPath: spec.nodeName
-            - name: CALICO_NETWORKING_BACKEND
-              valueFrom:
-                configMapKeyRef:
-                  name: calico-config
-                  key: calico_backend
-          volumeMounts:
-            - mountPath: /var/lib/cni/networks
-              name: host-local-net-dir
-            - mountPath: /host/opt/cni/bin
-              name: cni-bin-dir
-          securityContext:
-            privileged: true
-        # This container installs the CNI binaries
-        # and CNI network config file on each node.
-        - name: install-cni
-          image: docker.io/calico/cni:v3.20.0
-          command: ["/opt/cni/bin/install"]
-          envFrom:
-          - configMapRef:
-              # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode.
-              name: kubernetes-services-endpoint
-              optional: true
-          env:
-            # Name of the CNI config file to create.
-            - name: CNI_CONF_NAME
-              value: "10-calico.conflist"
-            # The CNI network config to install on each node.
-            - name: CNI_NETWORK_CONFIG
-              valueFrom:
-                configMapKeyRef:
-                  name: calico-config
-                  key: cni_network_config
-            # Set the hostname based on the k8s node name.
-            - name: KUBERNETES_NODE_NAME
-              valueFrom:
-                fieldRef:
-                  fieldPath: spec.nodeName
-            # CNI MTU Config variable
-            - name: CNI_MTU
-              valueFrom:
-                configMapKeyRef:
-                  name: calico-config
-                  key: veth_mtu
-            # Prevents the container from sleeping forever.
-            - name: SLEEP
-              value: "false"
-          volumeMounts:
-            - mountPath: /host/opt/cni/bin
-              name: cni-bin-dir
-            - mountPath: /host/etc/cni/net.d
-              name: cni-net-dir
-          securityContext:
-            privileged: true
-        # Adds a Flex Volume Driver that creates a per-pod Unix Domain Socket to allow Dikastes
-        # to communicate with Felix over the Policy Sync API.
-        - name: flexvol-driver
-          image: docker.io/calico/pod2daemon-flexvol:v3.20.0
-          volumeMounts:
-          - name: flexvol-driver-host
-            mountPath: /host/driver
-          securityContext:
-            privileged: true
-      containers:
-        # Runs calico-node container on each Kubernetes node. This
-        # container programs network policy and routes on each
-        # host.
-        - name: calico-node
-          image: docker.io/calico/node:v3.20.0
-          envFrom:
-          - configMapRef:
-              # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode.
-              name: kubernetes-services-endpoint
-              optional: true
-          env:
-            # Use Kubernetes API as the backing datastore.
-            - name: DATASTORE_TYPE
-              value: "kubernetes"
-            # Wait for the datastore.
-            - name: WAIT_FOR_DATASTORE
-              value: "true"
-            # Set based on the k8s node name.
-            - name: NODENAME
-              valueFrom:
-                fieldRef:
-                  fieldPath: spec.nodeName
-            # Choose the backend to use.
-            - name: CALICO_NETWORKING_BACKEND
-              valueFrom:
-                configMapKeyRef:
-                  name: calico-config
-                  key: calico_backend
-            # Cluster type to identify the deployment type
-            - name: CLUSTER_TYPE
-              value: "k8s"
-            # Auto-detect the BGP IP address.
-            - name: IP
-              value: "autodetect"
-            # Enable IPIP
-            - name: CALICO_IPV4POOL_IPIP
-              value: "Never"
-            # Enable or Disable VXLAN on the default IP pool.
-            - name: CALICO_IPV4POOL_VXLAN
-              value: "Always"
-            # Set MTU for tunnel device used if ipip is enabled
-            - name: FELIX_IPINIPMTU
-              valueFrom:
-                configMapKeyRef:
-                  name: calico-config
-                  key: veth_mtu
-            # Set MTU for the VXLAN tunnel device.
-            - name: FELIX_VXLANMTU
-              valueFrom:
-                configMapKeyRef:
-                  name: calico-config
-                  key: veth_mtu
-            # Set MTU for the Wireguard tunnel device.
-            - name: FELIX_WIREGUARDMTU
-              valueFrom:
-                configMapKeyRef:
-                  name: calico-config
-                  key: veth_mtu
-            # The default IPv4 pool to create on startup if none exists. Pod IPs will be
-            # chosen from this range. Changing this value after installation will have
-            # no effect. This should fall within `--cluster-cidr`.
-            - name: CALICO_IPV4POOL_CIDR
-              value: "10.36.0.0/16"
-            # Disable file logging so `kubectl logs` works.
-            - name: CALICO_DISABLE_FILE_LOGGING
-              value: "true"
-            # Set Felix endpoint to host default action to ACCEPT.
-            - name: FELIX_DEFAULTENDPOINTTOHOSTACTION
-              value: "ACCEPT"
-            # Disable IPv6 on Kubernetes.
-            - name: FELIX_IPV6SUPPORT
-              value: "false"
-            - name: FELIX_HEALTHENABLED
-              value: "true"
-          securityContext:
-            privileged: true
-          resources:
-            requests:
-              cpu: 250m
-          livenessProbe:
-            exec:
-              command:
-              - /bin/calico-node
-              - -felix-live
-              #- -bird-live
-            periodSeconds: 10
-            initialDelaySeconds: 10
-            failureThreshold: 6
-            timeoutSeconds: 10
-          readinessProbe:
-            exec:
-              command:
-              - /bin/calico-node
-              - -felix-ready
-              #- -bird-ready
-            periodSeconds: 10
-            timeoutSeconds: 10
-          volumeMounts:
-            # For maintaining CNI plugin API credentials.
-            - mountPath: /host/etc/cni/net.d
-              name: cni-net-dir
-              readOnly: false
-            - mountPath: /lib/modules
-              name: lib-modules
-              readOnly: true
-            - mountPath: /run/xtables.lock
-              name: xtables-lock
-              readOnly: false
-            - mountPath: /var/run/calico
-              name: var-run-calico
-              readOnly: false
-            - mountPath: /var/lib/calico
-              name: var-lib-calico
-              readOnly: false
-            - name: policysync
-              mountPath: /var/run/nodeagent
-            # For eBPF mode, we need to be able to mount the BPF filesystem at /sys/fs/bpf so we mount in the
-            # parent directory.
-            - name: sysfs
-              mountPath: /sys/fs/
-              # Bidirectional means that, if we mount the BPF filesystem at /sys/fs/bpf it will propagate to the host.
-              # If the host is known to mount that filesystem already then Bidirectional can be omitted.
-              mountPropagation: Bidirectional
-            - name: cni-log-dir
-              mountPath: /var/log/calico/cni
-              readOnly: true
-      volumes:
-        # Used by calico-node.
-        - name: lib-modules
-          hostPath:
-            path: /lib/modules
-        - name: var-run-calico
-          hostPath:
-            path: /var/run/calico
-        - name: var-lib-calico
-          hostPath:
-            path: /var/lib/calico
-        - name: xtables-lock
-          hostPath:
-            path: /run/xtables.lock
-            type: FileOrCreate
-        - name: sysfs
-          hostPath:
-            path: /sys/fs/
-            type: DirectoryOrCreate
-        # Used to install CNI.
-        - name: cni-bin-dir
-          hostPath:
-            path: /opt/cni/bin
-        - name: cni-net-dir
-          hostPath:
-            path: /etc/cni/net.d
-        # Used to access CNI logs.
-        - name: cni-log-dir
-          hostPath:
-            path: /var/log/calico/cni
-        # Mount in the directory for host-local IPAM allocations. This is
-        # used when upgrading from host-local to calico-ipam, and can be removed
-        # if not using the upgrade-ipam init container.
-        - name: host-local-net-dir
-          hostPath:
-            path: /var/lib/cni/networks
-        # Used to create per-pod Unix Domain Sockets
-        - name: policysync
-          hostPath:
-            type: DirectoryOrCreate
-            path: /var/run/nodeagent
-        # Used to install Flex Volume Driver
-        - name: flexvol-driver-host
-          hostPath:
-            type: DirectoryOrCreate
-            path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent~uds
----
-
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: calico-node
-  namespace: kube-system
-
----
-# Source: calico/templates/calico-kube-controllers.yaml
-# See https://github.com/projectcalico/kube-controllers
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: calico-kube-controllers
-  namespace: kube-system
-  labels:
-    k8s-app: calico-kube-controllers
-spec:
-  # The controllers can only have a single active instance.
-  replicas: 1
-  selector:
-    matchLabels:
-      k8s-app: calico-kube-controllers
-  strategy:
-    type: Recreate
-  template:
-    metadata:
-      name: calico-kube-controllers
-      namespace: kube-system
-      labels:
-        k8s-app: calico-kube-controllers
-    spec:
-      tolerations:
-        # Mark the pod as a critical add-on for rescheduling.
-        - key: CriticalAddonsOnly
-          operator: Exists
-        - key: node-role.kubernetes.io/master
-          effect: NoSchedule
-      serviceAccountName: calico-kube-controllers
-      priorityClassName: system-cluster-critical
-      containers:
-        - name: calico-kube-controllers
-          image: docker.io/calico/kube-controllers:v3.20.0
-          resouces:
-          env:
-            # Choose which controllers to run.
-            - name: ENABLED_CONTROLLERS
-              value: node
-            - name: DATASTORE_TYPE
-              value: kubernetes
-          livenessProbe:
-            exec:
-              command:
-              - /usr/bin/check-status
-              - -l
-            periodSeconds: 10
-            initialDelaySeconds: 10
-            failureThreshold: 6
-            timeoutSeconds: 10
-          readinessProbe:
-            exec:
-              command:
-              - /usr/bin/check-status
-              - -r
-            periodSeconds: 10
-
----
-
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: calico-kube-controllers
-  namespace: kube-system
-
----
-
-# This manifest creates a Pod Disruption Budget for Controller to allow K8s Cluster Autoscaler to evict
-
-apiVersion: policy/v1beta1
-kind: PodDisruptionBudget
-metadata:
-  name: calico-kube-controllers
-  namespace: kube-system
-  labels:
-    k8s-app: calico-kube-controllers
-spec:
-  maxUnavailable: 1
-  selector:
-    matchLabels:
-      k8s-app: calico-kube-controllers
-

deploy/calico-cni/calico.yaml

@@ -1,687 +0,0 @@
----
-# Source: calico/templates/calico-config.yaml
-# This ConfigMap is used to configure a self-hosted Calico installation.
-kind: ConfigMap
-apiVersion: v1
-metadata:
-  name: calico-config
-  namespace: kube-system
-data:
-  # Typha is disabled.
-  typha_service_name: "none"
-  # Configure the backend to use.
-  calico_backend: "bird"
-
-  # Configure the MTU to use for workload interfaces and tunnels.
-  # By default, MTU is auto-detected, and explicitly setting this field should not be required.
-  # You can override auto-detection by providing a non-zero value.
-  veth_mtu: "0"
-
-  # The CNI network configuration to install on each node. The special
-  # values in this config will be automatically populated.
-  cni_network_config: |-
-    {
-      "name": "k8s-pod-network",
-      "cniVersion": "0.3.1",
-      "plugins": [
-        {
-          "type": "calico",
-          "log_level": "info",
-          "log_file_path": "/var/log/calico/cni/cni.log",
-          "datastore_type": "kubernetes",
-          "nodename": "__KUBERNETES_NODE_NAME__",
-          "mtu": __CNI_MTU__,
-          "ipam": {
-              "type": "calico-ipam"
-          },
-          "policy": {
-              "type": "k8s"
-          },
-          "kubernetes": {
-              "kubeconfig": "__KUBECONFIG_FILEPATH__"
-          }
-        },
-        {
-          "type": "portmap",
-          "snat": true,
-          "capabilities": {"portMappings": true}
-        },
-        {
-          "type": "bandwidth",
-          "capabilities": {"bandwidth": true}
-        }
-      ]
-    }
-
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: calico-node
-rules:
-- apiGroups:
-  - ""
-  resources:
-  - pods
-  - nodes
-  - namespaces
-  verbs:
-  - get
-- apiGroups:
-  - discovery.k8s.io
-  resources:
-  - endpointslices
-  verbs:
-  - watch
-  - list
-- apiGroups:
-  - ""
-  resources:
-  - endpoints
-  - services
-  verbs:
-  - watch
-  - list
-  - get
-- apiGroups:
-  - ""
-  resources:
-  - configmaps
-  verbs:
-  - get
-- apiGroups:
-  - ""
-  resources:
-  - nodes/status
-  verbs:
-  - patch
-  - update
-- apiGroups:
-  - networking.k8s.io
-  resources:
-  - networkpolicies
-  verbs:
-  - watch
-  - list
-- apiGroups:
-  - ""
-  resources:
-  - pods
-  - namespaces
-  - serviceaccounts
-  verbs:
-  - list
-  - watch
-- apiGroups:
-  - ""
-  resources:
-  - pods/status
-  verbs:
-  - patch
-- apiGroups:
-  - crd.projectcalico.org
-  resources:
-  - globalfelixconfigs
-  - felixconfigurations
-  - bgppeers
-  - globalbgpconfigs
-  - bgpconfigurations
-  - ippools
-  - ipamblocks
-  - globalnetworkpolicies
-  - globalnetworksets
-  - networkpolicies
-  - networksets
-  - clusterinformations
-  - hostendpoints
-  - blockaffinities
-  verbs:
-  - get
-  - list
-  - watch
-- apiGroups:
-  - crd.projectcalico.org
-  resources:
-  - ippools
-  - felixconfigurations
-  - clusterinformations
-  verbs:
-  - create
-  - update
-- apiGroups:
-  - ""
-  resources:
-  - nodes
-  verbs:
-  - get
-  - list
-  - watch
-- apiGroups:
-  - crd.projectcalico.org
-  resources:
-  - bgpconfigurations
-  - bgppeers
-  verbs:
-  - create
-  - update
-- apiGroups:
-  - crd.projectcalico.org
-  resources:
-  - blockaffinities
-  - ipamblocks
-  - ipamhandles
-  verbs:
-  - get
-  - list
-  - create
-  - update
-  - delete
-- apiGroups:
-  - crd.projectcalico.org
-  resources:
-  - ipamconfigs
-  verbs:
-  - get
-- apiGroups:
-  - crd.projectcalico.org
-  resources:
-  - blockaffinities
-  verbs:
-  - watch
-- apiGroups:
-  - apps
-  resources:
-  - daemonsets
-  verbs:
-  - get
-
----
-
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: calico-kube-controllers
-rules:
-- apiGroups:
-  - ""
-  resources:
-  - nodes
-  verbs:
-  - watch
-  - list
-  - get
-- apiGroups:
-  - ""
-  resources:
-  - pods
-  verbs:
-  - get
-  - list
-  - watch
-- apiGroups:
-  - crd.projectcalico.org
-  resources:
-  - ippools
-  verbs:
-  - list
-- apiGroups:
-  - crd.projectcalico.org
-  resources:
-  - blockaffinities
-  - ipamblocks
-  - ipamhandles
-  verbs:
-  - get
-  - list
-  - create
-  - update
-  - delete
-  - watch
-- apiGroups:
-  - crd.projectcalico.org
-  resources:
-  - hostendpoints
-  verbs:
-  - get
-  - list
-  - create
-  - update
-  - delete
-- apiGroups:
-  - crd.projectcalico.org
-  resources:
-  - clusterinformations
-  verbs:
-  - get
-  - create
-  - update
-- apiGroups:
-  - crd.projectcalico.org
-  resources:
-  - kubecontrollersconfigurations
-  verbs:
-  - get
-  - create
-  - update
-  - watch
-
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: calico-node
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: calico-node
-subjects:
-- kind: ServiceAccount
-  name: calico-node
-  namespace: kube-system
-
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: calico-kube-controllers
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: calico-kube-controllers
-subjects:
-- kind: ServiceAccount
-  name: calico-kube-controllers
-  namespace: kube-system
----
-# Source: calico/templates/calico-node.yaml
-# This manifest installs the calico-node container, as well
-# as the CNI plugins and network config on
-# each master and worker node in a Kubernetes cluster.
-kind: DaemonSet
-apiVersion: apps/v1
-metadata:
-  name: calico-node
-  namespace: kube-system
-  labels:
-    k8s-app: calico-node
-spec:
-  selector:
-    matchLabels:
-      k8s-app: calico-node
-  updateStrategy:
-    type: RollingUpdate
-    rollingUpdate:
-      maxUnavailable: 1
-  template:
-    metadata:
-      labels:
-        k8s-app: calico-node
-    spec:
-      nodeSelector:
-        kubernetes.io/os: linux
-      hostNetwork: true
-      tolerations:
-        # Make sure calico-node gets scheduled on all nodes.
-        - effect: NoSchedule
-          operator: Exists
-        # Mark the pod as a critical add-on for rescheduling.
-        - key: CriticalAddonsOnly
-          operator: Exists
-        - effect: NoExecute
-          operator: Exists
-      serviceAccountName: calico-node
-      # Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force
-      # deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods.
-      terminationGracePeriodSeconds: 0
-      priorityClassName: system-node-critical
-      initContainers:
-        # This container performs upgrade from host-local IPAM to calico-ipam.
-        # It can be deleted if this is a fresh installation, or if you have already
-        # upgraded to use calico-ipam.
-        - name: upgrade-ipam
-          image: docker.io/calico/cni:v3.20.0
-          command: ["/opt/cni/bin/calico-ipam", "-upgrade"]
-          envFrom:
-          - configMapRef:
-              # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode.
-              name: kubernetes-services-endpoint
-              optional: true
-          env:
-            - name: KUBERNETES_NODE_NAME
-              valueFrom:
-                fieldRef:
-                  fieldPath: spec.nodeName
-            - name: CALICO_NETWORKING_BACKEND
-              valueFrom:
-                configMapKeyRef:
-                  name: calico-config
-                  key: calico_backend
-          volumeMounts:
-            - mountPath: /var/lib/cni/networks
-              name: host-local-net-dir
-            - mountPath: /host/opt/cni/bin
-              name: cni-bin-dir
-          securityContext:
-            privileged: true
-        # This container installs the CNI binaries
-        # and CNI network config file on each node.
-        - name: install-cni
-          image: docker.io/calico/cni:v3.20.0
-          command: ["/opt/cni/bin/install"]
-          envFrom:
-          - configMapRef:
-              # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode.
-              name: kubernetes-services-endpoint
-              optional: true
-          env:
-            # Name of the CNI config file to create.
-            - name: CNI_CONF_NAME
-              value: "10-calico.conflist"
-            # The CNI network config to install on each node.
-            - name: CNI_NETWORK_CONFIG
-              valueFrom:
-                configMapKeyRef:
-                  name: calico-config
-                  key: cni_network_config
-            # Set the hostname based on the k8s node name.
-            - name: KUBERNETES_NODE_NAME
-              valueFrom:
-                fieldRef:
-                  fieldPath: spec.nodeName
-            # CNI MTU Config variable
-            - name: CNI_MTU
-              valueFrom:
-                configMapKeyRef:
-                  name: calico-config
-                  key: veth_mtu
-            # Prevents the container from sleeping forever.
-            - name: SLEEP
-              value: "false"
-          volumeMounts:
-            - mountPath: /host/opt/cni/bin
-              name: cni-bin-dir
-            - mountPath: /host/etc/cni/net.d
-              name: cni-net-dir
-          securityContext:
-            privileged: true
-        # Adds a Flex Volume Driver that creates a per-pod Unix Domain Socket to allow Dikastes
-        # to communicate with Felix over the Policy Sync API.
-        - name: flexvol-driver
-          image: docker.io/calico/pod2daemon-flexvol:v3.20.0
-          volumeMounts:
-          - name: flexvol-driver-host
-            mountPath: /host/driver
-          securityContext:
-            privileged: true
-      containers:
-        # Runs calico-node container on each Kubernetes node. This
-        # container programs network policy and routes on each
-        # host.
-        - name: calico-node
-          image: docker.io/calico/node:v3.20.0
-          envFrom:
-          - configMapRef:
-              # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode.
-              name: kubernetes-services-endpoint
-              optional: true
-          env:
-            # Use Kubernetes API as the backing datastore.
-            - name: DATASTORE_TYPE
-              value: "kubernetes"
-            # Wait for the datastore.
-            - name: WAIT_FOR_DATASTORE
-              value: "true"
-            # Set based on the k8s node name.
-            - name: NODENAME
-              valueFrom:
-                fieldRef:
-                  fieldPath: spec.nodeName
-            # Choose the backend to use.
-            - name: CALICO_NETWORKING_BACKEND
-              valueFrom:
-                configMapKeyRef:
-                  name: calico-config
-                  key: calico_backend
-            # Cluster type to identify the deployment type
-            - name: CLUSTER_TYPE
-              value: "k8s"
-            # Auto-detect the BGP IP address.
-            - name: IP
-              value: "autodetect"
-            - name: IP_AUTODETECTION_METHOD
-              value: "can-reach=192.168.32.0"
-            # Enable IPIP
-            - name: CALICO_IPV4POOL_IPIP
-              value: "Never"
-            # Enable or Disable VXLAN on the default IP pool.
-            - name: CALICO_IPV4POOL_VXLAN
-              value: "Never"
-            # Set MTU for tunnel device used if ipip is enabled
-            - name: FELIX_IPINIPMTU
-              valueFrom:
-                configMapKeyRef:
-                  name: calico-config
-                  key: veth_mtu
-            # Set MTU for the VXLAN tunnel device.
-            - name: FELIX_VXLANMTU
-              valueFrom:
-                configMapKeyRef:
-                  name: calico-config
-                  key: veth_mtu
-            # Set MTU for the Wireguard tunnel device.
-            - name: FELIX_WIREGUARDMTU
-              valueFrom:
-                configMapKeyRef:
-                  name: calico-config
-                  key: veth_mtu
-            # The default IPv4 pool to create on startup if none exists. Pod IPs will be
-            # chosen from this range. Changing this value after installation will have
-            # no effect. This should fall within `--cluster-cidr`.
-            # - name: CALICO_IPV4POOL_CIDR
-            #   value: "192.168.0.0/16"
-            # Disable file logging so `kubectl logs` works.
-            - name: CALICO_DISABLE_FILE_LOGGING
-              value: "true"
-            # Set Felix endpoint to host default action to ACCEPT.
-            - name: FELIX_DEFAULTENDPOINTTOHOSTACTION
-              value: "ACCEPT"
-            # Disable IPv6 on Kubernetes.
-            - name: FELIX_IPV6SUPPORT
-              value: "false"
-            - name: FELIX_HEALTHENABLED
-              value: "true"
-          securityContext:
-            privileged: true
-          resources:
-            requests:
-              cpu: 250m
-          livenessProbe:
-            exec:
-              command:
-              - /bin/calico-node
-              - -felix-live
-              - -bird-live
-            periodSeconds: 10
-            initialDelaySeconds: 10
-            failureThreshold: 6
-            timeoutSeconds: 10
-          readinessProbe:
-            exec:
-              command:
-              - /bin/calico-node
-              - -felix-ready
-              - -bird-ready
-            periodSeconds: 10
-            timeoutSeconds: 10
-          volumeMounts:
-            # For maintaining CNI plugin API credentials.
-            - mountPath: /host/etc/cni/net.d
-              name: cni-net-dir
-              readOnly: false
-            - mountPath: /lib/modules
-              name: lib-modules
-              readOnly: true
-            - mountPath: /run/xtables.lock
-              name: xtables-lock
-              readOnly: false
-            - mountPath: /var/run/calico
-              name: var-run-calico
-              readOnly: false
-            - mountPath: /var/lib/calico
-              name: var-lib-calico
-              readOnly: false
-            - name: policysync
-              mountPath: /var/run/nodeagent
-            # For eBPF mode, we need to be able to mount the BPF filesystem at /sys/fs/bpf so we mount in the
-            # parent directory.
-            - name: sysfs
-              mountPath: /sys/fs/
-              # Bidirectional means that, if we mount the BPF filesystem at /sys/fs/bpf it will propagate to the host.
-              # If the host is known to mount that filesystem already then Bidirectional can be omitted.
-              mountPropagation: Bidirectional
-            - name: cni-log-dir
-              mountPath: /var/log/calico/cni
-              readOnly: true
-      volumes:
-        # Used by calico-node.
-        - name: lib-modules
-          hostPath:
-            path: /lib/modules
-        - name: var-run-calico
-          hostPath:
-            path: /var/run/calico
-        - name: var-lib-calico
-          hostPath:
-            path: /var/lib/calico
-        - name: xtables-lock
-          hostPath:
-            path: /run/xtables.lock
-            type: FileOrCreate
-        - name: sysfs
-          hostPath:
-            path: /sys/fs/
-            type: DirectoryOrCreate
-        # Used to install CNI.
-        - name: cni-bin-dir
-          hostPath:
-            path: /opt/cni/bin
-        - name: cni-net-dir
-          hostPath:
-            path: /etc/cni/net.d
-        # Used to access CNI logs.
-        - name: cni-log-dir
-          hostPath:
-            path: /var/log/calico/cni
-        # Mount in the directory for host-local IPAM allocations. This is
-        # used when upgrading from host-local to calico-ipam, and can be removed
-        # if not using the upgrade-ipam init container.
-        - name: host-local-net-dir
-          hostPath:
-            path: /var/lib/cni/networks
-        # Used to create per-pod Unix Domain Sockets
-        - name: policysync
-          hostPath:
-            type: DirectoryOrCreate
-            path: /var/run/nodeagent
-        # Used to install Flex Volume Driver
-        - name: flexvol-driver-host
-          hostPath:
-            type: DirectoryOrCreate
-            path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent~uds
----
-
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: calico-node
-  namespace: kube-system
-
----
-# Source: calico/templates/calico-kube-controllers.yaml
-# See https://github.com/projectcalico/kube-controllers
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: calico-kube-controllers
-  namespace: kube-system
-  labels:
-    k8s-app: calico-kube-controllers
-spec:
-  # The controllers can only have a single active instance.
-  replicas: 1
-  selector:
-    matchLabels:
-      k8s-app: calico-kube-controllers
-  strategy:
-    type: Recreate
-  template:
-    metadata:
-      name: calico-kube-controllers
-      namespace: kube-system
-      labels:
-        k8s-app: calico-kube-controllers
-    spec:
-      nodeSelector:
-        kubernetes.io/os: linux
-      tolerations:
-        # Mark the pod as a critical add-on for rescheduling.
-        - key: CriticalAddonsOnly
-          operator: Exists
-        - key: node-role.kubernetes.io/master
-          effect: NoSchedule
-      serviceAccountName: calico-kube-controllers
-      priorityClassName: system-cluster-critical
-      containers:
-        - name: calico-kube-controllers
-          image: docker.io/calico/kube-controllers:v3.20.0
-          env:
-            # Choose which controllers to run.
-            - name: ENABLED_CONTROLLERS
-              value: node
-            - name: DATASTORE_TYPE
-              value: kubernetes
-          resources:
-          livenessProbe:
-            exec:
-              command:
-              - /usr/bin/check-status
-              - -l
-            periodSeconds: 10
-            initialDelaySeconds: 10
-            failureThreshold: 6
-            timeoutSeconds: 10
-          readinessProbe:
-            exec:
-              command:
-              - /usr/bin/check-status
-              - -r
-            periodSeconds: 10
-
----
-
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: calico-kube-controllers
-  namespace: kube-system
-
----
-
-# This manifest creates a Pod Disruption Budget for Controller to allow K8s Cluster Autoscaler to evict
-
-apiVersion: policy/v1beta1
-kind: PodDisruptionBudget
-metadata:
-  name: calico-kube-controllers
-  namespace: kube-system
-  labels:
-    k8s-app: calico-kube-controllers
-spec:
-  maxUnavailable: 1
-  selector:
-    matchLabels:
-      k8s-app: calico-kube-controllers
-
----
-
-

deploy/etcd/etcd-cluster.yaml

@@ -0,0 +1,113 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: etcd
+  namespace:
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: etcd-server
+  namespace:
+spec:
+  type: ClusterIP
+  ports:
+    - name: client
+      port: 2379
+      protocol: TCP
+      targetPort: 2379
+  selector:
+    app: etcd
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: etcd
+  namespace:
+spec:
+  clusterIP: None
+  ports:
+    - port: 2379
+      name: client
+    - port: 2380
+      name: peer
+  selector:
+    app: etcd
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: etcd
+  labels:
+    app: etcd
+  namespace:
+spec:
+  serviceName: etcd
+  selector:
+    matchLabels:
+      app: etcd
+  replicas: 3
+  template:
+    metadata:
+      name: etcd
+      labels:
+        app: etcd
+    spec:
+      serviceAccountName: etcd
+      volumes:
+      - name: certs
+        secret:
+          secretName: etcd-certs
+      containers:
+        - name: etcd
+          image: quay.io/coreos/etcd:v3.5.1
+          ports:
+            - containerPort: 2379
+              name: client
+            - containerPort: 2380
+              name: peer
+          volumeMounts:
+          - name: data
+            mountPath: /var/run/etcd
+          - name: certs
+            mountPath: /etc/etcd/pki
+          command:
+          - etcd 
+          - --data-dir=/var/run/etcd
+          - --name=$(POD_NAME)
+          - --initial-cluster-state=new
+          - --initial-cluster=etcd-0=https://etcd-0.etcd.$(POD_NAMESPACE).svc.cluster.local:2380,etcd-1=https://etcd-1.etcd.$(POD_NAMESPACE).svc.cluster.local:2380,etcd-2=https://etcd-2.etcd.$(POD_NAMESPACE).svc.cluster.local:2380
+          - --initial-advertise-peer-urls=https://$(POD_NAME).etcd.$(POD_NAMESPACE).svc.cluster.local:2380
+          - --initial-cluster-token=kamaji
+          - --listen-client-urls=https://0.0.0.0:2379
+          - --advertise-client-urls=https://etcd-0.etcd.$(POD_NAMESPACE).svc.cluster.local:2379,https://etcd-1.etcd.$(POD_NAMESPACE).svc.cluster.local:2379,https://etcd-2.etcd.$(POD_NAMESPACE).svc.cluster.local:2379,https://etcd-server.$(POD_NAMESPACE).svc.cluster.local:2379
+          - --client-cert-auth=true
+          - --trusted-ca-file=/etc/etcd/pki/ca.crt
+          - --cert-file=/etc/etcd/pki/server.pem
+          - --key-file=/etc/etcd/pki/server-key.pem
+          - --listen-peer-urls=https://0.0.0.0:2380
+          - --peer-client-cert-auth=true
+          - --peer-trusted-ca-file=/etc/etcd/pki/ca.crt
+          - --peer-cert-file=/etc/etcd/pki/peer.pem 
+          - --peer-key-file=/etc/etcd/pki/peer-key.pem
+          - --snapshot-count=8000
+          - --auto-compaction-mode=periodic
+          - --auto-compaction-retention=5m
+          - --quota-backend-bytes=8589934592
+          env:
+          - name: POD_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.name
+          - name: POD_NAMESPACE
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.namespace
+  volumeClaimTemplates:
+  - metadata:
+      name: data
+    spec:
+      accessModes: ["ReadWriteOnce"]
+      resources:
+        requests:
+          storage: 8Gi

deploy/kamaji-azure.env

@@ -1,5 +1,31 @@
+# azure parameters
 export KAMAJI_REGION=westeurope
 export KAMAJI_RG=Kamaji
-# https://docs.microsoft.com/en-us/azure/aks/faq#why-are-two-resource-groups-created-with-aks
-export KAMAJI_CLUSTER=kamaji
 export KAMAJI_NODE_RG=MC_${KAMAJI_RG}_${KAMAJI_CLUSTER}_${KAMAJI_REGION}
+export KAMAJI_CLUSTER=kamaji
+export KAMAJI_VNET_NAME=kamaji-net
+export KAMAJI_VNET_ADDRESS=10.224.0.0/12
+export KAMAJI_SUBNET_NAME=kamaji-subnet
+export KAMAJI_SUBNET_ADDRESS=10.224.0.0/16
+
+# kamaji parameters
+export KAMAJI_NAMESPACE=kamaji-system
+
+# tenant cluster parameters
+export TENANT_NAMESPACE=default
+export TENANT_NAME=tenant-00
+export TENANT_DOMAIN=$KAMAJI_REGION.cloudapp.azure.com
+export TENANT_VERSION=v1.25.0
+export TENANT_PORT=6443 # port used to expose the tenant api server
+export TENANT_PROXY_PORT=8132 # port used to expose the konnectivity server
+export TENANT_POD_CIDR=10.36.0.0/16
+export TENANT_SVC_CIDR=10.96.0.0/16
+export TENANT_DNS_SERVICE=10.96.0.10
+
+export TENANT_VM_SIZE=Standard_D2ds_v4
+export TENANT_VM_IMAGE=UbuntuLTS
+export TENANT_SUBNET_NAME=$TENANT_NAME-subnet
+export TENANT_SUBNET_ADDRESS=10.225.0.0/16
+export TENANT_VMSS=$TENANT_NAME-vmss
+
+

deploy/kamaji-external-etcd.env

@@ -1,5 +0,0 @@
-# etcd machine addresses
-export ETCD0=192.168.32.10
-export ETCD1=192.168.32.11
-export ETCD2=192.168.32.12
-export ETCDHOSTS=($ETCD0 $ETCD1 $ETCD2)

deploy/kamaji-internal-etcd.env

@@ -1,5 +0,0 @@
-# etcd endpoints
-export ETCD_NAMESPACE=etcd-system
-export ETCD0=etcd-0.etcd.${ETCD_NAMESPACE}.svc.cluster.local
-export ETCD1=etcd-1.etcd.${ETCD_NAMESPACE}.svc.cluster.local
-export ETCD2=etcd-2.etcd.${ETCD_NAMESPACE}.svc.cluster.local

deploy/kamaji-tenant-azure.env

@@ -1,22 +0,0 @@
-export KAMAJI_REGION=westeurope
-
-# tenant cluster parameters
-export TENANT_NAMESPACE=tenants
-export TENANT_NAME=tenant-00
-export TENANT_DOMAIN=$KAMAJI_REGION.cloudapp.azure.com
-export TENANT_VERSION=v1.23.4
-export TENANT_ADDR=10.240.0.100 # IP used to expose the tenant control plane
-export TENANT_PORT=6443 # PORT used to expose the tenant control plane
-export TENANT_POD_CIDR=10.36.0.0/16
-export TENANT_SVC_CIDR=10.96.0.0/16
-export TENANT_DNS_SERVICE=10.96.0.10
-
-export TENANT_VM_SIZE=Standard_D2ds_v4
-export TENANT_VM_IMAGE=UbuntuLTS
-export TENANT_RG=$TENANT_NAME
-export TENANT_NSG=$TENANT_NAME-nsg
-export TENANT_VNET_NAME=$TENANT_NAME
-export TENANT_VNET_ADDRESS=192.168.0.0/16
-export TENANT_SUBNET_NAME=$TENANT_NAME-subnet
-export TENANT_SUBNET_ADDRESS=192.168.10.0/24
-export TENANT_VMSS=$TENANT_NAME-vmss

deploy/kamaji-tenant.env

@@ -1,17 +0,0 @@
-
-# tenant cluster parameters
-export TENANT_NAMESPACE=tenants
-export TENANT_NAME=tenant-00
-export TENANT_DOMAIN=clastix.labs
-export TENANT_VERSION=v1.23.1
-export TENANT_ADDR=192.168.32.150 # IP used to expose the tenant control plane
-export TENANT_PORT=6443 # PORT used to expose the tenant control plane
-export TENANT_POD_CIDR=10.36.0.0/16
-export TENANT_SVC_CIDR=10.96.0.0/16
-export TENANT_DNS_SERVICE=10.96.0.10
-
-# tenant node addresses
-export WORKER0=192.168.32.90
-export WORKER1=192.168.32.91
-export WORKER2=192.168.32.92
-export WORKER3=192.168.32.93

deploy/kamaji.env

@@ -0,0 +1,18 @@
+# kamaji parameters
+export KAMAJI_NAMESPACE=kamaji-system
+
+# tenant cluster parameters
+export TENANT_NAMESPACE=default
+export TENANT_NAME=tenant-00
+export TENANT_DOMAIN=clastix.labs
+export TENANT_VERSION=v1.25.0
+export TENANT_PORT=6443 # port used to expose the tenant api server
+export TENANT_PROXY_PORT=8132 # port used to expose the konnectivity server
+export TENANT_POD_CIDR=10.36.0.0/16
+export TENANT_SVC_CIDR=10.96.0.0/16
+export TENANT_DNS_SERVICE=10.96.0.10
+
+# tenant node addresses
+export WORKER0=172.12.0.10
+export WORKER1=172.12.0.11
+export WORKER2=172.12.0.12

deploy/kind/Makefile

@@ -2,7 +2,7 @@ kind_path := $(patsubst %/,%,$(dir $(abspath $(lastword $(MAKEFILE_LIST)))))
 
 include ../etcd/Makefile
 
-.PHONY: kind ingress-nginx kamaji-kind-worker-build
+.PHONY: kind ingress-nginx
 
 .DEFAULT_GOAL := kamaji
 
@@ -29,14 +29,5 @@ ingress-nginx: ingress-nginx-install
 ingress-nginx-install:
 	kubectl apply -f $(kind_path)/nginx-deploy.yaml
 
-kamaji-kind-worker-build:
-	docker build -f $(kind_path)/kamaji-kind-worker.dockerfile -t clastix/kamaji-kind-worker:$${WORKER_VERSION:-latest} .
-
-kamaji-kind-worker-push: kamaji-kind-worker-build
-	docker push clastix/kamaji-kind-worker:$${WORKER_VERSION:-latest}
-
 kamaji-kind-worker-join:
 	$(kind_path)/join-node.bash
-
-kamaji-kind-worker-join-through-konnectivity:
-	$(kind_path)/join-node-konnectivity.bash

deploy/kind/cni-kindnet-config.json

@@ -1,33 +0,0 @@
-{
-    "cniVersion": "0.3.1",
-    "name": "kindnet",
-    "plugins": [
-      {
-        "type": "ptp",
-        "ipMasq": false,
-        "ipam": {
-          "type": "host-local",
-          "dataDir": "/run/cni-ipam-state",
-          "routes": [
-            {
-              "dst": "0.0.0.0/0"
-            }
-          ],
-          "ranges": [
-            [
-              {
-                "subnet": "10.244.0.0/24"
-              }
-            ]
-          ]
-        },
-        "mtu": 1500
-      },
-      {
-        "type": "portmap",
-        "capabilities": {
-          "portMappings": true
-        }
-      }
-    ]
-  }

deploy/kind/join-node-konnectivity.bash

@@ -1,35 +0,0 @@
-#!/bin/bash
-
-set -e
-
-# Constants
-export DOCKER_IMAGE_NAME="clastix/kamaji-kind-worker"
-
-# Variables
-export KUBERNETES_VERSION=${1:-latest}
-export KUBECONFIG="${KUBECONFIG:-/tmp/kubeconfig}"
-
-if [ -z $2 ]
-then
-    MAPPING_PORT=""
-else
-    MAPPING_PORT="-p ${2}:80"
-fi
-
-export KONNECTIVITY_PROXY_HOST=${3:-konnectiviy.local}
-
-clear
-echo "Welcome to join a new node through Konnectivity"
-
-echo -ne "\nChecking right kubeconfig\n"
-kubectl cluster-info
-echo "Are you pointing to the right tenant control plane? (Type return to continue)"
-read
-
-JOIN_CMD="$(kubeadm --kubeconfig=${KUBECONFIG} token create --print-join-command) --ignore-preflight-errors=SystemVerification"
-echo "Deploying new node..."
-KIND_IP=$(docker inspect kamaji-control-plane --format='{{.NetworkSettings.Networks.kind.IPAddress}}')
-NODE=$(docker run -d --add-host $KONNECTIVITY_PROXY_HOST:$KIND_IP --privileged -v /lib/modules:/lib/modules:ro -v /var --net host $MAPPING_PORT $DOCKER_IMAGE_NAME:$KUBERNETES_VERSION)
-sleep 10
-echo "Joining new node..."
-docker exec -e JOIN_CMD="$JOIN_CMD" $NODE /bin/bash -c "$JOIN_CMD"

deploy/kind/join-node.bash

@@ -3,11 +3,11 @@
 set -e
 
 # Constants
-export DOCKER_IMAGE_NAME="clastix/kamaji-kind-worker"
+export DOCKER_IMAGE_NAME="kindest/node"
 export DOCKER_NETWORK="kind"
 
 # Variables
-export KUBERNETES_VERSION=${1:-latest}
+export KUBERNETES_VERSION=${1:-v1.23.5}
 export KUBECONFIG="${KUBECONFIG:-/tmp/kubeconfig}"
 
 if [ -z $2 ]
@@ -31,3 +31,6 @@ NODE=$(docker run -d --privileged -v /lib/modules:/lib/modules:ro -v /var --net
 sleep 10
 echo "Joining new node..."
 docker exec -e JOIN_CMD="$JOIN_CMD" $NODE /bin/bash -c "$JOIN_CMD"
+
+echo "Node has joined! Remember to install the kind-net CNI by issuing the following command:"
+echo "  $: kubectl apply -f https://raw.githubusercontent.com/aojea/kindnet/master/install-kindnet.yaml"

deploy/kind/kamaji-kind-worker.dockerfile

@@ -1,4 +0,0 @@
-ARG KUBERNETES_VERSION=v1.23.4
-FROM kindest/node:$KUBERNETES_VERSION
-
-COPY ./cni-kindnet-config.json /etc/cni/net.d/10-kindnet.conflist

deploy/kine/mysql/Makefile

@@ -0,0 +1,32 @@
+ROOT_DIR:=$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))
+
+mariadb: mariadb-certificates mariadb-secret mariadb-deployment
+
+mariadb-certificates:
+	rm -rf $(ROOT_DIR)/certs && mkdir $(ROOT_DIR)/certs
+	cfssl gencert -initca $(ROOT_DIR)/ca-csr.json | cfssljson -bare $(ROOT_DIR)/certs/ca
+	@mv $(ROOT_DIR)/certs/ca.pem $(ROOT_DIR)/certs/ca.crt
+	@mv $(ROOT_DIR)/certs/ca-key.pem $(ROOT_DIR)/certs/ca.key
+	cfssl gencert -ca=$(ROOT_DIR)/certs/ca.crt -ca-key=$(ROOT_DIR)/certs/ca.key \
+		-config=$(ROOT_DIR)/config.json -profile=server \
+		$(ROOT_DIR)/server-csr.json | cfssljson -bare $(ROOT_DIR)/certs/server
+	@mv $(ROOT_DIR)/certs/server.pem $(ROOT_DIR)/certs/server.crt
+	@mv $(ROOT_DIR)/certs/server-key.pem $(ROOT_DIR)/certs/server.key
+	chmod 644 $(ROOT_DIR)/certs/*
+
+mariadb-secret:
+	@kubectl create namespace kamaji-system --dry-run=client -o yaml | kubectl apply -f -
+	@kubectl -n kamaji-system create secret generic mysql-config \
+		--from-file=$(ROOT_DIR)/certs/ca.crt --from-file=$(ROOT_DIR)/certs/ca.key \
+		--from-file=$(ROOT_DIR)/certs/server.key --from-file=$(ROOT_DIR)/certs/server.crt \
+		--from-file=$(ROOT_DIR)/mysql-ssl.cnf \
+		--from-literal=MYSQL_ROOT_PASSWORD=root \
+		--dry-run=client -o yaml | kubectl apply -f -
+
+mariadb-deployment:
+	@kubectl -n kamaji-system apply -f $(ROOT_DIR)/mariadb.yaml
+
+mariadb-destroy:
+	@kubectl delete -n kamaji-system -f $(ROOT_DIR)/mariadb.yaml --ignore-not-found
+	@kubectl delete -n kamaji-system secret mysql-config --ignore-not-found
+	@kubectl delete -n kamaji-system secret kine-secret --ignore-not-found

deploy/kine/postgresql/Makefile

@@ -0,0 +1,27 @@
+ROOT_DIR:=$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))
+
+postgresql: cnpg-setup cnpg-deploy postgresql-secret
+
+cnpg-setup:
+	@kubectl apply -f https://raw.githubusercontent.com/cloudnative-pg/cloudnative-pg/main/releases/cnpg-1.16.0.yaml
+
+cnpg-deploy:
+	@kubectl -n cnpg-system rollout status deployment/cnpg-controller-manager
+	@kubectl create namespace kamaji-system --dry-run=client -o yaml | kubectl apply -f -
+	@kubectl -n kamaji-system apply -f postgresql.yaml
+	@while ! kubectl -n kamaji-system get secret postgresql-superuser > /dev/null 2>&1; do sleep 1; done
+
+CNPG = $(shell git rev-parse --show-toplevel)/bin/kubectl-cnpg
+cnpg:
+	@test -f $(shell git rev-parse --show-toplevel)/bin/kubectl-cnpg || curl -sSfL \
+      https://github.com/cloudnative-pg/cloudnative-pg/raw/main/hack/install-cnpg-plugin.sh | \
+      sh -s -- -b $(shell git rev-parse --show-toplevel)/bin
+
+postgresql-secret: cnpg
+	@kubectl -n kamaji-system get secret postgres-root-cert > /dev/null 2>&1 || $(CNPG) -n kamaji-system certificate postgres-root-cert \
+    		--cnpg-cluster postgresql \
+    		--cnpg-user $$(kubectl -n kamaji-system get secret postgresql-superuser -o jsonpath='{.data.username}' | base64 -d)
+
+postgresql-destroy:
+	@kubectl delete -f https://raw.githubusercontent.com/cloudnative-pg/cloudnative-pg/main/releases/cnpg-1.16.0.yaml --ignore-not-found && \
+	kubectl -n kamaji-system delete secret postgres-root-cert --ignore-not-found

deploy/kine/postgresql/postgresql.yaml

@@ -0,0 +1,13 @@
+apiVersion: postgresql.cnpg.io/v1
+kind: Cluster
+metadata:
+  name: postgresql
+spec:
+  description: PostgreSQL cluster used by Kamaji along with kine
+  instances: 3
+  postgresql:
+    pg_hba:
+      - hostssl app all all cert  # makes authentication entirely based on certificates
+  primaryUpdateStrategy: unsupervised
+  storage:
+    size: 1Gi

deploy/mysql/Makefile

@@ -1,31 +0,0 @@
-mariadb_path := $(patsubst %/,%,$(dir $(abspath $(lastword $(MAKEFILE_LIST)))))
-
-.PHONY: mariadb mariadb-certificates mariadb-secrets
-
-mariadb: mariadb-certificates mariadb-secrets mariadb-deployment
-
-mariadb-certificates:
-	rm -rf $(mariadb_path)/certs && mkdir $(mariadb_path)/certs
-	cfssl gencert -initca $(mariadb_path)/ca-csr.json | cfssljson -bare $(mariadb_path)/certs/ca
-	@mv $(mariadb_path)/certs/ca.pem $(mariadb_path)/certs/ca.crt
-	@mv $(mariadb_path)/certs/ca-key.pem $(mariadb_path)/certs/ca.key
-	cfssl gencert -ca=$(mariadb_path)/certs/ca.crt -ca-key=$(mariadb_path)/certs/ca.key \
-		-config=$(mariadb_path)/config.json -profile=server \
-		$(mariadb_path)/server-csr.json | cfssljson -bare $(mariadb_path)/certs/server
-	@mv $(mariadb_path)/certs/server.pem $(mariadb_path)/certs/server.crt
-	@mv $(mariadb_path)/certs/server-key.pem $(mariadb_path)/certs/server.key
-	chmod 644 $(mariadb_path)/certs/*
-
-mariadb-secrets:
-	@kubectl -n kamaji-system create secret generic mysql-config \
-		--from-file=$(mariadb_path)/certs/ca.crt --from-file=$(mariadb_path)/certs/ca.key \
-		--from-file=$(mariadb_path)/certs/server.key --from-file=$(mariadb_path)/certs/server.crt \
-		--from-file=$(mariadb_path)/mysql-ssl.cnf \
-		--from-literal=MYSQL_ROOT_PASSWORD=root
-
-mariadb-deployment:
-	@kubectl -n kamaji-system apply -f $(mariadb_path)/mariadb.yaml
-
-destroy:
-	@kubectl delete -n kamaji-system -f $(mariadb_path)/mariadb.yaml
-	@kubectl delete -n kamaji-system secret mysql-config

deploy/mysql/README.md

@@ -1,43 +0,0 @@
-# MySQL as Kubernetes Storage
-
-Kamaji offers the possibility of having a different storage system than `ETCD` thanks to [kine](https://github.com/k3s-io/kine). One of the implementations is [MySQL](https://www.mysql.com/).
-
-Kamaji project is developed using [kind](https://kind.sigs.k8s.io), therefore, MySQL (or [MariaDB](https://mariadb.org/) in this case) will be deployed into the local kubernetes cluster in order to be used as storage for the tenants.
-
-There is a Makefile to help with the process:
-
-* **Full Installation**
-
-```bash
-$ make mariadb
-```
-
-This action will perform all the necessary stuffs to have MariaDB as kubernetes storage backend using kine.
-
-* **Certificate creation**
-
-```bash
-$ make mariadb-certificates
-```
-
-Communication between kine and the backend is encrypted, therefore, some certificates must be created.
-
-* **Secret Deployment**
-
-```bash
-$ make mariadb-secrets
-```
-
-Previous certificates and MySQL configuration have to be available in order to be used. They will be under the secret `kamaji-system:mysql-config`.
-
-* **Deployment**
-
-```bash
-$ make mariadb-deployment
-```
-
-* **Uninstall Everything**
-
-```bash
-$ make destroy
-```

deploy/nodes-prerequisites.sh

@@ -18,7 +18,9 @@ EOF
 for i in "${!HOSTS[@]}"; do
   HOST=${HOSTS[$i]}
   ssh ${USER}@${HOST} -t 'sudo apt update && sudo apt install -y containerd'
-  ssh ${USER}@${HOST} -t 'sudo systemctl start containerd && sudo systemctl enable containerd'
+  ssh ${USER}@${HOST} -t 'sudo mkdir -p /etc/containerd'
+  ssh ${USER}@${HOST} -t 'containerd config default | sed -e "s#SystemdCgroup = false#SystemdCgroup = true#g" | sudo tee -a /etc/containerd/config.toml'
+  ssh ${USER}@${HOST} -t 'sudo systemctl restart containerd && sudo systemctl enable containerd'
   scp containerd.conf ${USER}@${HOST}:
   ssh ${USER}@${HOST} -t 'sudo chown -R root:root containerd.conf && sudo mv containerd.conf /etc/modules-load.d/containerd.conf'
   ssh ${USER}@${HOST} -t 'sudo modprobe overlay && sudo modprobe br_netfilter'

deploy/tenant-cloudinit.yaml

@@ -21,10 +21,12 @@ runcmd:
   - sudo modprobe overlay
   - sudo modprobe br_netfilter
   - sudo sysctl --system
-  - sudo systemctl start containerd
+  - sudo mkdir -p /etc/containerd
+  - containerd config default | sed -e 's#SystemdCgroup = false#SystemdCgroup = true#g' | sudo tee -a /etc/containerd/config.toml
+  - sudo systemctl restart containerd
   - sudo systemctl enable containerd
   - sudo curl -fsSLo /usr/share/keyrings/kubernetes-archive-keyring.gpg https://packages.cloud.google.com/apt/doc/apt-key.gpg
   - echo "deb [signed-by=/usr/share/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" | sudo tee /etc/apt/sources.list.d/kubernetes.list
   - sudo apt update
-  - sudo apt install -y kubelet kubeadm kubectl containerd
-  - sudo apt-mark hold kubelet kubeadm kubectl
+  - sudo apt install -y kubelet=1.25.0-00 kubeadm=1.25.0-00 kubectl=1.25.0-00
+  - sudo apt-mark hold kubelet kubeadm kubectl containerd

docs/README.md

@@ -1,10 +1,18 @@
 # Kamaji documentation
 
+- [Core Concepts](./concepts.md)
 - [Architecture](./architecture.md)
-- [Concepts](./concepts.md)
 - [Getting started](./getting-started-with-kamaji.md)
-- [Kamaji Deployment](./kamaji-deployment-guide.md)
-- [Tenant deployment](./kamaji-tenant-deployment-guide.md)
-- Deployment on cloud providers:
-  - [Azure](./kamaji-azure-deployment-guide.md)
+- Guides:
+  - [Deploy Kamaji on generic infrastructure](./kamaji-deployment-guide.md)
+  - [Deploy Kamaji on Azure](./kamaji-azure-deployment-guide.md)
+  - Deploy Kamaji on AWS
+  - Deploy Kamaji on GCP
+  - Deploy Kamaji on OpenStack
+  - [Setup Konnectivity service](./konnectivity.md)
+  - [MySQL as Kamaji datastore](./mysql-datastore.md)
+  - [PostgreSQL as Kamaji datastore](./postgresql-datastore.md)
+  - [Tenant Cluster Upgrade](./upgrade.md)
 - [Reference](./reference.md)
+- [CNCF Conformance](./conformance.md)
+- [Versioning](./versioning.md)

docs/concepts.md

@@ -1 +1,37 @@
-# Kamaji concepts
+# Core Concepts
+
+Kamaji is a Kubernetes Operator. It turns any Kubernetes cluster into an _“admin cluster”_ to orchestrate other Kubernetes clusters called _“tenant clusters”_. 
+
+## Tenant Control Plane
+What makes Kamaji special is that the Control Plane of a _“tenant cluster”_ is just one or more regular pods running in a namespace of the _“admin cluster”_ instead of a dedicated set of Virtual Machines. This solution makes running control planes at scale cheaper and easier to deploy and operate. The Tenant Control Plane components are packaged in the same way they are running in bare metal or virtual nodes. We leverage the `kubeadm` code to set up the control plane components as they were running on their own server. The unchanged images of upstream `kube-apiserver`, `kube-scheduler`, and `kube-controller-manager` are used.
+
+High Availability and rolling updates of the Tenant Control Plane pods are provided by a regular Deployment. Autoscaling based on the metrics is available. A Service is used to espose the Tenant Control Plane outside of the _“admin cluster”_. The `LoadBalancer` service type is used, `NodePort` and `ClusterIP` with an Ingress Controller are still viable options, depending on the case. 
+
+Kamaji offers a [Custom Resource Definition](https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/) to provide a declarative approach of managing a Tenant Control Plane. This *CRD* is called `TenantControlPlane`, or `tcp` in short.
+
+All the _“tenant clusters”_ built with Kamaji are fully compliant CNCF Kubernetes clusters and are compatible with the standard Kubernetes toolchains everybody knows and loves. See [CNCF compliance](./compliance.md).
+
+## Tenant worker nodes
+And what about the tenant worker nodes? They are just _"worker nodes"_, i.e. regular virtual or bare metal machines, connecting to the APIs server of the Tenant Control Plane. Kamaji's goal is to manage the lifecycle of hundreds of these _“tenant clusters”_, not only one, so how to add another tenant cluster to Kamaji? As you could expect, you have just deploys a new Tenant Control Plane in one of the _“admin cluster”_ namespace, and then joins the tenant worker nodes to it.
+
+We have in roadmap, the Cluster APIs support as well as a Terraform provider so that you can create _“tenant clusters”_ in a declarative way.
+
+## Datastores
+Putting the Tenant Control Plane in a pod is the easiest part. Also, we have to make sure each tenant cluster saves the state to be able to store and retrieve data. A dedicated `etcd` cluster for each tenant cluster doesn’t scale well for a managed service because `etcd` data persistence can be cumbersome at scale, rising the operational effort to mitigate it. So we have to find an alternative keeping in mind our goal for a resilient and cost-optimized solution at the same time.
+
+As we can deploy any Kubernetes cluster with an external `etcd` cluster, we explored this option for the tenant control planes. On the admin cluster, we can deploy a multi-tenant `etcd` datastore to save the state of multiple tenant clusters. Kamaji offers a Custom Resource Definition called `DataStore` to provide a declarative approach of managing Tenant datastores. With this solution, the resiliency is guaranteed by the usual `etcd` mechanism, and the pods' count remains under control, so it solves the main goal of resiliency and costs optimization. The trade-off here is that we have to operate an external datastore, in addition to `etcd` of the _“admin cluster”_ and manage the access to be sure that each _“tenant cluster”_ uses only its data.
+
+### Other storage drivers
+Kamaji offers the option of using a more capable datastore than `etcd` to save the state of multiple tenants' clusters. Thanks to the native [kine](https://github.com/k3s-io/kine) integration, you can run _MySQL_ or _PostgreSQL_ compatible databases as datastore for _“tenant clusters”_.
+
+### Pooling
+By default, Kamaji is expecting to persist all the _“tenant clusters”_ data in a unique datastore that could be backed by different drivers. However, you can pick a different datastore for a specific set of _“tenant clusters”_ that could have different resources assigned or a different tiering. Pooling of multiple datastore is an option you can leverage for a very large set of _“tenant clusters”_ so you can distribute the load properly. As future improvements, we have a _datastore scheduler_ feature in roadmap so that Kamaji itself can assign automatically a _“tenant cluster”_ to the best datastore in the pool.
+
+## Requirements of design
+These are requirements of design behind Kamaji:
+
+- Communication between the _“admin cluster”_ and a _“tenant cluster”_ is unidirectional. The _“admin cluster”_ manages a _“tenant cluster”_, but a _“tenant cluster”_ has no awareness of the _“admin cluster”_.
+- Communication between different _“tenant clusters”_ is not allowed.
+- The worker nodes of tenant should not run anything beyond tenant's workloads.
+
+Goals and scope may vary as the project evolves.

docs/conformance.md

@@ -0,0 +1,60 @@
+# CNCF Conformance
+For organizations using Kubernetes, conformance enables interoperability, consistency, and confirmability between Kubernetes installations. The Cloud Computing Native Foundation - CNCF - provides the [Certified Kubernetes Conformance Program](https://www.cncf.io/certification/software-conformance/). All the _“tenant clusters”_ built with Kamaji are CNCF conformant.
+
+The standard set of conformance tests is currently those defined by the `[Conformance]` tag in the
+[kubernetes e2e](https://github.com/kubernetes/kubernetes/tree/master/test/e2e) suite.
+
+## Running the conformance tests
+
+The standard tool for running CNCF conformance tests is [Sonobuoy](https://github.com/vmware-tanzu/sonobuoy). Sonobuoy is
+regularly built and kept up to date to execute against all currently supported versions of kubernetes.
+
+Download a [binary release](https://github.com/vmware-tanzu/sonobuoy/releases) of the CLI.
+
+Make sure to access your tenant cluster:
+
+```
+export KUBECONFIG=tenant.kubeconfig
+```
+
+Deploy a Sonobuoy pod to your tenant cluster with:
+
+```
+sonobuoy run --mode=certified-conformance
+```
+
+> You can run the command synchronously by adding the flag `--wait` but be aware that running the conformance tests can take an hour or more.
+
+View actively running pods:
+
+```
+sonobuoy status
+```
+
+To inspect the logs:
+
+```
+sonobuoy logs -f
+```
+
+Once `sonobuoy status` shows the run as `completed`, copy the output directory from the main Sonobuoy pod to a local directory:
+
+```
+outfile=$(sonobuoy retrieve)
+```
+
+This copies a single `.tar.gz` snapshot from the Sonobuoy pod into your local
+`.` directory. Extract the contents into `./results` with:
+
+```
+mkdir ./results; tar xzf $outfile -C ./results
+```
+
+To clean up Kubernetes objects created by Sonobuoy, run:
+
+```
+sonobuoy delete
+```
+
+
+

docs/getting-started-with-kamaji.md

@@ -1,6 +1,6 @@
 # Setup a minimal Kamaji for development
 
-This document explains how to deploy a minimal Kamaji setup on [KinD](https://kind.sigs.k8s.io/) for development scopes. Please refer to the [Kamaji documentation](../README.md) for understanding all the terms used in this guide, as for example: `admin cluster` and `tenant control plane`.
+This document explains how to deploy a minimal Kamaji setup on [KinD](https://kind.sigs.k8s.io/) for development scopes. Please refer to the [Kamaji documentation](../concepts.md) for understanding all the terms used in this guide, as for example: `admin cluster`, `tenant cluster`, and `tenant control plane`.
 
 ## Pre-requisites
 
@@ -8,22 +8,26 @@ We assume you have installed on your workstation:
 
 - [Docker](https://docs.docker.com/engine/install/)
 - [KinD](https://kind.sigs.k8s.io/)
-- [kubectl](https://kubernetes.io/docs/tasks/tools/)
-- [kubeadm](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/install-kubeadm/)
+- [kubectl@v1.25.0](https://kubernetes.io/docs/tasks/tools/)
+- [kubeadm@v1.25.0](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/install-kubeadm/)
 - [jq](https://stedolan.github.io/jq/)
 - [openssl](https://www.openssl.org/)
 - [cfssl](https://github.com/cloudflare/cfssl)
 - [cfssljson](https://github.com/cloudflare/cfssl)
 
+
+> Starting from Kamaji v0.0.2, `kubectl` and `kubeadm` need to meet at least minimum version to `v1.25.0`:
+> this is required due to the latest changes addressed from the release Kubernetes 1.25 release regarding the `kubelet-config` ConfigMap required for the node join.
+
 ## Setup Kamaji on KinD
 
 The instance of Kamaji is made of a single node hosting:
 
 - admin control-plane
 - admin worker
-- multi-tenant etcd cluster
+- multi-tenant datastore
 
-### Standard
+### Standard installation
 
 You can install your KinD cluster, ETCD multi-tenant cluster and Kamaji operator with a **single command**:
 
@@ -35,68 +39,55 @@ Now you can [create your first `TenantControlPlane`](#deploy-tenant-control-plan
 
 ### Data store-specific
 
-#### ETCD
+Kamaji offers the possibility of using a different storage system than `ETCD` for the tenants, like `MySQL` or `PostgreSQL` compatible databases.
 
-The multi-tenant etcd cluster is deployed as statefulset into the Kamaji node.
-
-Run `make reqs` to setup Kamaji's requisites on KinD:
+First, setup a KinD cluster:
 
 ```bash
-$ make -C deploy/kind reqs
+$ make -C deploy/kind kind
 ```
 
-At this moment you will have your KinD up and running and ETCD cluster in multitenant mode.
+#### ETCD
+
+Deploy a multi-tenant `ETCD` cluster into the Kamaji node:
+
+```bash
+$ make -C deploy/kind etcd-cluster
+```
 
 Now you're ready to [install Kamaji operator](#install-kamaji).
 
-#### Kine MySQL
+#### MySQL
 
-> The MySQL-compatible cluster provisioning is omitted here.
+Deploy a MySQL/MariaDB backend into the Kamaji node:
 
-Kamaji offers the possibility of using a different storage system than `ETCD` for the tenants, like MySQL compatible databases.
-
-Once a compatible-mysql database is running, we need to provide information about it to kamaji:
-
-```
---etcd-storage-type=kine-mysql
---kine-mysql-host=<database host>
---kine-mysql-port=<database port>
---kine-mysql-secret-name=<secret name>
---kine-mysql-secret-namespace=<secret namespace>
+```bash
+$ make -C deploy/kine/mysql mariadb
 ```
 
-The secret with the configuration and certificates for mysql should look like:
-```yaml
-apiVersion: v1
-data:
-  MYSQL_ROOT_PASSWORD: ...
-  ca.crt: ...
-  ca.key: ...
-  mysql-ssl.cnf: ...
-  server.crt:  ...
-  server.key:  ...
-kind: Secret
-metadata:
-  creationTimestamp: "2022-06-30T08:03:15Z"
-  name: mysql-config
-  namespace: kamaji-system
-  resourceVersion: "32228"
-  uid: 51b155a1-426c-42d2-8147-be680bf458a6
-type: Opaque
+Adjust the [Kamaji install manifest](../config/install.yaml) according to the example of a [MySQL DataStore](../config/samples/kamaji_v1alpha1_datastore_mysql.yaml) and make sure Kamaji uses the proper datastore name:
+
+```
+--datastore={.metadata.name}
 ```
 
-and `mysql-ssl.cnf`:
-```
-[mysqld]
-ssl-ca=/etc/mysql/conf.d/ca.crt
-ssl-cert=/etc/mysql/conf.d/server.crt
-ssl-key=/etc/mysql/conf.d/server.key
-require_secure_transport=ON
+Now you're ready to [install Kamaji operator](#install-kamaji).
+
+#### PostgreSQL
+
+Deploy a PostgreSQL backend into the Kamaji node:
+
+```bash
+$ make -C deploy/kine/postgresql postgresql
 ```
 
-You can read more about it [here](../deploy/mysql/README.md).
+Adjust the [Kamaji install manifest](../config/install.yaml) according to the example of a [PostgreSQL DataStore](../config/samples/kamaji_v1alpha1_datastore_postgresql.yaml) and make sure Kamaji uses the proper datastore name:
 
-Assuming you adjusted the [Kamaji manifest](./config/install.yaml) to connect to the MySQL-compatible database, you can now install it.
+```
+--datastore={.metadata.name}
+```
+
+Now you're ready to [install Kamaji operator](#install-kamaji).
 
 ### Install Kamaji
 
@@ -104,6 +95,8 @@ Assuming you adjusted the [Kamaji manifest](./config/install.yaml) to connect to
 $ kubectl apply -f config/install.yaml
 ```
 
+> If you experience some errors during the apply of the manifest as `resource mapping not found ... ensure CRDs are installed first`, just apply it again.
+
 ### Deploy Tenant Control Plane
 
 Now it is the moment of deploying your first tenant control plane.
@@ -134,8 +127,6 @@ spec:
           tenant.clastix.io: tenant1
           kind.clastix.io: service
       serviceType: NodePort
-    ingress:
-      enabled: false
   kubernetes:
     version: "v1.23.4"
     kubelet:
@@ -188,7 +179,7 @@ $ kubectl create -f https://raw.githubusercontent.com/aojea/kindnet/master/insta
 ### Join worker nodes
 
 ```bash
-$ make kamaji-kind-worker-join
+$ make -C deploy/kind kamaji-kind-worker-join
 ```
 
 > To add more worker nodes, run again the command above.
@@ -201,12 +192,12 @@ NAME           STATUS   ROLES    AGE   VERSION
 d2d4b468c9de   Ready    <none>   44s   v1.23.4
 ```
 
-> For more complex scenarios (exposing port, different version and so on), run `join-node.bash`
+> For more complex scenarios (exposing port, different version and so on), run `join-node.bash`.
 
 Tenant control plane provision has been finished in a minimal Kamaji setup based on KinD. Therefore, you could develop, test and make your own experiments with Kamaji.
 
 ## Cleanup
 
 ```bash
-$ make destroy
+$ make -C deploy/kind destroy
 ```

docs/kamaji-azure-deployment-guide.md

@@ -1,60 +1,90 @@
 # Setup Kamaji on Azure
+This guide will lead you through the process of creating a working Kamaji setup on on MS Azure.
 
-In this section, we're going to setup Kamaji on MS Azure:
+The material here is relatively dense. We strongly encourage you to dedicate time to walk through these instructions, with a mind to learning. We do NOT provide any "one-click" deployment here. However, once you've understood the components involved it is encouraged that you build suitable, auditable GitOps deployment processes around your final infrastructure.
 
-- one bootstrap local workstation
-- a regular AKS cluster as Kamaji Admin Cluster
-- a multi-tenant etcd internal cluster running on AKS
-- an arbitrary number of Azure virtual machines hosting `Tenant`s' workloads
+The guide requires:
 
-## Bootstrap machine
-This getting started guide is supposed to be run from a remote or local bootstrap machine.
-First, prepare the workspace directory:
+- one bootstrap workstation
+- an AKS Kubernetes cluster to run the Admin and Tenant Control Planes
+- an arbitrary number of Azure virtual machines to host `Tenant`s' workloads
 
-```
+## Summary
+
+  * [Prepare the bootstrap workspace](#prepare-the-bootstrap-workspace)
+  * [Access Admin cluster](#access-admin-cluster)
+  * [Install DataStore](#install-datastore)
+  * [Install Kamaji controller](#install-kamaji-controller)
+  * [Create Tenant Cluster](#create-tenant-cluster)
+  * [Cleanup](#cleanup)
+
+## Prepare the bootstrap workspace
+This guide is supposed to be run from a remote or local bootstrap machine. First, clone the repo and prepare the workspace directory:
+
+```bash
 git clone https://github.com/clastix/kamaji
 cd kamaji/deploy
 ```
 
-1. Follow the instructions in [Prepare the bootstrap workspace](./getting-started-with-kamaji.md#prepare-the-bootstrap-workspace).
-2. Install the [Azure CLI](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli).
-3. Make sure you have a valid Azure subscription
-4. Login to Azure:
+We assume you have installed on your workstation:
+
+- [kubectl](https://kubernetes.io/docs/tasks/tools/#kubectl)
+- [kubeadm](https://kubernetes.io/docs/tasks/tools/#kubeadm)
+- [helm](https://helm.sh/docs/intro/install/)
+- [jq](https://stedolan.github.io/jq/)
+- [Azure CLI](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli)
+
+Make sure you have a valid Azure subscription, and login to Azure:
 
 ```bash
 az account set --subscription "MySubscription"
 az login
 ```
-> Currently, the Kamaji setup, including Admin and Tenant clusters need to be deployed within the same Azure region. Cross-regions deployments are not (yet) validated.
+> Currently, the Kamaji setup, including Admin and Tenant clusters need to be deployed within the same Azure region. Cross-regions deployments are not supported.
 
-## Setup Admin cluster on AKS
-Throughout the instructions, shell variables are used to indicate values that you should adjust to your own Azure environment:
+## Access Admin cluster
+In Kamaji, an Admin Cluster is a regular Kubernetes cluster which hosts zero to many Tenant Cluster Control Planes. The admin cluster acts as management cluster for all the Tenant clusters and implements Monitoring, Logging, and Governance of all the Kamaji setup, including all Tenant clusters. For this guide, we're going to use an instance of Azure Kubernetes Service - AKS as the Admin Cluster.
+
+Throughout the following instructions, shell variables are used to indicate values that you should adjust to your own Azure environment:
 
 ```bash
 source kamaji-azure.env
-```
 
-> we use the Azure CLI to setup the Kamaji Admin cluster on AKS.
-
-```
 az group create \
   --name $KAMAJI_RG \
   --location $KAMAJI_REGION
 
+az network vnet create \
+  --resource-group $KAMAJI_RG \
+  --name $KAMAJI_VNET_NAME \
+  --location $KAMAJI_REGION \
+  --address-prefix $KAMAJI_VNET_ADDRESS
+
+az network vnet subnet create \
+  --resource-group $KAMAJI_RG \
+  --name $KAMAJI_SUBNET_NAME \
+  --vnet-name $KAMAJI_VNET_NAME \
+  --address-prefixes $KAMAJI_SUBNET_ADDRESS
+
+KAMAJI_SUBNET_ID=$(az network vnet subnet show \
+  --resource-group ${KAMAJI_RG} \
+  --vnet-name ${KAMAJI_VNET_NAME} \
+  --name ${KAMAJI_SUBNET_NAME} \
+  --query id --output tsv)
+
 az aks create \
   --resource-group $KAMAJI_RG \
   --name $KAMAJI_CLUSTER \
   --location $KAMAJI_REGION \
+  --vnet-subnet-id $KAMAJI_SUBNET_ID \
   --zones 1 2 3 \
   --node-count 3 \
-  --nodepool-name $KAMAJI_CLUSTER \
-  --ssh-key-value @~/.ssh/id_rsa.pub \
-  --no-wait
+  --nodepool-name $KAMAJI_CLUSTER
 ```
 
 Once the cluster formation succedes, get credentials to access the cluster as admin
 
-```
+```bash
 az aks get-credentials  \
   --resource-group $KAMAJI_RG \
   --name $KAMAJI_CLUSTER
@@ -62,38 +92,56 @@ az aks get-credentials  \
 
 And check you can access:
 
-```
+```bash
 kubectl cluster-info
 ```
 
-## Setup internal multi-tenant etcd
-Follow the instructions [here](./kamaji-deployment-guide.md#setup-internal-multi-tenant-etcd).
+## Install datastore
+The Kamaji controller needs to access a multi-tenant datastore in order to save data of the tenants' clusters. The [Helm Chart](../charts/kamaji/) provides the installation of an unamanaged `etcd`. However, a managed `etcd` is highly recommended in production.
 
-## Install Kamaji controller
-Follow the instructions [here](./kamaji-deployment-guide.md#install-kamaji-controller).
+The [kamaji-etcd](https://github.com/clastix/kamaji-etcd) project provides a viable option to setup a manged multi-tenant `etcd` as 3 replicas StatefulSet with data persistence:
 
-## Create Tenant Clusters
-To create a Tenant Cluster in Kamaji on AKS, we have to work on both the Kamaji and Azure infrastructure sides.
-
-```
-source kamaji-tenant-azure.env
+```bash
+helm repo add clastix https://clastix.github.io/charts
+helm repo update
+helm install etcd clastix/kamaji-etcd -n kamaji-system --create-namespace
 ```
 
-### On Kamaji side
+Optionally, Kamaji offers the possibility of using a different storage system for the tenants' clusters, as MySQL or PostgreSQL compatible database, thanks to the native [kine](https://github.com/k3s-io/kine) integration.
+
+## Install Kamaji Controller
+There are multiple ways to deploy Kamaji, including a [single YAML file](../config/install.yaml) and the [Helm Chart](../charts/kamaji).
+
+Install with `helm` using an unmanaged `etcd` as datastore:
+
+```bash
+helm repo add clastix https://clastix.github.io/charts
+helm repo update
+helm install kamaji clastix/kamaji -n kamaji-system --create-namespace
+```
+
+Alternatively, if you opted for a managed `etcd` datastore:
+
+```
+helm repo add clastix https://clastix.github.io/charts
+helm repo update
+helm install kamaji clastix/kamaji -n kamaji-system --create-namespace --set etcd.deploy=false    
+```
+
+Congratulations! You just turned your Azure Kubernetes AKS cluster into a Kamaji cluster capable to run multiple Tenant Control Planes.
+
+## Create Tenant Cluster
+
+### Tenant Control Plane
 With Kamaji on AKS, the tenant control plane is accessible:
 
-- from tenant work nodes through an internal loadbalancer as `https://${TENANT_ADDR}:6443`
-- from tenant admin user through an external loadbalancer `https://${TENANT_NAME}.${KAMAJI_REGION}.cloudapp.azure.com:443`
+- from tenant worker nodes through an internal loadbalancer
+- from tenant admin user through an external loadbalancer responding to `https://${TENANT_NAME}.${TENANT_NAME}.${TENANT_DOMAIN}:443`
 
-Where `TENANT_ADDR` is the Azure internal IP address assigned to the LoadBalancer service created by Kamaji to expose the Tenant Control Plane endpoint.
-
-#### Create the Tenant Control Plane
-
-Create the manifest for Tenant Control Plane:
+Create a tenant control plane of example:
 
 ```yaml
 cat > ${TENANT_NAMESPACE}-${TENANT_NAME}-tcp.yaml <<EOF
----
 apiVersion: kamaji.clastix.io/v1alpha1
 kind: TenantControlPlane
 metadata:
@@ -102,24 +150,37 @@ metadata:
 spec:
   controlPlane:
     deployment:
-      replicas: 2
+      replicas: 3
       additionalMetadata:
-        annotations:
-          environment.clastix.io: ${TENANT_NAME}
         labels:
           tenant.clastix.io: ${TENANT_NAME}
-          kind.clastix.io: deployment
+      extraArgs:
+        apiServer: []
+        controllerManager: []
+        scheduler: []
+      resources:
+        apiServer:
+          requests:
+            cpu: 250m
+            memory: 512Mi
+          limits: {}
+        controllerManager:
+          requests:
+            cpu: 125m
+            memory: 256Mi
+          limits: {}
+        scheduler:
+          requests:
+            cpu: 125m
+            memory: 256Mi
+          limits: {} 
     service:
       additionalMetadata:
-        annotations:
-          environment.clastix.io: ${TENANT_NAME}
-          service.beta.kubernetes.io/azure-load-balancer-internal: "true"
         labels:
           tenant.clastix.io: ${TENANT_NAME}
-          kind.clastix.io: service
+        annotations:
+          service.beta.kubernetes.io/azure-load-balancer-internal: "true"
       serviceType: LoadBalancer
-    ingress:
-      enabled: false    
   kubernetes:
     version: ${TENANT_VERSION}
     kubelet:
@@ -128,9 +189,9 @@ spec:
       - ResourceQuota
       - LimitRanger
   networkProfile:
-    port: 6443
+    port: ${TENANT_PORT}
     certSANs:
-    - ${TENANT_NAME}.${KAMAJI_REGION}.cloudapp.azure.com
+    - ${TENANT_NAME}.${TENANT_DOMAIN}
     serviceCidr: ${TENANT_SVC_CIDR}
     podCidr: ${TENANT_POD_CIDR}
     dnsServiceIPs:
@@ -138,6 +199,13 @@ spec:
   addons:
     coreDNS: {}
     kubeProxy: {}
+    konnectivity:
+      proxyPort: ${TENANT_PROXY_PORT}
+      resources:
+        requests:
+          cpu: 100m
+          memory: 128Mi
+        limits: {}
 ---
 apiVersion: v1
 kind: Service
@@ -155,57 +223,31 @@ spec:
     kamaji.clastix.io/soot: ${TENANT_NAME}
   type: LoadBalancer
 EOF
+
+kubectl -n ${TENANT_NAMESPACE} apply -f ${TENANT_NAMESPACE}-${TENANT_NAME}-tcp.yaml
 ```
 
 Make sure:
 
-- the `tcp.spec.controlPlane.service.serviceType=LoadBalancer` and the following annotation: `service.beta.kubernetes.io/azure-load-balancer-internal=true` is set. This tells AKS to expose the service within an Azure internal loadbalancer.
+- the following annotation: `service.beta.kubernetes.io/azure-load-balancer-internal=true` is set on the `tcp` service. It tells Azure to expose the service within an internal loadbalancer.
 
-- the public loadbalancer service has the following annotation: `service.beta.kubernetes.io/azure-dns-label-name=${TENANT_NAME}` to expose the Tenant Control Plane with domain name: `${TENANT_NAME}.${KAMAJI_REGION}.cloudapp.azure.com`.
+- the following annotation: `service.beta.kubernetes.io/azure-dns-label-name=${TENANT_NAME}` is set the public loadbalancer service. It tells Azure to expose the Tenant Control Plane with public domain name: `${TENANT_NAME}.${TENANT_DOMAIN}`.
 
-Create the Tenant Control Plane
+### Working with Tenant Control Plane
 
-```
-kubectl create namespace ${TENANT_NAMESPACE}
-kubectl apply -f ${TENANT_NAMESPACE}-${TENANT_NAME}-tcp.yaml
-```
-
-And check it out:
-
-```
-$ kubectl get tcp
-NAME        VERSION   CONTROL-PLANE-ENDPOINT   KUBECONFIG                   PRIVATE   AGE
-tenant-00   v1.23.4   10.240.0.100:6443        tenant-00-admin-kubeconfig   true      46m
-
-$ kubectl get svc
-NAME               TYPE           CLUSTER-IP     EXTERNAL-IP      PORT(S)          AGE
-tenant-00          LoadBalancer   10.0.223.161   10.240.0.100     6443:31902/TCP   46m
-tenant-00-public   LoadBalancer   10.0.205.97    20.101.215.149   443:30697/TCP    19h
-
-$ kubectl get deploy
-NAME        READY   UP-TO-DATE   AVAILABLE   AGE
-tenant-00   2/2     2            2           47m
-```
-
-Collect the internal IP address of Azure loadbalancer where the Tenant control Plane is exposed:
-
-```bash
-TENANT_ADDR=$(kubectl -n ${TENANT_NAMESPACE} get svc ${TENANT_NAME} -o json | jq -r ."status.loadBalancer.ingress[].ip")
-```
-
-#### Working with Tenant Control Plane
 Check the access to the Tenant Control Plane:
 
-```
+```bash
 curl -k https://${TENANT_NAME}.${KAMAJI_REGION}.cloudapp.azure.com/healthz
+curl -k https://${TENANT_NAME}.${KAMAJI_REGION}.cloudapp.azure.com/version
 ```
 
 Let's retrieve the `kubeconfig` in order to work with it:
 
-```
+```bash
 kubectl get secrets -n ${TENANT_NAMESPACE} ${TENANT_NAME}-admin-kubeconfig -o json \
   | jq -r '.data["admin.conf"]' \
-  | base64 -d \
+  | base64 --decode \
   > ${TENANT_NAMESPACE}-${TENANT_NAME}.kubeconfig
 
 kubectl --kubeconfig=${TENANT_NAMESPACE}-${TENANT_NAME}.kubeconfig config \
@@ -231,176 +273,112 @@ NAME         ENDPOINTS           AGE
 kubernetes   10.240.0.100:6443   57m
 ```
 
-Make sure it's `${TENANT_ADDR}:6443`.
+### Preparing Worker Nodes to join
+Currently Kamaji does not provide any helper for creation of tenant worker nodes. You should get a set of machines from your infrastructure provider, turn them into worker nodes, and then join to the tenant control plane with the `kubeadm`. In the future, we'll provide integration with Cluster APIs and other tools, as for example, Terrform.
 
-### Prepare the Infrastructure for the Tenant virtual machines
-Kamaji provides Control Plane as a Service, so the tenant user can join his own virtual machines as worker nodes. Each tenant can place his virtual machines in a dedicated Azure virtual network.
-
-Prepare the Tenant infrastructure:
-
-```
-az group create \
-    --name $TENANT_RG \
-    --location $KAMAJI_REGION
-
-az network nsg create \
-    --resource-group $TENANT_RG \
-    --name $TENANT_NSG
-
-az network nsg rule create \
-    --resource-group $TENANT_RG \
-    --nsg-name $TENANT_NSG \
-    --name $TENANT_NSG-ssh \
-    --protocol tcp \
-    --priority 1000 \
-    --destination-port-range 22 \
-    --access allow
-
-az network vnet create \
-    --resource-group $TENANT_RG \
-    --name $TENANT_VNET_NAME \
-    --address-prefix $TENANT_VNET_ADDRESS \
-    --subnet-name $TENANT_SUBNET_NAME \
-    --subnet-prefix $TENANT_SUBNET_ADDRESS
-
-az network vnet subnet create \
-   --resource-group $TENANT_RG \
-   --vnet-name $TENANT_VNET_NAME \
-   --name $TENANT_SUBNET_NAME \
-   --address-prefixes $TENANT_SUBNET_ADDRESS \
-   --network-security-group $TENANT_NSG
-```
-
-Connection between the Tenant virtual network and the Kamaji AKS virtual network leverages on the [Azure Network Peering](https://docs.microsoft.com/en-us/azure/virtual-network/virtual-network-peering-overview).
-
-Enable the network peering between the Tenant Virtual Network and the Kamaji AKS Virtual Network:
+Create an Azure VM Stateful Set to host worker nodes
 
 ```bash
-KAMAJI_VNET_NAME=`az network vnet list -g $KAMAJI_NODE_RG --query [].name --out tsv`
-KAMAJI_VNET_ID=`az network vnet list -g $KAMAJI_NODE_RG --query [].id --out tsv`
-TENANT_VNET_ID=`az network vnet list -g $TENANT_RG --query [].id --out tsv`
-
-az network vnet peering create \
-   --resource-group $TENANT_RG \
-   --name $TENANT_NAME-$KAMAJI_CLUSTER \
-   --vnet-name $TENANT_VNET_NAME \
-   --remote-vnet $KAMAJI_VNET_ID \
-   --allow-vnet-access
-
-az network vnet peering create \
-   --resource-group $KAMAJI_NODE_RG \
-   --name $KAMAJI_CLUSTER-$TENANT_NAME \
+az network vnet subnet create \
+   --resource-group $KAMAJI_RG \
+   --name $TENANT_SUBNET_NAME \
    --vnet-name $KAMAJI_VNET_NAME \
-   --remote-vnet $TENANT_VNET_ID \
-   --allow-vnet-access
-```
+   --address-prefixes $TENANT_SUBNET_ADDRESS
 
-[Azure Network Security Groups](https://docs.microsoft.com/en-us/azure/virtual-network/network-security-groups-overview) can be used to control the traffic between the Tenant virtual network and the Kamaji AKS virtual network for a stronger isolation. See the required [ports and protocols](https://kubernetes.io/docs/reference/ports-and-protocols/) between Kubernetes control plane and worker nodes. 
-
-### Create the tenant virtual machines
-Create an Azure VM Stateful Set to host virtual machines
-
-```
 az vmss create \
    --name $TENANT_VMSS \
-   --resource-group $TENANT_RG \
+   --resource-group $KAMAJI_RG \
    --image $TENANT_VM_IMAGE \
-   --public-ip-per-vm \
-   --vnet-name $TENANT_VNET_NAME \
+   --vnet-name $KAMAJI_VNET_NAME \
    --subnet $TENANT_SUBNET_NAME \
-   --ssh-key-value @~/.ssh/id_rsa.pub \
    --computer-name-prefix $TENANT_NAME- \
-   --nsg $TENANT_NSG \
    --custom-data ./tenant-cloudinit.yaml \
-   --instance-count 0 
+   --load-balancer "" \
+   --instance-count 0
 
 az vmss update \
-   --resource-group $TENANT_RG \
+   --resource-group $KAMAJI_RG \
    --name $TENANT_VMSS \
    --set virtualMachineProfile.networkProfile.networkInterfaceConfigurations[0].enableIPForwarding=true
 
 az vmss scale \
-   --resource-group $TENANT_RG \
+   --resource-group $KAMAJI_RG \
    --name $TENANT_VMSS \
    --new-capacity 3
 ```
 
 ### Join the tenant virtual machines to the tenant control plane
-The current approach for joining nodes is to use the `kubeadm` one therefore, we will create a bootstrap token to perform the action:
+The current approach for joining nodes is to use `kubeadm` and therefore, we will create a bootstrap token to perform the action. In order to facilitate the step, we will store the entire command of joining in a variable:
 
 ```bash
+TENANT_ADDR=$(kubectl -n ${TENANT_NAMESPACE} get svc ${TENANT_NAME} -o json | jq -r ."spec.loadBalancerIP")
+
 JOIN_CMD=$(echo "sudo kubeadm join ${TENANT_ADDR}:6443 ")$(kubeadm --kubeconfig=${TENANT_NAMESPACE}-${TENANT_NAME}.kubeconfig token create --print-join-command |cut -d" " -f4-)
 ```
 
 A bash loop will be used to join all the available nodes.
 
-
 ```bash
-HOSTS=($(az vmss list-instance-public-ips \
-    --resource-group $TENANT_RG \
-    --name $TENANT_VMSS \
-    --query "[].ipAddress" \
-    --output tsv))
+VMIDS=($(az vmss list-instances \
+   --resource-group $KAMAJI_RG \
+   --name $TENANT_VMSS \
+   --query [].instanceId \
+   --output tsv))
 
-for i in ${!HOSTS[@]}; do
-  HOST=${HOSTS[$i]}
-  echo $HOST
-  ssh ${USER}@${HOST} -t ${JOIN_CMD};
+for i in ${!VMIDS[@]}; do
+  VMID=${VMIDS[$i]}
+  az vmss run-command create \
+	  --name join-tenant-control-plane \
+	  --vmss-name  $TENANT_VMSS \
+	  --resource-group $KAMAJI_RG \
+	  --instance-id ${VMID} \
+	  --script "${JOIN_CMD}"
 done
 ```
 
 Checking the nodes:
 
 ```bash
-kubectl get nodes --kubeconfig=${CLUSTER_NAMESPACE}-${CLUSTER_NAME}.kubeconfig
+kubectl --kubeconfig=${TENANT_NAMESPACE}-${TENANT_NAME}.kubeconfig get nodes
 
-NAME                      STATUS      ROLES    AGE    VERSION
-kamaji-tenant-worker-00   NotReady    <none>   1m     v1.23.4
-kamaji-tenant-worker-01   NotReady    <none>   1m     v1.23.4
-kamaji-tenant-worker-02   NotReady    <none>   1m     v1.23.4
-kamaji-tenant-worker-03   NotReady    <none>   1m     v1.23.4
+NAME               STATUS     ROLES    AGE    VERSION
+tenant-00-000000   NotReady   <none>   112s   v1.25.0
+tenant-00-000002   NotReady   <none>   92s    v1.25.0
+tenant-00-000003   NotReady   <none>   71s    v1.25.0
 ```
 
-The cluster needs a [CNI](https://kubernetes.io/docs/concepts/extend-kubernetes/compute-storage-net/network-plugins/) plugin to get the nodes ready. In our case, we are going to install [calico](https://projectcalico.docs.tigera.io/about/about-calico).
+The cluster needs a [CNI](https://kubernetes.io/docs/concepts/extend-kubernetes/compute-storage-net/network-plugins/) plugin to get the nodes ready. In this guide, we are going to install [calico](https://projectcalico.docs.tigera.io/about/about-calico), but feel free to use one of your taste.
+
+Download the latest stable Calico manifest:
 
 ```bash
-kubectl apply -f calico-cni/calico-crd.yaml --kubeconfig=${TENANT_NAMESPACE}-${TENANT_NAME}.kubeconfig
-kubectl apply -f calico-cni/calico-azure.yaml --kubeconfig=${TENANT_NAMESPACE}-${TENANT_NAME}.kubeconfig
+curl https://raw.githubusercontent.com/projectcalico/calico/v3.24.1/manifests/calico.yaml -O
 ```
 
-And after a while, `kube-system` pods will be running.
+As per [documentation](https://projectcalico.docs.tigera.io/reference/public-cloud/azure), Calico in VXLAN mode is supported on Azure while IPIP packets are blocked by the Azure network fabric. Make sure you edit the manifest above and set the following variables:
+
+- `CLUSTER_TYPE="k8s"`
+- `CALICO_IPV4POOL_IPIP="Never"`
+- `CALICO_IPV4POOL_VXLAN="Always"`
+
+Apply to the tenant cluster:
 
 ```bash
-kubectl get po -n kube-system --kubeconfig=${TENANT_NAMESPACE}-${TENANT_NAME}.kubeconfig
-
-NAME                                       READY   STATUS    RESTARTS   AGE
-calico-kube-controllers-8594699699-dlhbj   1/1     Running   0          3m
-calico-node-kxf6n                          1/1     Running   0          3m
-calico-node-qtdlw                          1/1     Running   0          3m
-coredns-64897985d-2v5lc                    1/1     Running   0          5m
-coredns-64897985d-nq276                    1/1     Running   0          5m
-kube-proxy-cwdww                           1/1     Running   0          3m
-kube-proxy-m48v4                           1/1     Running   0          3m
+kubectl --kubeconfig=${TENANT_NAMESPACE}-${TENANT_NAME}.kubeconfig apply -f calico.yaml
 ```
 
-And the nodes will be ready
+And after a while, nodes will be ready
 
 ```bash
-kubectl get nodes --kubeconfig=${TENANT_NAMESPACE}-${TENANT_NAME}.kubeconfig
+kubectl --kubeconfig=${TENANT_NAMESPACE}-${TENANT_NAME}.kubeconfig get nodes 
 
-NAME                      STATUS      ROLES    AGE    VERSION
-kamaji-tenant-worker-01   Ready       <none>   10m    v1.23.4
-kamaji-tenant-worker-02   Ready       <none>   10m    v1.23.4
+NAME               STATUS   ROLES    AGE     VERSION
+tenant-00-000000   Ready    <none>   3m38s   v1.25.0
+tenant-00-000002   Ready    <none>   3m18s   v1.25.0
+tenant-00-000003   Ready    <none>   2m57s   v1.25.0
 ```
 
-
 ## Cleanup
-To get rid of the Tenant infrastructure, remove the RESOURCE_GROUP:
-
-```
-az group delete --name $TENANT_RG --yes --no-wait
-```
-
 To get rid of the Kamaji infrastructure, remove the RESOURCE_GROUP:
 
 ```