chore(golangci-lint): updating to v1.49.0

Signed-off-by: Viktor Oreshkin <imselfish@stek29.rocks>

.github/workflows/ci.yaml

@@ -12,10 +12,13 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
+      - uses: actions/setup-go@v2
+        with:
+          go-version: '1.18'
       - name: Run golangci-lint
         uses: golangci/golangci-lint-action@v2.3.0
         with:
-          version: v1.45.2
+          version: v1.49.0
           only-new-issues: false
           args: --timeout 5m --config .golangci.yml
   diff:
@@ -27,7 +30,7 @@ jobs:
           fetch-depth: 0
       - uses: actions/setup-go@v2
         with:
-          go-version: '1.17'
+          go-version: '1.18'
       - run: make yaml-installation-file
       - name: Checking if YAML installer file is not aligned
         run: if [[ $(git diff | wc -l) -gt 0 ]]; then echo ">>> Untracked generated files have not been committed" && git --no-pager diff && exit 1; fi

.github/workflows/docker-ci.yml

@@ -0,0 +1,74 @@
+name: docker-ci
+
+on:
+  push:
+    tags:        
+      - "v*"
+
+jobs:
+  docker-ci:
+    runs-on: ubuntu-20.04
+    steps:
+
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v3
+        with:
+          images: |
+             quay.io/${{ github.repository }}
+             docker.io/${{ github.repository }}
+          tags: |
+            type=semver,pattern={{raw}}
+          flavor: |
+            latest=false
+
+      - name: Set up QEMU
+        id: qemu
+        uses: docker/setup-qemu-action@v1
+        with:
+          platforms: arm64,arm
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v1
+        with:
+          install: true
+
+      - name: Inspect builder
+        run: |
+          echo "Name:      ${{ steps.buildx.outputs.name }}"
+          echo "Endpoint:  ${{ steps.buildx.outputs.endpoint }}"
+          echo "Status:    ${{ steps.buildx.outputs.status }}"
+          echo "Flags:     ${{ steps.buildx.outputs.flags }}"
+          echo "Platforms: ${{ steps.buildx.outputs.platforms }}"
+
+      - name: Login to quay.io Container Registry
+        uses: docker/login-action@v1 
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_IO_USERNAME }}
+          password: ${{ secrets.QUAY_IO_TOKEN }}
+
+      - name: Login to docker.io Container Registry
+        uses: docker/login-action@v1 
+        with:
+          registry: docker.io
+          username: ${{ secrets.DOCKER_IO_USERNAME }}
+          password: ${{ secrets.DOCKER_IO_TOKEN }}
+
+      - name: Build and push
+        id: build-release
+        uses: docker/build-push-action@v2
+        with:
+          file: Dockerfile
+          context: .
+          platforms: linux/amd64,linux/arm64,linux/arm
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          build-args:
+
+      - name: Image digest
+        run: echo ${{ steps.build-release.outputs.digest }}

.github/workflows/e2e.yaml

@@ -0,0 +1,44 @@
+name: e2e
+
+on:
+  push:
+    branches: [ "*" ]
+    paths:
+      - '.github/workflows/e2e.yml'
+      - 'api/**'
+      - 'controllers/**'
+      - 'e2e/*'
+      - 'Dockerfile'
+      - 'go.*'
+      - 'main.go'
+      - 'Makefile'
+      - 'internal/**'
+  pull_request:
+    branches: [ "*" ]
+    paths:
+      - '.github/workflows/e2e.yml'
+      - 'api/**'
+      - 'controllers/**'
+      - 'e2e/*'
+      - 'Dockerfile'
+      - 'go.*'
+      - 'main.go'
+      - 'Makefile'
+      - 'internal/**'
+
+jobs:
+  kind:
+    name: Kubernetes
+    runs-on: ubuntu-18.04
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-go@v2
+        with:
+          go-version: '1.18'
+      - run: |
+          sudo apt-get update
+          sudo apt-get install -y golang-cfssl
+      - name: e2e testing
+        run: make e2e

.github/workflows/helm.yaml

@@ -0,0 +1,36 @@
+name: Helm Chart
+
+on:
+  push:
+    branches: [ "*" ]
+    tags: [ "helm-v*" ]
+  pull_request:
+    branches: [ "*" ]
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: azure/setup-helm@v1
+        with:
+          version: 3.3.4
+      - name: Linting Chart
+        run: helm lint ./charts/kamaji
+  release:
+    if: startsWith(github.ref, 'refs/tags/helm-v')
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Publish Helm chart
+        uses: stefanprodan/helm-gh-pages@master
+        with:
+          token: ${{ secrets.BOT_GITHUB_TOKEN }}
+          charts_dir: charts
+          charts_url: https://clastix.github.io/charts
+          owner: clastix
+          repository: charts
+          branch: gh-pages
+          target_dir: .
+          commit_username: prometherion
+          commit_email: dario@tranchitella.eu

.golangci.yml

@@ -4,6 +4,10 @@ linters-settings:
       - standard
       - default
       - prefix(github.com/clastix/kamaji)
+  goheader:
+    template: |-
+      Copyright 2022 Clastix Labs
+      SPDX-License-Identifier: Apache-2.0
 
 linters:
   disable:
@@ -23,6 +27,10 @@ linters:
     - exhaustivestruct
     - wsl
     - exhaustive
+    - nosprintfhostport
+    - nonamedreturns
+    - interfacebloat
+    - exhaustruct
     - lll
     - gosec
     - gomoddirectives

Dockerfile

@@ -1,5 +1,13 @@
 # Build the manager binary
-FROM golang:1.17 as builder
+FROM golang:1.18 as builder
+
+ARG TARGETARCH
+ARG GIT_HEAD_COMMIT
+ARG GIT_TAG_COMMIT
+ARG GIT_LAST_TAG
+ARG GIT_MODIFIED
+ARG GIT_REPO
+ARG BUILD_DATE
 
 WORKDIR /workspace
 # Copy the Go Modules manifests
@@ -16,7 +24,9 @@ COPY controllers/ controllers/
 COPY internal/ internal/
 
 # Build
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o manager main.go
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=$TARGETARCH go build \
+    -ldflags "-X github.com/clastix/kamaji/internal.GitRepo=$GIT_REPO -X github.com/clastix/kamaji/internal.GitTag=$GIT_LAST_TAG -X github.com/clastix/kamaji/internal.GitCommit=$GIT_HEAD_COMMIT -X github.com/clastix/kamaji/internal.GitDirty=$GIT_MODIFIED -X github.com/clastix/kamaji/internal.BuildTime=$BUILD_DATE" \
+    -a -o manager main.go
 
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details

Makefile

@@ -36,11 +36,9 @@ IMAGE_TAG_BASE ?= clastix.io/operator
 BUNDLE_IMG ?= $(IMAGE_TAG_BASE)-bundle:v$(VERSION)
 
 # Image URL to use all building/pushing image targets
-IMG ?= controller:latest
+IMG ?= clastix/kamaji:latest
 # Produce CRDs that work back to Kubernetes 1.11 (no version conversion)
 CRD_OPTIONS ?= "crd:trivialVersions=true,preserveUnknownFields=false"
-# ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary.
-ENVTEST_K8S_VERSION = 1.21
 
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
 ifeq (,$(shell go env GOBIN))
@@ -50,7 +48,6 @@ GOBIN=$(shell go env GOBIN)
 endif
 
 # Setting SHELL to bash allows bash commands to be executed by recipes.
-# This is a requirement for 'setup-envtest.sh' in the test target.
 # Options are set to exit when a recipe line exits non-zero or a piped command fails.
 SHELL = /usr/bin/env bash -o pipefail
 .SHELLFLAGS = -ec
@@ -73,33 +70,66 @@ all: build
 help: ## Display this help.
 	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf "  \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
 
+##@ Binary
+
+.PHONY: helm
+HELM = $(shell pwd)/bin/helm
+helm: ## Download helm locally if necessary.
+	$(call go-install-tool,$(HELM),helm.sh/helm/v3/cmd/helm@v3.9.0)
+
+GINKGO = $(shell pwd)/bin/ginkgo
+ginkgo: ## Download ginkgo locally if necessary.
+	$(call go-install-tool,$(GINKGO),github.com/onsi/ginkgo/ginkgo@v1.16.5)
+
+KIND = $(shell pwd)/bin/kind
+kind: ## Download kind locally if necessary.
+	$(call go-install-tool,$(KIND),sigs.k8s.io/kind/cmd/kind@v0.14.0)
+
+CONTROLLER_GEN = $(shell pwd)/bin/controller-gen
+controller-gen: ## Download controller-gen locally if necessary.
+	$(call go-install-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.6.1)
+
+KUSTOMIZE = $(shell pwd)/bin/kustomize
+kustomize: ## Download kustomize locally if necessary.
+	$(call install-kustomize,$(KUSTOMIZE),3.8.7)
+
 ##@ Development
 
 manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects.
 	$(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=manager-role webhook paths="./..." output:crd:artifacts:config=config/crd/bases
+	cp config/crd/bases/kamaji.clastix.io_tenantcontrolplanes.yaml charts/kamaji/crds/tenantcontrolplane.yaml
+	cp config/crd/bases/kamaji.clastix.io_datastores.yaml charts/kamaji/crds/datastore.yaml
 
 generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations.
 	$(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..."
 
-fmt: ## Run go fmt against code.
-	go fmt ./...
-
-vet: ## Run go vet against code.
-	go vet ./...
-
-test: manifests generate fmt vet envtest ## Run tests.
-	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" go test ./... -coverprofile cover.out
+test:
+	go test ./... -coverprofile cover.out
 
 ##@ Build
 
+# Get information about git current status
+GIT_HEAD_COMMIT ?= $$(git rev-parse --short HEAD)
+GIT_TAG_COMMIT  ?= $$(git rev-parse --short $(VERSION))
+GIT_MODIFIED_1  ?= $$(git diff $(GIT_HEAD_COMMIT) $(GIT_TAG_COMMIT) --quiet && echo "" || echo ".dev")
+GIT_MODIFIED_2  ?= $$(git diff --quiet && echo "" || echo ".dirty")
+GIT_MODIFIED    ?= $$(echo "$(GIT_MODIFIED_1)$(GIT_MODIFIED_2)")
+GIT_REPO        ?= $$(git config --get remote.origin.url)
+BUILD_DATE      ?= $$(git log -1 --format="%at" | xargs -I{} date -d @{} +%Y-%m-%dT%H:%M:%S)
+
 build: generate fmt vet ## Build manager binary.
 	go build -o bin/manager main.go
 
 run: manifests generate fmt vet ## Run a controller from your host.
 	go run ./main.go
 
-docker-build: test ## Build docker image with the manager.
-	docker build -t ${IMG} .
+docker-build: ## Build docker image with the manager.
+	docker build -t ${IMG} . --build-arg GIT_HEAD_COMMIT=$(GIT_HEAD_COMMIT) \
+		--build-arg GIT_TAG_COMMIT=$(GIT_TAG_COMMIT) \
+		--build-arg GIT_MODIFIED=$(GIT_MODIFIED) \
+		--build-arg GIT_REPO=$(GIT_REPO) \
+		--build-arg GIT_LAST_TAG=$(VERSION) \
+		--build-arg BUILD_DATE=$(BUILD_DATE)
 
 docker-push: ## Push docker image with the manager.
 	docker push ${IMG}
@@ -109,8 +139,8 @@ docker-push: ## Push docker image with the manager.
 dev: generate manifests uninstall install rbac ## Full installation for development purposes
 	go fmt ./...
 
-load: dev docker-build
-	kind load docker-image --name kamaji ${IMG}
+load: docker-build kind
+	$(KIND) load docker-image --name kamaji ${IMG}
 
 rbac: manifests kustomize ## Install RBAC into the K8s cluster specified in ~/.kube/config.
 	$(KUSTOMIZE) build config/rbac | kubectl apply -f -
@@ -131,33 +161,6 @@ undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/confi
 yaml-installation-file: manifests kustomize ## Create yaml installation file
 	$(KUSTOMIZE) build config/default > config/install.yaml
 
-
-CONTROLLER_GEN = $(shell pwd)/bin/controller-gen
-controller-gen: ## Download controller-gen locally if necessary.
-	$(call go-get-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.6.1)
-
-KUSTOMIZE = $(shell pwd)/bin/kustomize
-kustomize: ## Download kustomize locally if necessary.
-	$(call go-get-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/v3@v3.8.7)
-
-ENVTEST = $(shell pwd)/bin/setup-envtest
-envtest: ## Download envtest-setup locally if necessary.
-	$(call go-get-tool,$(ENVTEST),sigs.k8s.io/controller-runtime/tools/setup-envtest@latest)
-
-# go-get-tool will 'go get' any package $2 and install it to $1.
-PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST))))
-define go-get-tool
-@[ -f $(1) ] || { \
-set -e ;\
-TMP_DIR=$$(mktemp -d) ;\
-cd $$TMP_DIR ;\
-go mod init tmp ;\
-echo "Downloading $(2)" ;\
-GOBIN=$(PROJECT_DIR)/bin go get $(2) ;\
-rm -rf $$TMP_DIR ;\
-}
-endef
-
 .PHONY: bundle
 bundle: manifests kustomize ## Generate bundle manifests and metadata, then validate generated files.
 	operator-sdk generate kustomize manifests -q
@@ -213,3 +216,34 @@ catalog-build: opm ## Build a catalog image.
 .PHONY: catalog-push
 catalog-push: ## Push a catalog image.
 	$(MAKE) docker-push IMG=$(CATALOG_IMG)
+
+define install-kustomize
+@[ -f $(1) ] || { \
+set -e ;\
+echo "Installing v$(2)" ;\
+cd bin ;\
+wget "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" ;\
+bash ./install_kustomize.sh $(2) ;\
+}
+endef
+
+# go-install-tool will 'go install' any package $2 and install it to $1.
+PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST))))
+define go-install-tool
+@[ -f $(1) ] || { \
+set -e ;\
+echo "Installing $(2)" ;\
+GOBIN=$(PROJECT_DIR)/bin go install $(2) ;\
+}
+endef
+
+.PHONY: env
+env:
+	@make -C deploy/kind kind ingress-nginx
+
+##@ e2e
+
+.PHONY: e2e
+e2e: env load helm ginkgo ## Create a KinD cluster, install Kamaji on it and run the test suite.
+	$(HELM) upgrade --debug --install kamaji ./charts/kamaji --create-namespace --namespace kamaji-system --set "image.pullPolicy=Never"
+	$(GINKGO) -v ./e2e

PROJECT

@@ -16,4 +16,12 @@ resources:
   kind: TenantControlPlane
   path: github.com/clastix/kamaji/api/v1alpha1
   version: v1alpha1
+- api:
+    crdVersion: v1
+    namespaced: false
+  domain: clastix.io
+  group: kamaji
+  kind: DataStore
+  path: github.com/clastix/kamaji/api/v1alpha1
+  version: v1alpha1
 version: "3"

README.md

@@ -8,25 +8,26 @@
   </a>
 </p>
 
-**Kamaji** is a tool aimed to build and operate a **Managed Kubernetes Service** with a fraction of the operational burden. With **Kamaji**, you can deploy and operate hundreds of Kubernetes clusters as a hyper-scale cloud provider.
+**Kamaji** deploys and operates **Kubernetes** at scale with a fraction of the operational burden.
 
 <p align="center" style="padding: 6px 6px">
   <img src="assets/kamaji-logo.png" />
 </p>
 
-> This project is still in the early development stage which means it's not ready for production as APIs, commands, flags, etc. are subject to change, but also that your feedback can still help to shape it. Please try it out and let us know what you like, dislike, what works, what doesn't, etc.
-
 ## Why we are building it?
-Global hyper-scalers are leading the Managed Kubernetes space, while regional cloud providers, as well as large corporations, are struggling to offer the same level of experience to their developers because of the lack of the right tools. Also, Kubernetes solutions for the on-premises are designed with an enterprise-first approach and they are too costly when deployed at scale. Project Kamaji aims to solve this pain by leveraging multi-tenancy and simplifying how to run Kubernetes clusters at scale with a fraction of the operational burden.
+Global hyper-scalers are leading the Managed Kubernetes space, while other cloud providers, as well as large corporations, are struggling to offer the same experience to their DevOps teams because of the lack of the right tools. Also, current Kubernetes solutions are mainly designed with an enterprise-first approach and they are too costly when deployed at scale.
+
+**Kamaji** aims to solve these pains by leveraging multi-tenancy and simplifying how to run multiple control planes on the same infrastructure with a fraction of the operational burden.
 
 ## How it works
-Kamaji turns any CNCF conformant Kubernetes cluster into an _“admin cluster”_ to orchestrate other Kubernetes clusters we're calling _“tenant clusters”_. As with every Kubernetes cluster, a tenant cluster has a set of nodes and a control plane, composed of several components: `APIs server`, `scheduler`, `controller manager`. What Kamaji does is deploy those components as a pod running in the admin cluster.
-
-
-And what about the tenant worker nodes? They are just worker nodes: regular instances, e.g. virtual or bare metal, connecting to the APIs server of the tenant cluster. Kamaji's goal is to manage the lifecycle of hundreds of these clusters, not only one, so how can we add another tenant cluster? As you could expect, Kamaji just deploys a new tenant control plane as a new pod in the admin cluster, and then it joins the new tenant worker nodes.
+Kamaji turns any Kubernetes cluster into an _“admin cluster”_ to orchestrate other Kubernetes clusters called _“tenant clusters”_. What makes Kamaji special is that Control Planes of _“tenant clusters”_ are just regular pods running in the _“admin cluster”_ instead of dedicated Virtual Machines. This solution makes running control planes at scale cheaper and easier to deploy and operate. View [Core Concepts](./docs/concepts.md) for a deeper understanding of principles behind Kamaji's design.
 
 <p align="center">
-  <img src="assets/kamaji-light.png" />
+  <img src="assets/kamaji-light.png#gh-light-mode-only" />
+</p>
+
+<p align="center">
+  <img src="assets/kamaji-dark.png#gh-dark-mode-only" />
 </p>
 
 All the tenant clusters built with Kamaji are fully compliant CNCF Kubernetes clusters and are compatible with the standard Kubernetes toolchains everybody knows and loves.
@@ -35,60 +36,48 @@ All the tenant clusters built with Kamaji are fully compliant CNCF Kubernetes cl
   <img src="assets/screenshot.png" />
 </p>
 
-## Save the state
-Putting the tenant cluster control plane in a pod is the easiest part. Also, we have to make sure each tenant cluster saves the state to be able to store and retrieve data.
+## Getting started
 
-A dedicated `etcd` cluster for each tenant cluster doesn’t scale well for a managed service because `etcd` data persistence can be cumbersome at scale, rising the operational effort to mitigate it. So we have to find an alternative keeping in mind our goal for a resilient and cost-optimized solution at the same time. As we can deploy any Kubernetes cluster with an external `etcd` cluster, we explored this option for the tenant control planes. On the admin cluster, we deploy a multi-tenant `etcd` cluster storing the state of multiple tenant clusters.
+Please refer to the [Getting Started guide](./docs/getting-started-with-kamaji.md) to deploy a minimal setup of Kamaji on KinD.
 
-With this solution, the resiliency is guaranteed by the usual `etcd` mechanism, and the pods' count remains under control, so it solves the main goal of resiliency and costs optimization. The trade-off here is that we have to operate an external `etcd` cluster and manage the access to be sure that each tenant cluster uses only its data. Also, there are limits in size in `etcd`, defaulted to 2GB and configurable to a maximum of 8GB. We’re solving this issue by pooling multiple `etcd` and sharding the tenant control planes.
+> This project is still in the early development stage which means it's not ready for production as APIs, commands, flags, etc. are subject to change, but also that your feedback can still help to shape it. Please try it out and let us know what you like, dislike, what works, what doesn't, etc.
 
 ## Use cases
-Kamaji project has been initially started as a solution for actual and common problems such as minimizing the Total Cost of Ownership while running Kubernetes at scale. However, it can open a wider range of use cases. Here are a few:
+Kamaji project has been initially started as a solution for actual and common problems such as minimizing the Total Cost of Ownership while running Kubernetes at large scale. However, it can open a wider range of use cases.
 
-### Managed Kubernetes
-Enabling companies to provide Cloud Native Infrastructure with ease by introducing a strong separation of concerns between management and workloads. Centralize clusters management, monitoring, and observability by leaving developers to focus on the applications, increase productivity and reduce operational costs. 
+Here are a few:
 
-### Control Plane as a Service
-Provide Kubernetes control plane in a self-service fashion by running management and workloads on different infrastructures and cost centers with the option of Bring Your Own Device - BYOD.
-
-### Edge Computing
-Distribute Kubernetes workloads across edge computing locations without having to manage multiple clusters across various providers. Centralize management of hundreds of control planes while leaving workloads to run isolated on their own dedicated infrastructure.
-
-### Cluster Simulations
-Test a new Kubernetes API or experimental flag or a new tool without impacting production operations. Kamaji will let you simulate such things in a safe and controlled environment.
+- **Managed Kubernetes:** enable companies to provide Cloud Native Infrastructure with ease by introducing a strong separation of concerns between management and workloads. Centralize clusters management, monitoring, and observability by leaving developers to focus on applications, increase productivity and reduce operational costs.
+- **Kubernetes as a Service:** provide Kubernetes clusters in a self-service fashion by running management and workloads on different infrastructures with the option of Bring Your Own Device, BYOD.
+- **Control Plane as a Service:** provide multiple Kubernetes control planes running on top of a single Kubernetes cluster. Tenants who use namespaces based isolation often still need access to cluster wide resources like Cluster Roles, Admission Webhooks, or Custom Resource Definitions.
+- **Edge Computing:** distribute Kubernetes workloads across edge computing locations without having to manage multiple clusters across various providers. Centralize management of hundreds of control planes while leaving workloads to run isolated on their own dedicated infrastructure.
+- **Cluster Simulation:** check new Kubernetes API or experimental flag or a new tool without impacting production operations. Kamaji will let you simulate such things in a safe and controlled environment.
+- **Workloads Testing:** check the behaviour of your workloads on different and multiple versions of Kubernetes with ease by deploying multiple Control Planes in a single cluster.
 
 ## Features
 
-### Self Service Kubernetes
-Leave users the freedom to self-provision their Kubernetes clusters according to the assigned boundaries.
-
-### Multi-cluster Management
-Centrally manage multiple tenant clusters from a single admin cluster. Happy SREs. 
-
-### Cheaper Control Planes
-Place multiple tenant control planes on a single node, instead of having three nodes for a single control plane.
-
-### Stronger Multi-Tenancy
-Leave tenants to access the control plane with admin permissions while keeping the tenant isolated at the infrastructure level.
-
-### Kubernetes Inception
-Use Kubernetes to manage Kubernetes by re-using all the Kubernetes goodies you already know and love.
-
-### Full APIs compliant
-Tenant clusters are fully CNCF compliant built with upstream Kubernetes binaries. A client does not see differences between a Kamaji provisioned cluster and a dedicated cluster.
+- **Self Service Kubernetes:** leave users the freedom to self-provision their Kubernetes clusters according to the assigned boundaries.
+- **Multi-cluster Management:** centrally manage multiple tenant clusters from a single admin cluster. Happy SREs. 
+- **Cheaper Control Planes:** place multiple tenant control planes on a single node, instead of having three nodes for a single control plane.
+- **Stronger Multi-Tenancy:** leave tenants to access the control plane with admin permissions while keeping the tenant isolated at the infrastructure level.
+- **Kubernetes Inception:** use Kubernetes to manage Kubernetes by re-using all the Kubernetes goodies you already know and love.
+- **Full APIs compliant:** tenant clusters are fully CNCF compliant built with upstream Kubernetes binaries. A user does not see differences between a Kamaji provisioned cluster and a dedicated cluster.
 
 ## Roadmap
 
 - [ ] Benchmarking and stress-test
 - [x] Support for dynamic address allocation on native Load Balancer
 - [x] Zero Downtime Tenant Control Plane upgrade
-- [ ] `konnectivity` integration
+- [x] `konnectivity` integration
 - [ ] Provisioning of Tenant Control Plane through Cluster APIs
-- [ ] Prometheus metrics for monitoring and alerting
-- [ ] `kine` integration, i.e. use MySQL, SQLite, PostgreSQL as datastore
+- [ ] Terraform provider
+- [ ] Custom Prometheus metrics for monitoring and alerting
+- [x] `kine` integration for MySQL as datastore
+- [x] `kine` integration for PostgreSQL as datastore
 - [ ] Deeper `kubeadm` integration
-- [ ] `etcd` pooling
-- [ ] Tenant Control Planes sharding
+- [ ] Pooling of multiple `etcd` datastores
+- [ ] Autoscaling of Tenant Control Plane pods
+
 
 ## Documentation
 Please, check the project's [documentation](./docs/) for getting started with Kamaji.
@@ -96,27 +85,26 @@ Please, check the project's [documentation](./docs/) for getting started with Ka
 ## Contributions
 Kamaji is Open Source with Apache 2 license and any contribution is welcome.
 
-## FAQ
+## Community
+Join the [Kubernetes Slack Workspace](https://slack.k8s.io/) and the [`#kamaji`](https://kubernetes.slack.com/archives/C03GLTTMWNN) channel to meet end-users and contributors.
+
+## FAQs
 Q. What does Kamaji means?
 
 A. Kamaji is named as the character _Kamaji_ from the Japanese movie [_Spirited Away_](https://en.wikipedia.org/wiki/Spirited_Away).
 
 Q. Is Kamaji another Kubernetes distribution?
 
-A. No, Kamaji is a tool you can install on top of any CNCF conformant Kubernetes to provide hundreds of managed Tenant clusters as a service. We tested Kamaji on vanilla Kubernetes 1.23+, KinD, and MS Azure AKS. We expect it to work smoothly on other Kubernetes distributions. The tenant clusters made with Kamaji are conformant CNCF Kubernetes vanilla clusters built with `kubeadm`.
+A. No, Kamaji is a Kubernetes Operator you can install on top of any Kubernetes cluster to provide hundreds of managed Kubernetes clusters as a service. We tested Kamaji on vanilla Kubernetes 1.22+, KinD, and Azure AKS. We expect it to work smoothly on other Kubernetes distributions. The tenant clusters made with Kamaji are conformant CNCF Kubernetes clusters as we leverage on [`kubeadm`](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/).
 
-Q. Is it safe to run Kubernetes control plane components in a pod?
+Q. Is it safe to run Kubernetes control plane components in a pod instead of dedicated virtual machines?
 
-A. Yes, the tenant control plane components are packaged in the same way they are running in bare metal or virtual nodes. We leverage the `kubeadm` code to set up the control plane components as they were running on a server. The same unchanged images of upstream `APIs server`, `scheduler`, `controller manager` are used.
+A. Yes, the tenant control plane components are packaged in the same way they are running in bare metal or virtual nodes. We leverage the `kubeadm` code to set up the control plane components as they were running on their own server. The unchanged images of upstream `kube-apiserver`, `kube-scheduler`, and `kube-controller-manager` are used.
 
-Q. And what about multi-tenant `etcd`? I never heard of it.
+Q. You already provide a Kubernetes multi-tenancy solution with [Capsule](https://capsule.clastix.io). Why does Kamaji matter?
 
-A. Even if multi-tenant deployment for `etcd` is not a common practice, multi-tenancy, RBAC, and client authentication has been [supported](https://etcd.io/docs/v3.5/op-guide/authentication/) in `etcd` from a long time. 
+A. A multi-tenancy solution, like Capsule shares the Kubernetes control plane among all tenants keeping tenant namespaces isolated by policies. While the solution is the right choice by balancing between features and ease of usage, there are cases where a tenant user requires access to the control plane, for example, when a tenant requires to manage CRDs on his own. With Kamaji, you can provide cluster admin permissions to the tenant.
 
-Q. You already provide a Kubernetes multi-tenancy solution with [Capsule](capsule.clastix.io). Why does Kamaji matter?
+Q. Well you convinced me, how to get a try?
 
-A. Lighter Multi-Tenancy solutions, like Capsule shares the Kubernetes control plane among all tenants keeping tenant namespaces isolated by policies. While these solutions are the right choice by balancing between features and ease of usage, there are cases where a tenant user requires access to the control plane, for example, when a tenant requires to manage CRDs on his own. With Kamaji, you can provide admin permissions to the tenants.
-
-Q. So I need a costly cloud infrastructure to try Kamaji?
-
-A. No, it is possible to try Kamaji on your laptop with [KinD](./deploy/kind/README.md).
+A. It is possible to get started with Kamaji on a laptop with [KinD](./docs/getting-started-with-kamaji.md) installed.

api/v1alpha1/datastore_funcs.go

@@ -0,0 +1,39 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package v1alpha1
+
+import (
+	"context"
+	"fmt"
+
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+// GetContent is the resolver for the container of the Secret.
+// The bare content has priority over the external reference.
+func (in *ContentRef) GetContent(ctx context.Context, client client.Client) ([]byte, error) {
+	if content := in.Content; len(content) > 0 {
+		return content, nil
+	}
+
+	secretRef := in.SecretRef
+
+	if secretRef == nil {
+		return nil, fmt.Errorf("no bare content and no external Secret reference")
+	}
+
+	secret, namespacedName := &corev1.Secret{}, types.NamespacedName{Name: secretRef.Name, Namespace: secretRef.Namespace}
+	if err := client.Get(ctx, namespacedName, secret); err != nil {
+		return nil, err
+	}
+
+	v, ok := secret.Data[secretRef.KeyPath]
+	if !ok {
+		return nil, fmt.Errorf("secret %s does not have key %s", namespacedName.String(), secretRef.KeyPath)
+	}
+
+	return v, nil
+}

api/v1alpha1/datastore_types.go

@@ -0,0 +1,104 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package v1alpha1
+
+import (
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+type Driver string //+kubebuilder:validation:Enum=etcd;MySQL;PostgreSQL
+
+var (
+	EtcdDriver           Driver = "etcd"
+	KineMySQLDriver      Driver = "MySQL"
+	KinePostgreSQLDriver Driver = "PostgreSQL"
+)
+
+// DataStoreSpec defines the desired state of DataStore.
+type DataStoreSpec struct {
+	// The driver to use to connect to the shared datastore.
+	Driver Driver `json:"driver"`
+	// List of the endpoints to connect to the shared datastore.
+	// No need for protocol, just bare IP/FQDN and port.
+	Endpoints []string `json:"endpoints"` //+kubebuilder:validation:MinLength=1
+	// In case of authentication enabled for the given data store, specifies the username and password pair.
+	// This value is optional.
+	BasicAuth *BasicAuth `json:"basicAuth,omitempty"`
+	// Defines the TLS/SSL configuration required to connect to the data store in a secure way.
+	TLSConfig TLSConfig `json:"tlsConfig"`
+}
+
+// TLSConfig contains the information used to connect to the data store using a secured connection.
+type TLSConfig struct {
+	// Retrieve the Certificate Authority certificate and private key, such as bare content of the file, or a SecretReference.
+	// The key reference is required since etcd authentication is based on certificates, and Kamaji is responsible in creating this.
+	CertificateAuthority CertKeyPair `json:"certificateAuthority"`
+	// Specifies the SSL/TLS key and private key pair used to connect to the data store.
+	ClientCertificate ClientCertificate `json:"clientCertificate"`
+}
+
+type ClientCertificate struct {
+	Certificate ContentRef `json:"certificate"`
+	PrivateKey  ContentRef `json:"privateKey"`
+}
+
+type CertKeyPair struct {
+	Certificate ContentRef  `json:"certificate"`
+	PrivateKey  *ContentRef `json:"privateKey,omitempty"`
+}
+
+// BasicAuth contains the required information to perform the connection using user credentials to the data store.
+type BasicAuth struct {
+	Username ContentRef `json:"username"`
+	Password ContentRef `json:"password"`
+}
+
+type ContentRef struct {
+	// Bare content of the file, base64 encoded.
+	// It has precedence over the SecretReference value.
+	Content   []byte           `json:"content,omitempty"`
+	SecretRef *SecretReference `json:"secretReference,omitempty"`
+}
+
+type SecretReference struct {
+	corev1.SecretReference `json:",inline"`
+	// Name of the key for the given Secret reference where the content is stored.
+	// This value is mandatory.
+	KeyPath string `json:"keyPath"`
+}
+
+// DataStoreStatus defines the observed state of DataStore.
+type DataStoreStatus struct {
+	// List of the Tenant Control Planes, namespaced named, using this data store.
+	UsedBy []string `json:"usedBy,omitempty"`
+}
+
+//+kubebuilder:object:root=true
+//+kubebuilder:subresource:status
+//+kubebuilder:resource:scope=Cluster
+//+kubebuilder:printcolumn:name="Driver",type="string",JSONPath=".spec.driver",description="Kamaji data store driver"
+//+kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="Age"
+
+// DataStore is the Schema for the datastores API.
+type DataStore struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec   DataStoreSpec   `json:"spec,omitempty"`
+	Status DataStoreStatus `json:"status,omitempty"`
+}
+
+//+kubebuilder:object:root=true
+
+// DataStoreList contains a list of DataStore.
+type DataStoreList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []DataStore `json:"items"`
+}
+
+func init() {
+	SchemeBuilder.Register(&DataStore{}, &DataStoreList{})
+}

api/v1alpha1/groupversion_info.go

@@ -2,8 +2,9 @@
 // SPDX-License-Identifier: Apache-2.0
 
 // Package v1alpha1 contains API Schema definitions for the kamaji v1alpha1 API group
-//+kubebuilder:object:generate=true
-//+groupName=kamaji.clastix.io
+// +kubebuilder:object:generate=true
+// +groupName=kamaji.clastix.io
+//nolint
 package v1alpha1
 
 import (

api/v1alpha1/tenantcontrolplane_addons_funcs.go

@@ -0,0 +1,17 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package v1alpha1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func (in AddonStatus) GetChecksum() string {
+	return in.Checksum
+}
+
+func (in *AddonStatus) SetChecksum(checksum string) {
+	in.LastUpdate = metav1.Now()
+	in.Checksum = checksum
+}

api/v1alpha1/tenantcontrolplane_funcs.go

@@ -6,14 +6,41 @@ package v1alpha1
 import (
 	"context"
 	"fmt"
+	"net"
+	"strconv"
 
 	"github.com/pkg/errors"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	kamajierrors "github.com/clastix/kamaji/internal/errors"
 )
 
-func (in *TenantControlPlane) GetAddress(ctx context.Context, client client.Client) (string, error) {
+// AssignedControlPlaneAddress returns the announced address and port of a Tenant Control Plane.
+// In case of non-well formed values, or missing announcement, an error is returned.
+func (in *TenantControlPlane) AssignedControlPlaneAddress() (string, int32, error) {
+	if len(in.Status.ControlPlaneEndpoint) == 0 {
+		return "", 0, fmt.Errorf("the Tenant Control Plane is not yet exposed")
+	}
+
+	address, portString, err := net.SplitHostPort(in.Status.ControlPlaneEndpoint)
+	if err != nil {
+		return "", 0, errors.Wrap(err, "cannot split host port from Tenant Control Plane endpoint")
+	}
+
+	port, err := strconv.Atoi(portString)
+	if err != nil {
+		return "", 0, errors.Wrap(err, "cannot convert Tenant Control Plane port from endpoint")
+	}
+
+	return address, int32(port), nil
+}
+
+// DeclaredControlPlaneAddress returns the desired Tenant Control Plane address.
+// In case of dynamic allocation, e.g. using a Load Balancer, it queries the API Server looking for the allocated IP.
+// When an IP has not been yet assigned, or it is expected, an error is returned.
+func (in *TenantControlPlane) DeclaredControlPlaneAddress(ctx context.Context, client client.Client) (string, error) {
 	var loadBalancerStatus corev1.LoadBalancerStatus
 
 	svc := &corev1.Service{}
@@ -26,10 +53,12 @@ func (in *TenantControlPlane) GetAddress(ctx context.Context, client client.Clie
 	case len(in.Spec.NetworkProfile.Address) > 0:
 		// Returning the hard-coded value in the specification in case of non LoadBalanced resources
 		return in.Spec.NetworkProfile.Address, nil
+	case svc.Spec.Type == corev1.ServiceTypeClusterIP:
+		return svc.Spec.ClusterIP, nil
 	case svc.Spec.Type == corev1.ServiceTypeLoadBalancer:
 		loadBalancerStatus = svc.Status.LoadBalancer
 		if len(loadBalancerStatus.Ingress) == 0 {
-			return "", fmt.Errorf("cannot retrieve the TenantControlPlane address, Service resource is not yet exposed as LoadBalancer")
+			return "", kamajierrors.NonExposedLoadBalancerError{}
 		}
 
 		for _, lb := range loadBalancerStatus.Ingress {
@@ -39,5 +68,5 @@ func (in *TenantControlPlane) GetAddress(ctx context.Context, client client.Clie
 		}
 	}
 
-	return "", fmt.Errorf("the actual resource doesn't have yet a valid IP address")
+	return "", kamajierrors.MissingValidIPError{}
 }

api/v1alpha1/tenantcontrolplane_interfaces.go

@@ -0,0 +1,12 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package v1alpha1
+
+// KubeadmConfigChecksumDependant is the interface used to retrieve the checksum of the kubeadm phases and addons
+// configuration, required to validate the changes and, upon from that, perform the required reconciliation.
+// +kubebuilder:object:generate=false
+type KubeadmConfigChecksumDependant interface {
+	GetChecksum() string
+	SetChecksum(string)
+}

api/v1alpha1/tenantcontrolplane_kubeadmphase_funcs.go

@@ -0,0 +1,17 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package v1alpha1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func (in KubeadmPhaseStatus) GetChecksum() string {
+	return in.Checksum
+}
+
+func (in *KubeadmPhaseStatus) SetChecksum(checksum string) {
+	in.LastUpdate = metav1.Now()
+	in.Checksum = checksum
+}

api/v1alpha1/tenantcontrolplane_status.go

@@ -0,0 +1,236 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package v1alpha1
+
+import (
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	networkingv1 "k8s.io/api/networking/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// APIServerCertificatesStatus defines the observed state of ETCD Certificate for API server.
+type APIServerCertificatesStatus struct {
+	SecretName string      `json:"secretName,omitempty"`
+	LastUpdate metav1.Time `json:"lastUpdate,omitempty"`
+	Checksum   string      `json:"checksum,omitempty"`
+}
+
+// ETCDCertificateStatus defines the observed state of ETCD Certificate for API server.
+type ETCDCertificateStatus struct {
+	SecretName string      `json:"secretName,omitempty"`
+	LastUpdate metav1.Time `json:"lastUpdate,omitempty"`
+	Checksum   string      `json:"checksum,omitempty"`
+}
+
+// ETCDCertificatesStatus defines the observed state of ETCD Certificate for API server.
+type ETCDCertificatesStatus struct {
+	APIServer APIServerCertificatesStatus `json:"apiServer,omitempty"`
+	CA        ETCDCertificateStatus       `json:"ca,omitempty"`
+}
+
+// CertificatePrivateKeyPairStatus defines the status.
+type CertificatePrivateKeyPairStatus struct {
+	SecretName string      `json:"secretName,omitempty"`
+	LastUpdate metav1.Time `json:"lastUpdate,omitempty"`
+	Checksum   string      `json:"checksum,omitempty"`
+}
+
+// PublicKeyPrivateKeyPairStatus defines the status.
+type PublicKeyPrivateKeyPairStatus struct {
+	SecretName string      `json:"secretName,omitempty"`
+	LastUpdate metav1.Time `json:"lastUpdate,omitempty"`
+	Checksum   string      `json:"checksum,omitempty"`
+}
+
+// CertificatesStatus defines the observed state of ETCD TLSConfig.
+type CertificatesStatus struct {
+	CA                     CertificatePrivateKeyPairStatus `json:"ca,omitempty"`
+	APIServer              CertificatePrivateKeyPairStatus `json:"apiServer,omitempty"`
+	APIServerKubeletClient CertificatePrivateKeyPairStatus `json:"apiServerKubeletClient,omitempty"`
+	FrontProxyCA           CertificatePrivateKeyPairStatus `json:"frontProxyCA,omitempty"`
+	FrontProxyClient       CertificatePrivateKeyPairStatus `json:"frontProxyClient,omitempty"`
+	SA                     PublicKeyPrivateKeyPairStatus   `json:"sa,omitempty"`
+	ETCD                   *ETCDCertificatesStatus         `json:"etcd,omitempty"`
+}
+
+type DataStoreCertificateStatus struct {
+	SecretName string      `json:"secretName,omitempty"`
+	Checksum   string      `json:"checksum,omitempty"`
+	LastUpdate metav1.Time `json:"lastUpdate,omitempty"`
+}
+
+type DataStoreConfigStatus struct {
+	SecretName string `json:"secretName,omitempty"`
+	Checksum   string `json:"checksum,omitempty"`
+}
+
+type DataStoreSetupStatus struct {
+	Schema     string      `json:"schema,omitempty"`
+	User       string      `json:"user,omitempty"`
+	LastUpdate metav1.Time `json:"lastUpdate,omitempty"`
+	Checksum   string      `json:"checksum,omitempty"`
+}
+
+// StorageStatus defines the observed state of StorageStatus.
+type StorageStatus struct {
+	Driver      string                     `json:"driver,omitempty"`
+	Config      DataStoreConfigStatus      `json:"config,omitempty"`
+	Setup       DataStoreSetupStatus       `json:"setup,omitempty"`
+	Certificate DataStoreCertificateStatus `json:"certificate,omitempty"`
+}
+
+// KubeconfigStatus contains information about the generated kubeconfig.
+type KubeconfigStatus struct {
+	SecretName string      `json:"secretName,omitempty"`
+	LastUpdate metav1.Time `json:"lastUpdate,omitempty"`
+	Checksum   string      `json:"checksum,omitempty"`
+}
+
+// KubeconfigsStatus stores information about all the generated kubeconfig resources.
+type KubeconfigsStatus struct {
+	Admin             KubeconfigStatus `json:"admin,omitempty"`
+	ControllerManager KubeconfigStatus `json:"controllerManager,omitempty"`
+	Scheduler         KubeconfigStatus `json:"scheduler,omitempty"`
+}
+
+// KubeadmConfigStatus contains the status of the configuration required by kubeadm.
+type KubeadmConfigStatus struct {
+	ConfigmapName string      `json:"configmapName,omitempty"`
+	LastUpdate    metav1.Time `json:"lastUpdate,omitempty"`
+	// Checksum of the kubeadm configuration to detect changes
+	Checksum string `json:"checksum,omitempty"`
+}
+
+// KubeadmPhaseStatus contains the status of a kubeadm phase action.
+type KubeadmPhaseStatus struct {
+	Checksum   string      `json:"checksum,omitempty"`
+	LastUpdate metav1.Time `json:"lastUpdate,omitempty"`
+}
+
+// KubeadmPhasesStatus contains the status of the different kubeadm phases action.
+type KubeadmPhasesStatus struct {
+	UploadConfigKubeadm KubeadmPhaseStatus `json:"uploadConfigKubeadm"`
+	UploadConfigKubelet KubeadmPhaseStatus `json:"uploadConfigKubelet"`
+	BootstrapToken      KubeadmPhaseStatus `json:"bootstrapToken"`
+}
+
+type ExternalKubernetesObjectStatus struct {
+	Name      string `json:"name,omitempty"`
+	Namespace string `json:"namespace,omitempty"`
+	Checksum  string `json:"checksum,omitempty"`
+	// Last time when k8s object was updated
+	LastUpdate metav1.Time `json:"lastUpdate,omitempty"`
+}
+
+// KonnectivityStatus defines the status of Konnectivity as Addon.
+type KonnectivityStatus struct {
+	Enabled            bool                            `json:"enabled"`
+	ConfigMap          KonnectivityConfigMap           `json:"configMap,omitempty"`
+	Certificate        CertificatePrivateKeyPairStatus `json:"certificate,omitempty"`
+	Kubeconfig         KubeconfigStatus                `json:"kubeconfig,omitempty"`
+	ServiceAccount     ExternalKubernetesObjectStatus  `json:"sa,omitempty"`
+	ClusterRoleBinding ExternalKubernetesObjectStatus  `json:"clusterrolebinding,omitempty"`
+	Agent              ExternalKubernetesObjectStatus  `json:"agent,omitempty"`
+	Service            KubernetesServiceStatus         `json:"service,omitempty"`
+}
+
+type KonnectivityConfigMap struct {
+	Name     string `json:"name,omitempty"`
+	Checksum string `json:"checksum,omitempty"`
+}
+
+// AddonStatus defines the observed state of an Addon.
+type AddonStatus struct {
+	Enabled    bool        `json:"enabled"`
+	Checksum   string      `json:"checksum,omitempty"`
+	LastUpdate metav1.Time `json:"lastUpdate,omitempty"`
+}
+
+// AddonsStatus defines the observed state of the different Addons.
+type AddonsStatus struct {
+	CoreDNS      AddonStatus        `json:"coreDNS,omitempty"`
+	KubeProxy    AddonStatus        `json:"kubeProxy,omitempty"`
+	Konnectivity KonnectivityStatus `json:"konnectivity,omitempty"`
+}
+
+// TenantControlPlaneStatus defines the observed state of TenantControlPlane.
+type TenantControlPlaneStatus struct {
+	// Storage Status contains information about Kubernetes storage system
+	Storage StorageStatus `json:"storage,omitempty"`
+	// Certificates contains information about the different certificates
+	// that are necessary to run a kubernetes control plane
+	Certificates CertificatesStatus `json:"certificates,omitempty"`
+	// KubeConfig contains information about the kubenconfigs that control plane pieces need
+	KubeConfig KubeconfigsStatus `json:"kubeconfig,omitempty"`
+	// Kubernetes contains information about the reconciliation of the required Kubernetes resources deployed in the admin cluster
+	Kubernetes KubernetesStatus `json:"kubernetesResources,omitempty"`
+	// KubeadmConfig contains the status of the configuration required by kubeadm
+	KubeadmConfig KubeadmConfigStatus `json:"kubeadmconfig,omitempty"`
+	// KubeadmPhase contains the status of the kubeadm phases action
+	KubeadmPhase KubeadmPhasesStatus `json:"kubeadmPhase,omitempty"`
+	// ControlPlaneEndpoint contains the status of the kubernetes control plane
+	ControlPlaneEndpoint string `json:"controlPlaneEndpoint,omitempty"`
+	// Addons contains the status of the different Addons
+	Addons AddonsStatus `json:"addons,omitempty"`
+}
+
+// KubernetesStatus defines the status of the resources deployed in the management cluster,
+// such as Deployment and Service.
+type KubernetesStatus struct {
+	// KubernetesVersion contains the information regarding the running Kubernetes version, and its upgrade status.
+	Version    KubernetesVersion          `json:"version,omitempty"`
+	Deployment KubernetesDeploymentStatus `json:"deployment,omitempty"`
+	Service    KubernetesServiceStatus    `json:"service,omitempty"`
+	Ingress    *KubernetesIngressStatus   `json:"ingress,omitempty"`
+}
+
+// +kubebuilder:validation:Enum=Provisioning;Upgrading;Ready;NotReady
+type KubernetesVersionStatus string
+
+var (
+	VersionProvisioning KubernetesVersionStatus = "Provisioning"
+	VersionUpgrading    KubernetesVersionStatus = "Upgrading"
+	VersionReady        KubernetesVersionStatus = "Ready"
+	VersionNotReady     KubernetesVersionStatus = "NotReady"
+)
+
+type KubernetesVersion struct {
+	// Version is the running Kubernetes version of the Tenant Control Plane.
+	Version string `json:"version,omitempty"`
+	// +kubebuilder:default=Provisioning
+	// Status returns the current status of the Kubernetes version, such as its provisioning state, or completed upgrade.
+	Status *KubernetesVersionStatus `json:"status,omitempty"`
+}
+
+// KubernetesDeploymentStatus defines the status for the Tenant Control Plane Deployment in the management cluster.
+type KubernetesDeploymentStatus struct {
+	appsv1.DeploymentStatus `json:",inline"`
+	// The name of the Deployment for the given cluster.
+	Name string `json:"name"`
+	// The namespace which the Deployment for the given cluster is deployed.
+	Namespace string `json:"namespace"`
+	// Last time when deployment was updated
+	LastUpdate metav1.Time `json:"lastUpdate,omitempty"`
+}
+
+// KubernetesServiceStatus defines the status for the Tenant Control Plane Service in the management cluster.
+type KubernetesServiceStatus struct {
+	corev1.ServiceStatus `json:",inline"`
+	// The name of the Service for the given cluster.
+	Name string `json:"name"`
+	// The namespace which the Service for the given cluster is deployed.
+	Namespace string `json:"namespace"`
+	// The port where the service is running
+	Port int32 `json:"port"`
+}
+
+// KubernetesIngressStatus defines the status for the Tenant Control Plane Ingress in the management cluster.
+type KubernetesIngressStatus struct {
+	networkingv1.IngressStatus `json:",inline"`
+	// The name of the Ingress for the given cluster.
+	Name string `json:"name"`
+	// The namespace which the Ingress for the given cluster is deployed.
+	Namespace string `json:"namespace"`
+}

api/v1alpha1/tenantcontrolplane_types.go

@@ -4,12 +4,8 @@
 package v1alpha1
 
 import (
-	appv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
-	networkingv1 "k8s.io/api/networking/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-
-	"github.com/clastix/kamaji/internal/etcd"
 )
 
 // NetworkProfileSpec defines the desired state of NetworkProfile.
@@ -22,15 +18,18 @@ type NetworkProfileSpec struct {
 	AllowAddressAsExternalIP bool `json:"allowAddressAsExternalIP,omitempty"`
 	// Port where API server of will be exposed
 	// +kubebuilder:default=6443
-	Port int32 `json:"port"`
-
-	// Domain of the tenant control plane
-	Domain string `json:"domain"`
+	Port int32 `json:"port,omitempty"`
+	// CertSANs sets extra Subject Alternative Names (SANs) for the API Server signing certificate.
+	// Use this field to add additional hostnames when exposing the Tenant Control Plane with third solutions.
+	CertSANs []string `json:"certSANs,omitempty"`
 	// Kubernetes Service
-	ServiceCIDR string `json:"serviceCidr"`
+	// +kubebuilder:default="10.96.0.0/16"
+	ServiceCIDR string `json:"serviceCidr,omitempty"`
 	// CIDR for Kubernetes Pods
-	PodCIDR       string   `json:"podCidr"`
-	DNSServiceIPs []string `json:"dnsServiceIPs"`
+	// +kubebuilder:default="10.244.0.0/16"
+	PodCIDR string `json:"podCidr,omitempty"`
+	// +kubebuilder:default={"10.96.0.10"}
+	DNSServiceIPs []string `json:"dnsServiceIPs,omitempty"`
 }
 
 type KubeletSpec struct {
@@ -65,23 +64,43 @@ type ControlPlane struct {
 	// Defining the options for the Tenant Control Plane Service resource.
 	Service ServiceSpec `json:"service"`
 	// Defining the options for an Optional Ingress which will expose API Server of the Tenant Control Plane
-	Ingress IngressSpec `json:"ingress,omitempty"`
+	Ingress *IngressSpec `json:"ingress,omitempty"`
 }
 
 // IngressSpec defines the options for the ingress which will expose API Server of the Tenant Control Plane.
 type IngressSpec struct {
 	AdditionalMetadata AdditionalMetadata `json:"additionalMetadata,omitempty"`
-	Enabled            bool               `json:"enabled"`
 	IngressClassName   string             `json:"ingressClassName,omitempty"`
 	// Hostname is an optional field which will be used as Ingress's Host. If it is not defined,
 	// Ingress's host will be "<tenant>.<namespace>.<domain>", where domain is specified under NetworkProfileSpec
 	Hostname string `json:"hostname,omitempty"`
 }
 
+type ControlPlaneComponentsResources struct {
+	APIServer         *corev1.ResourceRequirements `json:"apiServer,omitempty"`
+	ControllerManager *corev1.ResourceRequirements `json:"controllerManager,omitempty"`
+	Scheduler         *corev1.ResourceRequirements `json:"scheduler,omitempty"`
+}
+
 type DeploymentSpec struct {
 	// +kubebuilder:default=2
-	Replicas           int32              `json:"replicas,omitempty"`
-	AdditionalMetadata AdditionalMetadata `json:"additionalMetadata,omitempty"`
+	Replicas int32 `json:"replicas,omitempty"`
+	// Resources defines the amount of memory and CPU to allocate to each component of the Control Plane
+	// (kube-apiserver, controller-manager, and scheduler).
+	Resources *ControlPlaneComponentsResources `json:"resources,omitempty"`
+	// ExtraArgs allows adding additional arguments to the Control Plane components,
+	// such as kube-apiserver, controller-manager, and scheduler.
+	ExtraArgs          *ControlPlaneExtraArgs `json:"extraArgs,omitempty"`
+	AdditionalMetadata AdditionalMetadata     `json:"additionalMetadata,omitempty"`
+}
+
+// ControlPlaneExtraArgs allows specifying additional arguments to the Control Plane components.
+type ControlPlaneExtraArgs struct {
+	APIServer         []string `json:"apiServer,omitempty"`
+	ControllerManager []string `json:"controllerManager,omitempty"`
+	Scheduler         []string `json:"scheduler,omitempty"`
+	// Available only if Kamaji is running using Kine as backing storage.
+	Kine []string `json:"kine,omitempty"`
 }
 
 type ServiceSpec struct {
@@ -90,178 +109,42 @@ type ServiceSpec struct {
 	ServiceType ServiceType `json:"serviceType"`
 }
 
+// AddonSpec defines the spec for every addon.
+type AddonSpec struct{}
+
+// KonnectivitySpec defines the spec for Konnectivity.
+type KonnectivitySpec struct {
+	// Port of Konnectivity proxy server.
+	ProxyPort int32 `json:"proxyPort"`
+	// Version for Konnectivity server and agent.
+	// +kubebuilder:default=v0.0.31
+	Version string `json:"version,omitempty"`
+	// ServerImage defines the container image for Konnectivity's server.
+	// +kubebuilder:default=us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-server
+	ServerImage string `json:"serverImage,omitempty"`
+	// AgentImage defines the container image for Konnectivity's agent.
+	// +kubebuilder:default=us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-agent
+	AgentImage string `json:"agentImage,omitempty"`
+	// Resources define the amount of CPU and memory to allocate to the Konnectivity server.
+	Resources *corev1.ResourceRequirements `json:"resources,omitempty"`
+}
+
+// AddonsSpec defines the enabled addons and their features.
+type AddonsSpec struct {
+	CoreDNS      *AddonSpec        `json:"coreDNS,omitempty"`
+	Konnectivity *KonnectivitySpec `json:"konnectivity,omitempty"`
+	KubeProxy    *AddonSpec        `json:"kubeProxy,omitempty"`
+}
+
 // TenantControlPlaneSpec defines the desired state of TenantControlPlane.
 type TenantControlPlaneSpec struct {
 	ControlPlane ControlPlane `json:"controlPlane"`
-
 	// Kubernetes specification for tenant control plane
 	Kubernetes KubernetesSpec `json:"kubernetes"`
-
 	// NetworkProfile specifies how the network is
 	NetworkProfile NetworkProfileSpec `json:"networkProfile,omitempty"`
-}
-
-// ETCDAPIServerCertificate defines the observed state of ETCD Certificate for API server.
-type APIServerCertificatesStatus struct {
-	SecretName string      `json:"secretName,omitempty"`
-	LastUpdate metav1.Time `json:"lastUpdate,omitempty"`
-}
-
-// ETCDAPIServerCertificate defines the observed state of ETCD Certificate for API server.
-type ETCDCertificateStatus struct {
-	SecretName string      `json:"secretName,omitempty"`
-	LastUpdate metav1.Time `json:"lastUpdate,omitempty"`
-}
-
-// ETCDAPIServerCertificate defines the observed state of ETCD Certificate for API server.
-type ETCDCertificatesStatus struct {
-	APIServer ETCDCertificateStatus `json:"apiServer,omitempty"`
-	CA        ETCDCertificateStatus `json:"ca,omitempty"`
-}
-
-// CertificatePrivateKeyPair defines the status.
-type CertificatePrivateKeyPairStatus struct {
-	SecretName string      `json:"secretName,omitempty"`
-	LastUpdate metav1.Time `json:"lastUpdate,omitempty"`
-}
-
-// CertificatePrivateKeyPair defines the status.
-type PublicKeyPrivateKeyPairStatus struct {
-	SecretName string      `json:"secretName,omitempty"`
-	LastUpdate metav1.Time `json:"lastUpdate,omitempty"`
-}
-
-// ETCDCertificates defines the observed state of ETCD Certificates.
-type CertificatesStatus struct {
-	CA                     CertificatePrivateKeyPairStatus `json:"ca,omitempty"`
-	APIServer              CertificatePrivateKeyPairStatus `json:"apiServer,omitempty"`
-	APIServerKubeletClient CertificatePrivateKeyPairStatus `json:"apiServerKubeletClient,omitempty"`
-	FrontProxyCA           CertificatePrivateKeyPairStatus `json:"frontProxyCA,omitempty"`
-	FrontProxyClient       CertificatePrivateKeyPairStatus `json:"frontProxyClient,omitempty"`
-	SA                     PublicKeyPrivateKeyPairStatus   `json:"sa,omitempty"`
-	ETCD                   *ETCDCertificatesStatus         `json:"etcd,omitempty"`
-}
-
-// ETCDStatus defines the observed state of ETCDStatus.
-type ETCDStatus struct {
-	Role etcd.Role `json:"role,omitempty"`
-	User etcd.User `json:"user,omitempty"`
-}
-
-// StorageStatus defines the observed state of StorageStatus.
-type StorageStatus struct {
-	ETCD *ETCDStatus `json:"etcd,omitempty"`
-}
-
-// TenantControlPlaneKubeconfigsStatus contains information about a the generated kubeconfig.
-type KubeconfigStatus struct {
-	SecretName string      `json:"secretName,omitempty"`
-	LastUpdate metav1.Time `json:"lastUpdate,omitempty"`
-}
-
-// TenantControlPlaneKubeconfigsStatus stores information about all the generated kubeconfigs.
-type KubeconfigsStatus struct {
-	Admin             KubeconfigStatus `json:"admin,omitempty"`
-	ControllerManager KubeconfigStatus `json:"controlerManager,omitempty"`
-	Scheduler         KubeconfigStatus `json:"scheduler,omitempty"`
-}
-
-// KubeadmConfigStatus contains the status of the configuration required by kubeadm.
-type KubeadmConfigStatus struct {
-	ConfigmapName   string      `json:"configmapName,omitempty"`
-	LastUpdate      metav1.Time `json:"lastUpdate,omitempty"`
-	ResourceVersion string      `json:"resourceVersion"`
-}
-
-// KubeadmPhasesStatus contains the status of of a kubeadm phase action.
-type KubeadmPhaseStatus struct {
-	KubeadmConfigResourceVersion string      `json:"kubeadmConfigResourceVersion,omitempty"`
-	LastUpdate                   metav1.Time `json:"lastUpdate,omitempty"`
-}
-
-// KubeadmPhasesStatus contains the status of the different kubeadm phases action.
-type KubeadmPhasesStatus struct {
-	UploadConfigKubeadm KubeadmPhaseStatus `json:"uploadConfigKubeadm"`
-	UploadConfigKubelet KubeadmPhaseStatus `json:"uploadConfigKubelet"`
-	AddonCoreDNS        KubeadmPhaseStatus `json:"addonCoreDNS"`
-	AddonKubeProxy      KubeadmPhaseStatus `json:"addonKubeProxy"`
-	BootstrapToken      KubeadmPhaseStatus `json:"bootstrapToken"`
-}
-
-// TenantControlPlaneStatus defines the observed state of TenantControlPlane.
-type TenantControlPlaneStatus struct {
-	// Storage Status contains information about Kubernetes storage system
-	Storage StorageStatus `json:"storage,omitempty"`
-	// Certificates contains information about the different certificates
-	// that are necessary to run a kubernetes control plane
-	Certificates CertificatesStatus `json:"certificates,omitempty"`
-	// KubeConfig contains information about the kubenconfigs that control plane pieces need
-	KubeConfig KubeconfigsStatus `json:"kubeconfig,omitempty"`
-	// Kubernetes contains information about the reconciliation of the required Kubernetes resources deployed in the admin cluster
-	Kubernetes KubernetesStatus `json:"kubernetesResources,omitempty"`
-	// KubeadmConfig contains the status of the configuration required by kubeadm
-	KubeadmConfig KubeadmConfigStatus `json:"kubeadmconfig,omitempty"`
-	// KubeadmPhase contains the status of the kubeadm phases action
-	KubeadmPhase KubeadmPhasesStatus `json:"kubeadmPhase,omitempty"`
-	// ControlPlaneEndpoint contains the status of the kubernetes control plane
-	ControlPlaneEndpoint string `json:"controlPlaneEndpoint,omitempty"`
-}
-
-// KubernetesStatus defines the status of the resources deployed in the management cluster,
-// such as Deployment and Service.
-type KubernetesStatus struct {
-	// KubernetesVersion contains the information regarding the running Kubernetes version, and its upgrade status.
-	Version    KubernetesVersion          `json:"version,omitempty"`
-	Deployment KubernetesDeploymentStatus `json:"deployment,omitempty"`
-	Service    KubernetesServiceStatus    `json:"service,omitempty"`
-	Ingress    KubernetesIngressStatus    `json:"ingress,omitempty"`
-}
-
-// +kubebuilder:validation:Enum=Provisioning;Upgrading;Ready;NotReady
-type KubernetesVersionStatus string
-
-var (
-	VersionProvisioning KubernetesVersionStatus = "Provisioning"
-	VersionUpgrading    KubernetesVersionStatus = "Upgrading"
-	VersionReady        KubernetesVersionStatus = "Ready"
-	VersionNotReady     KubernetesVersionStatus = "NotReady"
-)
-
-type KubernetesVersion struct {
-	// Version is the running Kubernetes version of the Tenant Control Plane.
-	Version string `json:"version,omitempty"`
-	// +kubebuilder:default=Provisioning
-	// Status returns the current status of the Kubernetes version, such as its provisioning state, or completed upgrade.
-	Status *KubernetesVersionStatus `json:"status"`
-}
-
-// KubernetesDeploymentStatus defines the status for the Tenant Control Plane Deployment in the management cluster.
-type KubernetesDeploymentStatus struct {
-	appv1.DeploymentStatus `json:",inline"`
-	// The name of the Deployment for the given cluster.
-	Name string `json:"name"`
-	// The namespace which the Deployment for the given cluster is deployed.
-	Namespace string `json:"namespace"`
-}
-
-// KubernetesServiceStatus defines the status for the Tenant Control Plane Service in the management cluster.
-type KubernetesServiceStatus struct {
-	corev1.ServiceStatus `json:",inline"`
-	// The name of the Service for the given cluster.
-	Name string `json:"name"`
-	// The namespace which the Service for the given cluster is deployed.
-	Namespace string `json:"namespace"`
-	// The port where the service is running
-	Port int32 `json:"port"`
-}
-
-// KubernetesIngressStatus defines the status for the Tenant Control Plane Ingress in the management cluster.
-type KubernetesIngressStatus struct {
-	networkingv1.IngressStatus `json:",inline"`
-	// The name of the Ingress for the given cluster.
-	Name string `json:"name"`
-	// The namespace which the Ingress for the given cluster is deployed.
-	Namespace string `json:"namespace"`
+	// Addons contain which addons are enabled
+	Addons AddonsSpec `json:"addons,omitempty"`
 }
 
 // +kubebuilder:object:root=true

api/v1alpha1/zz_generated.deepcopy.go

@@ -9,6 +9,7 @@
 package v1alpha1
 
 import (
+	"k8s.io/api/core/v1"
 	runtime "k8s.io/apimachinery/pkg/runtime"
 )
 
@@ -57,6 +58,85 @@ func (in *AdditionalMetadata) DeepCopy() *AdditionalMetadata {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *AddonSpec) DeepCopyInto(out *AddonSpec) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AddonSpec.
+func (in *AddonSpec) DeepCopy() *AddonSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(AddonSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *AddonStatus) DeepCopyInto(out *AddonStatus) {
+	*out = *in
+	in.LastUpdate.DeepCopyInto(&out.LastUpdate)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AddonStatus.
+func (in *AddonStatus) DeepCopy() *AddonStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(AddonStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *AddonsSpec) DeepCopyInto(out *AddonsSpec) {
+	*out = *in
+	if in.CoreDNS != nil {
+		in, out := &in.CoreDNS, &out.CoreDNS
+		*out = new(AddonSpec)
+		**out = **in
+	}
+	if in.Konnectivity != nil {
+		in, out := &in.Konnectivity, &out.Konnectivity
+		*out = new(KonnectivitySpec)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.KubeProxy != nil {
+		in, out := &in.KubeProxy, &out.KubeProxy
+		*out = new(AddonSpec)
+		**out = **in
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AddonsSpec.
+func (in *AddonsSpec) DeepCopy() *AddonsSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(AddonsSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *AddonsStatus) DeepCopyInto(out *AddonsStatus) {
+	*out = *in
+	in.CoreDNS.DeepCopyInto(&out.CoreDNS)
+	in.KubeProxy.DeepCopyInto(&out.KubeProxy)
+	in.Konnectivity.DeepCopyInto(&out.Konnectivity)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AddonsStatus.
+func (in *AddonsStatus) DeepCopy() *AddonsStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(AddonsStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in AdmissionControllers) DeepCopyInto(out *AdmissionControllers) {
 	{
@@ -76,6 +156,44 @@ func (in AdmissionControllers) DeepCopy() AdmissionControllers {
 	return *out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *BasicAuth) DeepCopyInto(out *BasicAuth) {
+	*out = *in
+	in.Username.DeepCopyInto(&out.Username)
+	in.Password.DeepCopyInto(&out.Password)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BasicAuth.
+func (in *BasicAuth) DeepCopy() *BasicAuth {
+	if in == nil {
+		return nil
+	}
+	out := new(BasicAuth)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *CertKeyPair) DeepCopyInto(out *CertKeyPair) {
+	*out = *in
+	in.Certificate.DeepCopyInto(&out.Certificate)
+	if in.PrivateKey != nil {
+		in, out := &in.PrivateKey, &out.PrivateKey
+		*out = new(ContentRef)
+		(*in).DeepCopyInto(*out)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CertKeyPair.
+func (in *CertKeyPair) DeepCopy() *CertKeyPair {
+	if in == nil {
+		return nil
+	}
+	out := new(CertKeyPair)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *CertificatePrivateKeyPairStatus) DeepCopyInto(out *CertificatePrivateKeyPairStatus) {
 	*out = *in
@@ -118,12 +236,58 @@ func (in *CertificatesStatus) DeepCopy() *CertificatesStatus {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ClientCertificate) DeepCopyInto(out *ClientCertificate) {
+	*out = *in
+	in.Certificate.DeepCopyInto(&out.Certificate)
+	in.PrivateKey.DeepCopyInto(&out.PrivateKey)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClientCertificate.
+func (in *ClientCertificate) DeepCopy() *ClientCertificate {
+	if in == nil {
+		return nil
+	}
+	out := new(ClientCertificate)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ContentRef) DeepCopyInto(out *ContentRef) {
+	*out = *in
+	if in.Content != nil {
+		in, out := &in.Content, &out.Content
+		*out = make([]byte, len(*in))
+		copy(*out, *in)
+	}
+	if in.SecretRef != nil {
+		in, out := &in.SecretRef, &out.SecretRef
+		*out = new(SecretReference)
+		**out = **in
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ContentRef.
+func (in *ContentRef) DeepCopy() *ContentRef {
+	if in == nil {
+		return nil
+	}
+	out := new(ContentRef)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ControlPlane) DeepCopyInto(out *ControlPlane) {
 	*out = *in
 	in.Deployment.DeepCopyInto(&out.Deployment)
 	in.Service.DeepCopyInto(&out.Service)
-	in.Ingress.DeepCopyInto(&out.Ingress)
+	if in.Ingress != nil {
+		in, out := &in.Ingress, &out.Ingress
+		*out = new(IngressSpec)
+		(*in).DeepCopyInto(*out)
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ControlPlane.
@@ -136,9 +300,236 @@ func (in *ControlPlane) DeepCopy() *ControlPlane {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ControlPlaneComponentsResources) DeepCopyInto(out *ControlPlaneComponentsResources) {
+	*out = *in
+	if in.APIServer != nil {
+		in, out := &in.APIServer, &out.APIServer
+		*out = new(v1.ResourceRequirements)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.ControllerManager != nil {
+		in, out := &in.ControllerManager, &out.ControllerManager
+		*out = new(v1.ResourceRequirements)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.Scheduler != nil {
+		in, out := &in.Scheduler, &out.Scheduler
+		*out = new(v1.ResourceRequirements)
+		(*in).DeepCopyInto(*out)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ControlPlaneComponentsResources.
+func (in *ControlPlaneComponentsResources) DeepCopy() *ControlPlaneComponentsResources {
+	if in == nil {
+		return nil
+	}
+	out := new(ControlPlaneComponentsResources)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ControlPlaneExtraArgs) DeepCopyInto(out *ControlPlaneExtraArgs) {
+	*out = *in
+	if in.APIServer != nil {
+		in, out := &in.APIServer, &out.APIServer
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
+	if in.ControllerManager != nil {
+		in, out := &in.ControllerManager, &out.ControllerManager
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
+	if in.Scheduler != nil {
+		in, out := &in.Scheduler, &out.Scheduler
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
+	if in.Kine != nil {
+		in, out := &in.Kine, &out.Kine
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ControlPlaneExtraArgs.
+func (in *ControlPlaneExtraArgs) DeepCopy() *ControlPlaneExtraArgs {
+	if in == nil {
+		return nil
+	}
+	out := new(ControlPlaneExtraArgs)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *DataStore) DeepCopyInto(out *DataStore) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
+	in.Spec.DeepCopyInto(&out.Spec)
+	in.Status.DeepCopyInto(&out.Status)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataStore.
+func (in *DataStore) DeepCopy() *DataStore {
+	if in == nil {
+		return nil
+	}
+	out := new(DataStore)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *DataStore) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *DataStoreCertificateStatus) DeepCopyInto(out *DataStoreCertificateStatus) {
+	*out = *in
+	in.LastUpdate.DeepCopyInto(&out.LastUpdate)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataStoreCertificateStatus.
+func (in *DataStoreCertificateStatus) DeepCopy() *DataStoreCertificateStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(DataStoreCertificateStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *DataStoreConfigStatus) DeepCopyInto(out *DataStoreConfigStatus) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataStoreConfigStatus.
+func (in *DataStoreConfigStatus) DeepCopy() *DataStoreConfigStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(DataStoreConfigStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *DataStoreList) DeepCopyInto(out *DataStoreList) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ListMeta.DeepCopyInto(&out.ListMeta)
+	if in.Items != nil {
+		in, out := &in.Items, &out.Items
+		*out = make([]DataStore, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataStoreList.
+func (in *DataStoreList) DeepCopy() *DataStoreList {
+	if in == nil {
+		return nil
+	}
+	out := new(DataStoreList)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *DataStoreList) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *DataStoreSetupStatus) DeepCopyInto(out *DataStoreSetupStatus) {
+	*out = *in
+	in.LastUpdate.DeepCopyInto(&out.LastUpdate)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataStoreSetupStatus.
+func (in *DataStoreSetupStatus) DeepCopy() *DataStoreSetupStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(DataStoreSetupStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *DataStoreSpec) DeepCopyInto(out *DataStoreSpec) {
+	*out = *in
+	if in.Endpoints != nil {
+		in, out := &in.Endpoints, &out.Endpoints
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
+	if in.BasicAuth != nil {
+		in, out := &in.BasicAuth, &out.BasicAuth
+		*out = new(BasicAuth)
+		(*in).DeepCopyInto(*out)
+	}
+	in.TLSConfig.DeepCopyInto(&out.TLSConfig)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataStoreSpec.
+func (in *DataStoreSpec) DeepCopy() *DataStoreSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(DataStoreSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *DataStoreStatus) DeepCopyInto(out *DataStoreStatus) {
+	*out = *in
+	if in.UsedBy != nil {
+		in, out := &in.UsedBy, &out.UsedBy
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataStoreStatus.
+func (in *DataStoreStatus) DeepCopy() *DataStoreStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(DataStoreStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *DeploymentSpec) DeepCopyInto(out *DeploymentSpec) {
 	*out = *in
+	if in.Resources != nil {
+		in, out := &in.Resources, &out.Resources
+		*out = new(ControlPlaneComponentsResources)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.ExtraArgs != nil {
+		in, out := &in.ExtraArgs, &out.ExtraArgs
+		*out = new(ControlPlaneExtraArgs)
+		(*in).DeepCopyInto(*out)
+	}
 	in.AdditionalMetadata.DeepCopyInto(&out.AdditionalMetadata)
 }
 
@@ -186,18 +577,17 @@ func (in *ETCDCertificatesStatus) DeepCopy() *ETCDCertificatesStatus {
 }
 
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *ETCDStatus) DeepCopyInto(out *ETCDStatus) {
+func (in *ExternalKubernetesObjectStatus) DeepCopyInto(out *ExternalKubernetesObjectStatus) {
 	*out = *in
-	in.Role.DeepCopyInto(&out.Role)
-	in.User.DeepCopyInto(&out.User)
+	in.LastUpdate.DeepCopyInto(&out.LastUpdate)
 }
 
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ETCDStatus.
-func (in *ETCDStatus) DeepCopy() *ETCDStatus {
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExternalKubernetesObjectStatus.
+func (in *ExternalKubernetesObjectStatus) DeepCopy() *ExternalKubernetesObjectStatus {
 	if in == nil {
 		return nil
 	}
-	out := new(ETCDStatus)
+	out := new(ExternalKubernetesObjectStatus)
 	in.DeepCopyInto(out)
 	return out
 }
@@ -218,6 +608,63 @@ func (in *IngressSpec) DeepCopy() *IngressSpec {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *KonnectivityConfigMap) DeepCopyInto(out *KonnectivityConfigMap) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KonnectivityConfigMap.
+func (in *KonnectivityConfigMap) DeepCopy() *KonnectivityConfigMap {
+	if in == nil {
+		return nil
+	}
+	out := new(KonnectivityConfigMap)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *KonnectivitySpec) DeepCopyInto(out *KonnectivitySpec) {
+	*out = *in
+	if in.Resources != nil {
+		in, out := &in.Resources, &out.Resources
+		*out = new(v1.ResourceRequirements)
+		(*in).DeepCopyInto(*out)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KonnectivitySpec.
+func (in *KonnectivitySpec) DeepCopy() *KonnectivitySpec {
+	if in == nil {
+		return nil
+	}
+	out := new(KonnectivitySpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *KonnectivityStatus) DeepCopyInto(out *KonnectivityStatus) {
+	*out = *in
+	out.ConfigMap = in.ConfigMap
+	in.Certificate.DeepCopyInto(&out.Certificate)
+	in.Kubeconfig.DeepCopyInto(&out.Kubeconfig)
+	in.ServiceAccount.DeepCopyInto(&out.ServiceAccount)
+	in.ClusterRoleBinding.DeepCopyInto(&out.ClusterRoleBinding)
+	in.Agent.DeepCopyInto(&out.Agent)
+	in.Service.DeepCopyInto(&out.Service)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KonnectivityStatus.
+func (in *KonnectivityStatus) DeepCopy() *KonnectivityStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(KonnectivityStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *KubeadmConfigStatus) DeepCopyInto(out *KubeadmConfigStatus) {
 	*out = *in
@@ -255,8 +702,6 @@ func (in *KubeadmPhasesStatus) DeepCopyInto(out *KubeadmPhasesStatus) {
 	*out = *in
 	in.UploadConfigKubeadm.DeepCopyInto(&out.UploadConfigKubeadm)
 	in.UploadConfigKubelet.DeepCopyInto(&out.UploadConfigKubelet)
-	in.AddonCoreDNS.DeepCopyInto(&out.AddonCoreDNS)
-	in.AddonKubeProxy.DeepCopyInto(&out.AddonKubeProxy)
 	in.BootstrapToken.DeepCopyInto(&out.BootstrapToken)
 }
 
@@ -323,6 +768,7 @@ func (in *KubeletSpec) DeepCopy() *KubeletSpec {
 func (in *KubernetesDeploymentStatus) DeepCopyInto(out *KubernetesDeploymentStatus) {
 	*out = *in
 	in.DeploymentStatus.DeepCopyInto(&out.DeploymentStatus)
+	in.LastUpdate.DeepCopyInto(&out.LastUpdate)
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubernetesDeploymentStatus.
@@ -394,7 +840,11 @@ func (in *KubernetesStatus) DeepCopyInto(out *KubernetesStatus) {
 	in.Version.DeepCopyInto(&out.Version)
 	in.Deployment.DeepCopyInto(&out.Deployment)
 	in.Service.DeepCopyInto(&out.Service)
-	in.Ingress.DeepCopyInto(&out.Ingress)
+	if in.Ingress != nil {
+		in, out := &in.Ingress, &out.Ingress
+		*out = new(KubernetesIngressStatus)
+		(*in).DeepCopyInto(*out)
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubernetesStatus.
@@ -430,6 +880,11 @@ func (in *KubernetesVersion) DeepCopy() *KubernetesVersion {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *NetworkProfileSpec) DeepCopyInto(out *NetworkProfileSpec) {
 	*out = *in
+	if in.CertSANs != nil {
+		in, out := &in.CertSANs, &out.CertSANs
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
 	if in.DNSServiceIPs != nil {
 		in, out := &in.DNSServiceIPs, &out.DNSServiceIPs
 		*out = make([]string, len(*in))
@@ -463,6 +918,22 @@ func (in *PublicKeyPrivateKeyPairStatus) DeepCopy() *PublicKeyPrivateKeyPairStat
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *SecretReference) DeepCopyInto(out *SecretReference) {
+	*out = *in
+	out.SecretReference = in.SecretReference
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecretReference.
+func (in *SecretReference) DeepCopy() *SecretReference {
+	if in == nil {
+		return nil
+	}
+	out := new(SecretReference)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ServiceSpec) DeepCopyInto(out *ServiceSpec) {
 	*out = *in
@@ -482,11 +953,9 @@ func (in *ServiceSpec) DeepCopy() *ServiceSpec {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *StorageStatus) DeepCopyInto(out *StorageStatus) {
 	*out = *in
-	if in.ETCD != nil {
-		in, out := &in.ETCD, &out.ETCD
-		*out = new(ETCDStatus)
-		(*in).DeepCopyInto(*out)
-	}
+	out.Config = in.Config
+	in.Setup.DeepCopyInto(&out.Setup)
+	in.Certificate.DeepCopyInto(&out.Certificate)
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StorageStatus.
@@ -499,6 +968,23 @@ func (in *StorageStatus) DeepCopy() *StorageStatus {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *TLSConfig) DeepCopyInto(out *TLSConfig) {
+	*out = *in
+	in.CertificateAuthority.DeepCopyInto(&out.CertificateAuthority)
+	in.ClientCertificate.DeepCopyInto(&out.ClientCertificate)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TLSConfig.
+func (in *TLSConfig) DeepCopy() *TLSConfig {
+	if in == nil {
+		return nil
+	}
+	out := new(TLSConfig)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *TenantControlPlane) DeepCopyInto(out *TenantControlPlane) {
 	*out = *in
@@ -564,6 +1050,7 @@ func (in *TenantControlPlaneSpec) DeepCopyInto(out *TenantControlPlaneSpec) {
 	in.ControlPlane.DeepCopyInto(&out.ControlPlane)
 	in.Kubernetes.DeepCopyInto(&out.Kubernetes)
 	in.NetworkProfile.DeepCopyInto(&out.NetworkProfile)
+	in.Addons.DeepCopyInto(&out.Addons)
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TenantControlPlaneSpec.
@@ -585,6 +1072,7 @@ func (in *TenantControlPlaneStatus) DeepCopyInto(out *TenantControlPlaneStatus)
 	in.Kubernetes.DeepCopyInto(&out.Kubernetes)
 	in.KubeadmConfig.DeepCopyInto(&out.KubeadmConfig)
 	in.KubeadmPhase.DeepCopyInto(&out.KubeadmPhase)
+	in.Addons.DeepCopyInto(&out.Addons)
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TenantControlPlaneStatus.

charts/kamaji/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v2
 name: kamaji
-description: A Kubernetes distribution aimed to build and operate a Managed Kubernetes service with a fraction of operational burde.
+description: Kamaji is a tool aimed to build and operate a Managed Kubernetes Service with a fraction of the operational burden. With Kamaji, you can deploy and operate hundreds of Kubernetes clusters as a hyper-scaler.
 
 # A chart can be either an 'application' or a 'library' chart.
 #
@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.0
+version: 0.3.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
@@ -23,8 +23,8 @@ version: 0.1.0
 # It is recommended to use it with quotes.
 appVersion: 0.1.0
 
-home: https://github.com/clastix/kamaji-internal/tree/master/helm/kamaji
-sources: ["https://github.com/clastix/kamaji-internal"]
+home: https://github.com/clastix/kamaji
+sources: ["https://github.com/clastix/kamaji"]
 kubeVersion: ">=1.18"
 maintainers:
 - email: iam@mendrugory.com
@@ -33,3 +33,5 @@ maintainers:
   name: Dario Tranchitella
 - email: me@maxgio.it
   name: Massimiliano Giovagnoli
+- email: me@bsctl.io
+  name: Adriano Pezzuto

charts/kamaji/README.md

@@ -0,0 +1,141 @@
+# kamaji
+
+![Version: 0.1.0](https://img.shields.io/badge/Version-0.1.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.1.0](https://img.shields.io/badge/AppVersion-0.1.0-informational?style=flat-square)
+
+Kamaji is a tool aimed to build and operate a Managed Kubernetes Service with a fraction of the operational burden. With Kamaji, you can deploy and operate hundreds of Kubernetes clusters as a hyper-scaler.
+
+## Maintainers
+
+| Name | Email | Url |
+| ---- | ------ | --- |
+| Gonzalo Gabriel Jiménez Fuentes | <iam@mendrugory.com> |  |
+| Dario Tranchitella | <dario@tranchitella.eu> |  |
+| Massimiliano Giovagnoli | <me@maxgio.it> |  |
+| Adriano Pezzuto | <me@bsctl.io> |  |
+
+## Source Code
+
+* <https://github.com/clastix/kamaji>
+
+## Requirements
+
+Kubernetes: `>=1.18`
+
+[Kamaji](https://github.com/clastix/kamaji) requires a [multi-tenant `etcd`](https://github.com/clastix/kamaji-internal/blob/master/deploy/getting-started-with-kamaji.md#setup-internal-multi-tenant-etcd) cluster.
+This Helm Chart starting from v0.1.1 provides the installation of an internal `etcd` in order to streamline the local test. If you'd like to use an externally managed etcd instance, you can specify the overrides and by setting the value `etcd.deploy=false`.
+
+> For production use an externally managed `etcd` is highly recommended, the `etcd` addon offered by this Chart is not considered production-grade.
+
+## Install Kamaji
+
+To install the Chart with the release name `kamaji`:
+
+        helm upgrade --install --namespace kamaji-system --create-namespace clastix/kamaji
+
+Show the status:
+
+        helm status kamaji -n kamaji-system
+
+Upgrade the Chart
+
+        helm upgrade kamaji -n kamaji-system clastix/kamaji
+
+Uninstall the Chart
+
+        helm uninstall kamaji -n kamaji-system
+
+## Customize the installation
+
+There are two methods for specifying overrides of values during Chart installation: `--values` and `--set`.
+
+The `--values` option is the preferred method because it allows you to keep your overrides in a YAML file, rather than specifying them all on the command line. Create a copy of the YAML file `values.yaml` and add your overrides to it.
+
+Specify your overrides file when you install the Chart:
+
+        helm upgrade kamaji --install --namespace kamaji-system --create-namespace clastix/kamaji --values myvalues.yaml
+
+The values in your overrides file `myvalues.yaml` will override their counterparts in the Chart's values.yaml file. Any values in `values.yaml` that weren’t overridden will keep their defaults.
+
+If you only need to make minor customizations, you can specify them on the command line by using the `--set` option. For example:
+
+        helm upgrade kamaji --install --namespace kamaji-system --create-namespace clastix/kamaji --set etcd.deploy=false
+
+Here the values you can override:
+
+## Values
+
+| Key | Type | Default | Description |
+|-----|------|---------|-------------|
+| affinity | object | `{}` | Kubernetes affinity rules to apply to Kamaji controller pods |
+| configPath | string | `"./kamaji.yaml"` | Configuration file path alternative. (default "./kamaji.yaml") |
+| datastore.basicAuth.passwordSecret.keyPath | string | `nil` | The Secret key where the data is stored. |
+| datastore.basicAuth.passwordSecret.name | string | `nil` | The name of the Secret containing the password used to connect to the relational database. |
+| datastore.basicAuth.passwordSecret.namespace | string | `nil` | The namespace of the Secret containing the password used to connect to the relational database. |
+| datastore.basicAuth.usernameSecret.keyPath | string | `nil` | The Secret key where the data is stored. |
+| datastore.basicAuth.usernameSecret.name | string | `nil` | The name of the Secret containing the username used to connect to the relational database. |
+| datastore.basicAuth.usernameSecret.namespace | string | `nil` | The namespace of the Secret containing the username used to connect to the relational database. |
+| datastore.driver | string | `"etcd"` | (string) The Kamaji Datastore driver, supported: etcd, MySQL, PostgreSQL (defaults=etcd). |
+| datastore.endpoints | list | `[]` | (array) List of endpoints of the selected Datastore. When letting the Chart install the etcd datastore, this field is populated automatically. |
+| datastore.nameOverride | string | `nil` | The Datastore name override, if empty defaults to `default` |
+| datastore.tlsConfig.certificateAuthority.certificate.keyPath | string | `nil` | Key of the Secret which contains the content of the certificate. |
+| datastore.tlsConfig.certificateAuthority.certificate.name | string | `nil` | Name of the Secret containing the CA required to establish the mandatory SSL/TLS connection to the datastore. |
+| datastore.tlsConfig.certificateAuthority.certificate.namespace | string | `nil` | Namespace of the Secret containing the CA required to establish the mandatory SSL/TLS connection to the datastore. |
+| datastore.tlsConfig.certificateAuthority.privateKey.keyPath | string | `nil` | Key of the Secret which contains the content of the private key. |
+| datastore.tlsConfig.certificateAuthority.privateKey.name | string | `nil` | Name of the Secret containing the CA private key required to establish the mandatory SSL/TLS connection to the datastore. |
+| datastore.tlsConfig.certificateAuthority.privateKey.namespace | string | `nil` | Namespace of the Secret containing the CA private key required to establish the mandatory SSL/TLS connection to the datastore. |
+| datastore.tlsConfig.clientCertificate.certificate.keyPath | string | `nil` | Key of the Secret which contains the content of the certificate. |
+| datastore.tlsConfig.clientCertificate.certificate.name | string | `nil` | Name of the Secret containing the client certificate required to establish the mandatory SSL/TLS connection to the datastore. |
+| datastore.tlsConfig.clientCertificate.certificate.namespace | string | `nil` | Namespace of the Secret containing the client certificate required to establish the mandatory SSL/TLS connection to the datastore. |
+| datastore.tlsConfig.clientCertificate.privateKey.keyPath | string | `nil` | Key of the Secret which contains the content of the private key. |
+| datastore.tlsConfig.clientCertificate.privateKey.name | string | `nil` | Name of the Secret containing the client certificate private key required to establish the mandatory SSL/TLS connection to the datastore. |
+| datastore.tlsConfig.clientCertificate.privateKey.namespace | string | `nil` | Namespace of the Secret containing the client certificate private key required to establish the mandatory SSL/TLS connection to the datastore. |
+| etcd.compactionInterval | int | `0` | ETCD Compaction interval (e.g. "5m0s"). (default: "0" (disabled)) |
+| etcd.deploy | bool | `true` | Install an etcd with enabled multi-tenancy along with Kamaji |
+| etcd.image | object | `{"pullPolicy":"IfNotPresent","repository":"quay.io/coreos/etcd","tag":"v3.5.4"}` | Install specific etcd image |
+| etcd.livenessProbe | object | `{"failureThreshold":8,"httpGet":{"path":"/health?serializable=true","port":2381,"scheme":"HTTP"},"initialDelaySeconds":10,"periodSeconds":10,"timeoutSeconds":15}` | The livenessProbe for the etcd container |
+| etcd.overrides.caSecret.name | string | `"etcd-certs"` | Name of the secret which contains CA's certificate and private key. (default: "etcd-certs") |
+| etcd.overrides.caSecret.namespace | string | `"kamaji-system"` | Namespace of the secret which contains CA's certificate and private key. (default: "kamaji-system") |
+| etcd.overrides.clientSecret.name | string | `"root-client-certs"` | Name of the secret which contains ETCD client certificates. (default: "root-client-certs") |
+| etcd.overrides.clientSecret.namespace | string | `"kamaji-system"` | Name of the namespace where the secret which contains ETCD client certificates is. (default: "kamaji-system") |
+| etcd.overrides.endpoints | object | `{"etcd-0":"https://etcd-0.etcd.kamaji-system.svc.cluster.local","etcd-1":"https://etcd-1.etcd.kamaji-system.svc.cluster.local","etcd-2":"https://etcd-2.etcd.kamaji-system.svc.cluster.local"}` | (map) Dictionary of the endpoints for the etcd cluster's members, key is the name of the etcd server. Don't define any port, inflected from .etcd.peerApiPort value. |
+| etcd.peerApiPort | int | `2380` | The peer API port which servers are listening to. |
+| etcd.persistence.accessModes[0] | string | `"ReadWriteOnce"` |  |
+| etcd.persistence.size | string | `"10Gi"` |  |
+| etcd.persistence.storageClass | string | `""` |  |
+| etcd.port | int | `2379` | The client request port. |
+| etcd.serviceAccount.create | bool | `true` | Create a ServiceAccount, required to install and provision the etcd backing storage (default: true) |
+| etcd.serviceAccount.name | string | `""` | Define the ServiceAccount name to use during the setup and provision of the etcd backing storage (default: "") |
+| extraArgs | list | `[]` | A list of extra arguments to add to the kamaji controller default ones |
+| fullnameOverride | string | `""` |  |
+| healthProbeBindAddress | string | `":8081"` | The address the probe endpoint binds to. (default ":8081") |
+| image.pullPolicy | string | `"Always"` |  |
+| image.repository | string | `"clastix/kamaji"` | The container image of the Kamaji controller. |
+| image.tag | string | `"latest"` |  |
+| imagePullSecrets | list | `[]` |  |
+| livenessProbe | object | `{"httpGet":{"path":"/healthz","port":"healthcheck"},"initialDelaySeconds":15,"periodSeconds":20}` | The livenessProbe for the controller container |
+| loggingDevel.enable | bool | `false` | (string) Development Mode defaults(encoder=consoleEncoder,logLevel=Debug,stackTraceLevel=Warn). Production Mode defaults(encoder=jsonEncoder,logLevel=Info,stackTraceLevel=Error) (default false) |
+| metricsBindAddress | string | `":8080"` | (string) The address the metric endpoint binds to. (default ":8080") |
+| nameOverride | string | `""` |  |
+| nodeSelector | object | `{}` | Kubernetes node selector rules to schedule Kamaji controller |
+| podAnnotations | object | `{}` | The annotations to apply to the Kamaji controller pods. |
+| podSecurityContext | object | `{"runAsNonRoot":true}` | The securityContext to apply to the Kamaji controller pods. |
+| readinessProbe | object | `{"httpGet":{"path":"/readyz","port":"healthcheck"},"initialDelaySeconds":5,"periodSeconds":10}` | The readinessProbe for the controller container |
+| replicaCount | int | `1` | The number of the pod replicas for the Kamaji controller. |
+| resources.limits.cpu | string | `"200m"` |  |
+| resources.limits.memory | string | `"100Mi"` |  |
+| resources.requests.cpu | string | `"100m"` |  |
+| resources.requests.memory | string | `"20Mi"` |  |
+| securityContext | object | `{"allowPrivilegeEscalation":false}` | The securityContext to apply to the Kamaji controller container only. It does not apply to the Kamaji RBAC proxy container. |
+| service.port | int | `8443` |  |
+| service.type | string | `"ClusterIP"` |  |
+| serviceAccount.annotations | object | `{}` |  |
+| serviceAccount.create | bool | `true` |  |
+| serviceAccount.name | string | `"kamaji-controller-manager"` |  |
+| temporaryDirectoryPath | string | `"/tmp/kamaji"` | Directory which will be used to work with temporary files. (default "/tmp/kamaji") |
+| tolerations | list | `[]` | Kubernetes node taints that the Kamaji controller pods would tolerate |
+
+## Installing and managing etcd as DataStore
+
+Kamaji supports multiple data store, although `etcd` is the default one: thus, an initial cluster will be created upon the Chart installation.
+
+The `DataStore` resource can be configured with the proper values in case of overrides when using a different driver, otherwise all the required data will be inherited by the Chart values.

charts/kamaji/README.md.gotmpl

@@ -0,0 +1,62 @@
+{{ template "chart.header" . }}
+{{ template "chart.deprecationWarning" . }}
+
+{{ template "chart.badgesSection" . }}
+
+{{ template "chart.description" . }}
+
+{{ template "chart.maintainersSection" . }}
+
+{{ template "chart.sourcesSection" . }}
+
+{{ template "chart.requirementsSection" . }}
+
+[Kamaji](https://github.com/clastix/kamaji) requires a [multi-tenant `etcd`](https://github.com/clastix/kamaji-internal/blob/master/deploy/getting-started-with-kamaji.md#setup-internal-multi-tenant-etcd) cluster.
+This Helm Chart starting from v0.1.1 provides the installation of an internal `etcd` in order to streamline the local test. If you'd like to use an externally managed etcd instance, you can specify the overrides and by setting the value `etcd.deploy=false`.
+
+> For production use an externally managed `etcd` is highly recommended, the `etcd` addon offered by this Chart is not considered production-grade.
+
+## Install Kamaji
+
+To install the Chart with the release name `kamaji`:
+
+
+        helm upgrade --install --namespace kamaji-system --create-namespace clastix/kamaji
+
+Show the status:
+
+        helm status kamaji -n kamaji-system
+
+Upgrade the Chart
+
+        helm upgrade kamaji -n kamaji-system clastix/kamaji
+
+Uninstall the Chart
+
+        helm uninstall kamaji -n kamaji-system
+
+## Customize the installation
+
+There are two methods for specifying overrides of values during Chart installation: `--values` and `--set`.
+
+The `--values` option is the preferred method because it allows you to keep your overrides in a YAML file, rather than specifying them all on the command line. Create a copy of the YAML file `values.yaml` and add your overrides to it.
+
+Specify your overrides file when you install the Chart:
+
+        helm upgrade kamaji --install --namespace kamaji-system --create-namespace clastix/kamaji --values myvalues.yaml
+
+The values in your overrides file `myvalues.yaml` will override their counterparts in the Chart's values.yaml file. Any values in `values.yaml` that weren’t overridden will keep their defaults.
+
+If you only need to make minor customizations, you can specify them on the command line by using the `--set` option. For example:
+
+        helm upgrade kamaji --install --namespace kamaji-system --create-namespace clastix/kamaji --set etcd.deploy=false
+
+Here the values you can override:
+
+{{ template "chart.valuesSection" . }}
+
+## Installing and managing etcd as DataStore
+
+Kamaji supports multiple data store, although `etcd` is the default one: thus, an initial cluster will be created upon the Chart installation.
+
+The `DataStore` resource can be configured with the proper values in case of overrides when using a different driver, otherwise all the required data will be inherited by the Chart values.

charts/kamaji/crds/datastore.yaml

@@ -0,0 +1,269 @@
+
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.6.1
+  creationTimestamp: null
+  name: datastores.kamaji.clastix.io
+spec:
+  group: kamaji.clastix.io
+  names:
+    kind: DataStore
+    listKind: DataStoreList
+    plural: datastores
+    singular: datastore
+  scope: Cluster
+  versions:
+  - additionalPrinterColumns:
+    - description: Kamaji data store driver
+      jsonPath: .spec.driver
+      name: Driver
+      type: string
+    - description: Age
+      jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1alpha1
+    schema:
+      openAPIV3Schema:
+        description: DataStore is the Schema for the datastores API.
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: DataStoreSpec defines the desired state of DataStore.
+            properties:
+              basicAuth:
+                description: In case of authentication enabled for the given data
+                  store, specifies the username and password pair. This value is optional.
+                properties:
+                  password:
+                    properties:
+                      content:
+                        description: Bare content of the file, base64 encoded. It
+                          has precedence over the SecretReference value.
+                        format: byte
+                        type: string
+                      secretReference:
+                        properties:
+                          keyPath:
+                            description: Name of the key for the given Secret reference
+                              where the content is stored. This value is mandatory.
+                            type: string
+                          name:
+                            description: Name is unique within a namespace to reference
+                              a secret resource.
+                            type: string
+                          namespace:
+                            description: Namespace defines the space within which
+                              the secret name must be unique.
+                            type: string
+                        required:
+                        - keyPath
+                        type: object
+                    type: object
+                  username:
+                    properties:
+                      content:
+                        description: Bare content of the file, base64 encoded. It
+                          has precedence over the SecretReference value.
+                        format: byte
+                        type: string
+                      secretReference:
+                        properties:
+                          keyPath:
+                            description: Name of the key for the given Secret reference
+                              where the content is stored. This value is mandatory.
+                            type: string
+                          name:
+                            description: Name is unique within a namespace to reference
+                              a secret resource.
+                            type: string
+                          namespace:
+                            description: Namespace defines the space within which
+                              the secret name must be unique.
+                            type: string
+                        required:
+                        - keyPath
+                        type: object
+                    type: object
+                required:
+                - password
+                - username
+                type: object
+              driver:
+                description: The driver to use to connect to the shared datastore.
+                type: string
+              endpoints:
+                description: List of the endpoints to connect to the shared datastore.
+                  No need for protocol, just bare IP/FQDN and port.
+                items:
+                  type: string
+                type: array
+              tlsConfig:
+                description: Defines the TLS/SSL configuration required to connect
+                  to the data store in a secure way.
+                properties:
+                  certificateAuthority:
+                    description: Retrieve the Certificate Authority certificate and
+                      private key, such as bare content of the file, or a SecretReference.
+                      The key reference is required since etcd authentication is based
+                      on certificates, and Kamaji is responsible in creating this.
+                    properties:
+                      certificate:
+                        properties:
+                          content:
+                            description: Bare content of the file, base64 encoded.
+                              It has precedence over the SecretReference value.
+                            format: byte
+                            type: string
+                          secretReference:
+                            properties:
+                              keyPath:
+                                description: Name of the key for the given Secret
+                                  reference where the content is stored. This value
+                                  is mandatory.
+                                type: string
+                              name:
+                                description: Name is unique within a namespace to
+                                  reference a secret resource.
+                                type: string
+                              namespace:
+                                description: Namespace defines the space within which
+                                  the secret name must be unique.
+                                type: string
+                            required:
+                            - keyPath
+                            type: object
+                        type: object
+                      privateKey:
+                        properties:
+                          content:
+                            description: Bare content of the file, base64 encoded.
+                              It has precedence over the SecretReference value.
+                            format: byte
+                            type: string
+                          secretReference:
+                            properties:
+                              keyPath:
+                                description: Name of the key for the given Secret
+                                  reference where the content is stored. This value
+                                  is mandatory.
+                                type: string
+                              name:
+                                description: Name is unique within a namespace to
+                                  reference a secret resource.
+                                type: string
+                              namespace:
+                                description: Namespace defines the space within which
+                                  the secret name must be unique.
+                                type: string
+                            required:
+                            - keyPath
+                            type: object
+                        type: object
+                    required:
+                    - certificate
+                    type: object
+                  clientCertificate:
+                    description: Specifies the SSL/TLS key and private key pair used
+                      to connect to the data store.
+                    properties:
+                      certificate:
+                        properties:
+                          content:
+                            description: Bare content of the file, base64 encoded.
+                              It has precedence over the SecretReference value.
+                            format: byte
+                            type: string
+                          secretReference:
+                            properties:
+                              keyPath:
+                                description: Name of the key for the given Secret
+                                  reference where the content is stored. This value
+                                  is mandatory.
+                                type: string
+                              name:
+                                description: Name is unique within a namespace to
+                                  reference a secret resource.
+                                type: string
+                              namespace:
+                                description: Namespace defines the space within which
+                                  the secret name must be unique.
+                                type: string
+                            required:
+                            - keyPath
+                            type: object
+                        type: object
+                      privateKey:
+                        properties:
+                          content:
+                            description: Bare content of the file, base64 encoded.
+                              It has precedence over the SecretReference value.
+                            format: byte
+                            type: string
+                          secretReference:
+                            properties:
+                              keyPath:
+                                description: Name of the key for the given Secret
+                                  reference where the content is stored. This value
+                                  is mandatory.
+                                type: string
+                              name:
+                                description: Name is unique within a namespace to
+                                  reference a secret resource.
+                                type: string
+                              namespace:
+                                description: Namespace defines the space within which
+                                  the secret name must be unique.
+                                type: string
+                            required:
+                            - keyPath
+                            type: object
+                        type: object
+                    required:
+                    - certificate
+                    - privateKey
+                    type: object
+                required:
+                - certificateAuthority
+                - clientCertificate
+                type: object
+            required:
+            - driver
+            - endpoints
+            - tlsConfig
+            type: object
+          status:
+            description: DataStoreStatus defines the observed state of DataStore.
+            properties:
+              usedBy:
+                description: List of the Tenant Control Planes, namespaced named,
+                  using this data store.
+                items:
+                  type: string
+                type: array
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []

charts/kamaji/templates/NOTES.txt

@@ -0,0 +1,5 @@
+List the available CRDs installed by Kamaji:
+  kubectl get customresourcedefinitions.apiextensions.k8s.io
+
+List all the Tenant Control Plane resources deployed in your cluster:
+  kubectl get tenantcontrolplanes.kamaji.clastix.io --all-namespaces

charts/kamaji/templates/_helpers_datastore.tpl

@@ -0,0 +1,90 @@
+{{/*
+Create a default fully qualified datastore name.
+*/}}
+{{- define "datastore.fullname" -}}
+{{- default "default" .Values.datastore.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "datastore.labels" -}}
+kamaji.clastix.io/datastore: {{ .Values.datastore.driver }}
+helm.sh/chart: {{ include "kamaji.chart" . }}
+{{ include "kamaji.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Datastore endpoints, in case of ETCD, retrieving the one provided by the chart.
+*/}}
+{{- define "datastore.endpoints" -}}
+{{- if eq .Values.datastore.driver "etcd" }}
+{{ include "etcd.endpoints" . }}
+{{- else }}
+{{ .Values.datastore.endpoints }}
+{{- end }}
+{{- end }}
+
+{{/*
+The Certificate Authority section for the DataSource object.
+*/}}
+{{- define "datastore.certificateAuthority" -}}
+{{- if eq .Values.datastore.driver "etcd" }}
+certificate:
+  secretReference:
+    name: {{ include "etcd.caSecretName" . }}
+    namespace: {{ include "etcd.caSecretNamespace" . }}
+    keyPath: ca.crt
+privateKey:
+  secretReference:
+    name: {{ include "etcd.caSecretName" . }}
+    namespace: {{ include "etcd.caSecretNamespace" . }}
+    keyPath: ca.key
+{{- else }}
+certificate:
+  secretReference:
+    name: {{ .Values.datastore.tlsConfig.certificateAuthority.certificate.name }}
+    namespace: {{ .Values.datastore.tlsConfig.certificateAuthority.certificate.namespace }}
+    keyPath: {{ .Values.datastore.tlsConfig.certificateAuthority.certificate.keyPath }}
+{{- if .Values.datastore.tlsConfig.certificateAuthority.privateKey.name }}
+privateKey:
+  secretReference:
+    name: {{ .Values.datastore.tlsConfig.certificateAuthority.privateKey.name }}
+    namespace: {{ .Values.datastore.tlsConfig.certificateAuthority.privateKey.namespace }}
+    keyPath: {{ .Values.datastore.tlsConfig.certificateAuthority.privateKey.keyPath }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+The Client Certificate section for the DataSource object.
+*/}}
+{{- define "datastore.clientCertificate" -}}
+{{- if eq .Values.datastore.driver "etcd" }}
+certificate:
+    secretReference:
+      name: {{ include "etcd.clientSecretName" . }}
+      namespace: {{ include "etcd.clientSecretNamespace" . }}
+      keyPath: tls.crt
+privateKey:
+    secretReference:
+      name: {{ include "etcd.clientSecretName" . }}
+      namespace: {{ include "etcd.clientSecretNamespace" . }}
+      keyPath: tls.key
+{{- else }}
+certificate:
+  secretReference:
+    name: {{ .Values.datastore.tlsConfig.clientCertificate.certificate.name }}
+    namespace: {{ .Values.datastore.tlsConfig.clientCertificate.certificate.namespace }}
+    keyPath: {{ .Values.datastore.tlsConfig.clientCertificate.certificate.keyPath }}
+privateKey:
+  secretReference:
+    name: {{ .Values.datastore.tlsConfig.clientCertificate.privateKey.name }}
+    namespace: {{ .Values.datastore.tlsConfig.clientCertificate.privateKey.namespace }}
+    keyPath: {{ .Values.datastore.tlsConfig.clientCertificate.privateKey.keyPath }}
+{{- end }}
+{{- end }}

charts/kamaji/templates/_helpers_etcd.tpl

@@ -0,0 +1,142 @@
+{{/*
+Create a default fully qualified etcd name.
+*/}}
+{{- define "etcd.fullname" -}}
+{{- printf "etcd" }}
+{{- end }}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "etcd.serviceAccountName" -}}
+{{- if .Values.etcd.serviceAccount.create }}
+{{- default (include "etcd.fullname" .) .Values.etcd.serviceAccount.name }}
+{{- else }}
+{{- default "default" .Values.etcd.serviceAccount.name }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create the name of the Service to use
+*/}}
+{{- define "etcd.serviceName" -}}
+{{- printf "%s" (include "etcd.fullname" .) | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "etcd.labels" -}}
+app.kubernetes.io/name: {{ include "kamaji.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+app.kubernetes.io/components: etcd
+{{- end }}
+
+{{/*
+Selector labels.
+*/}}
+{{- define "etcd.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "kamaji.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+app.kubernetes.io/component: etcd
+{{- end }}
+
+{{/*
+Name of the etcd CA secret.
+*/}}
+{{- define "etcd.caSecretName" }}
+{{- if .Values.etcd.deploy }}
+{{- printf "%s-%s" (include "etcd.fullname" .) "certs" | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- required "A valid .Values.etcd.overrides.caSecret.name required!" .Values.etcd.overrides.caSecret.name }}
+{{- end }}
+{{- end }}
+
+{{/*
+Namespace of the etcd CA secret.
+*/}}
+{{- define "etcd.caSecretNamespace" }}
+{{- if .Values.etcd.deploy }}
+{{- .Release.Namespace }}
+{{- else }}
+{{- required "A valid .Values.etcd.overrides.caSecret.namespace required!" .Values.etcd.overrides.caSecret.namespace }}
+{{- end }}
+{{- end }}
+
+{{/*
+Name of the certificate signing requests for the certificates required by etcd.
+*/}}
+{{- define "etcd.csrConfigMapName" }}
+{{- printf "%s-csr" (include "etcd.fullname" .) }}
+{{- end }}
+
+{{/*
+Name of the etcd root-client secret.
+*/}}
+{{- define "etcd.clientSecretName" }}
+{{- if .Values.etcd.deploy }}
+{{- printf "root-client-certs" }}
+{{- else }}
+{{- required "A valid .Values.etcd.overrides.clientSecret.name required!" .Values.etcd.overrides.clientSecret.name }}
+{{- end }}
+{{- end }}
+
+{{/*
+Namespace of the etcd root-client secret.
+*/}}
+{{- define "etcd.clientSecretNamespace" }}
+{{- if .Values.etcd.deploy }}
+{{- .Release.Namespace }}
+{{- else }}
+{{- required "A valid .Values.etcd.overrides.clientSecret.namespace required!" .Values.etcd.overrides.clientSecret.namespace }}
+{{- end }}
+{{- end }}
+
+{{/*
+Comma separated list of etcd endpoints, using the overrides in case of unmanaged etcd.
+*/}}
+{{- define "etcd.endpoints" }}
+{{- $list := list -}}
+{{- if .Values.etcd.deploy }}
+    {{- range $count := until 3 -}}
+        {{- $list = append $list (printf "%s-%d.%s.%s.svc.cluster.local:%d" "etcd" $count ( include "etcd.serviceName" . ) $.Release.Namespace (int $.Values.etcd.port) ) -}}
+    {{- end }}
+{{- else if .Values.etcd.overrides.endpoints }}
+    {{- range $v := .Values.etcd.overrides.endpoints -}}
+        {{- $list = append $list (printf "%s:%d" $v (int $.Values.etcd.port) ) -}}
+    {{- end -}}
+{{- else if not .Values.etcd.overrides.endpoints }}
+    {{- fail "A valid .Values.etcd.overrides.endpoints required!" }}
+{{- end }}
+{{- $list | toYaml }}
+{{- end }}
+
+{{/*
+Key-value of the etcd peers, using the overrides in case of unmanaged etcd.
+*/}}
+{{- define "etcd.initialCluster" }}
+{{- $list := list -}}
+{{- if .Values.etcd.deploy }}
+    {{- range $i, $count := until 3 -}}
+        {{- $list = append $list ( printf "etcd-%d=https://%s-%d.%s.%s.svc.cluster.local:%d" $i "etcd" $count ( include "etcd.serviceName" . ) $.Release.Namespace (int $.Values.etcd.peerApiPort) ) -}}
+    {{- end }}
+{{- else if .Values.etcd.overrides.endpoints }}
+    {{- range $k, $v := .Values.etcd.overrides.endpoints -}}
+        {{- $list = append $list ( printf "%s=%s:%d" $k $v (int $.Values.etcd.peerApiPort) ) -}}
+    {{- end -}}
+{{- else if not .Values.etcd.overrides.endpoints }}
+    {{- fail "A valid .Values.etcd.overrides.endpoints required!" }}
+{{- end }}
+{{- join "," $list -}}
+{{- end }}
+
+{{/*
+Retrieve the current Kubernetes version to launch a kubectl container with the minimum version skew possible.
+*/}}
+{{- define "etcd.jobsTagKubeVersion" -}}
+{{- if contains "-eks-" .Capabilities.KubeVersion.GitVersion }}
+{{- print "v" .Capabilities.KubeVersion.Major "." (.Capabilities.KubeVersion.Minor | replace "+" "") -}}
+{{- else }}
+{{- print "v" .Capabilities.KubeVersion.Major "." .Capabilities.KubeVersion.Minor -}}
+{{- end }}
+{{- end }}

charts/kamaji/templates/controller.yaml

@@ -40,16 +40,11 @@ spec:
           protocol: TCP
       - args:
         - --config-file={{ .Values.configPath }}
-        - --etcd-ca-secret-name={{ .Values.etcd.caSecret.name }}
-        - --etcd-ca-secret-namespace={{ .Values.etcd.caSecret.namespace }}
-        - --etcd-client-secret-name={{ .Values.etcd.clientSecret.name }}
-        - --etcd-client-secret-namespace={{ .Values.etcd.clientSecret.namespace }}
-        - --etcd-compaction-interval={{ .Values.etcd.compactionInterval }}
-        - --etcd-endpoints={{ .Values.etcd.endpoints }}
         - --health-probe-bind-address={{ .Values.healthProbeBindAddress }}
         - --leader-elect
         - --metrics-bind-address={{ .Values.metricsBindAddress }}
         - --tmp-directory={{ .Values.temporaryDirectoryPath }}
+        - --datastore={{ include "datastore.fullname" . }}
         {{- if .Values.loggingDevel.enable }}
         - --zap-devel
         {{- end }}

charts/kamaji/templates/datastore.yaml

@@ -0,0 +1,19 @@
+apiVersion: kamaji.clastix.io/v1alpha1
+kind: DataStore
+metadata:
+  name: {{ include "datastore.fullname" . }}
+  labels:
+    {{- include "datastore.labels" . | nindent 4 }}
+spec:
+  driver: {{ .Values.datastore.driver }}
+  endpoints:
+    {{- include "datastore.endpoints" . | indent 4 }}
+{{- if (and .Values.datastore.basicAuth.usernameSecret.name .Values.datastore.basicAuth.passwordSecret.name) }}
+  basicAuth:
+    {{- .Values.datastore.basicAuth | toYaml | nindent 4 }}
+{{- end }}
+  tlsConfig:
+    certificateAuthority:
+      {{- include "datastore.certificateAuthority" . | indent 6 }}
+    clientCertificate:
+      {{- include "datastore.clientCertificate" . | indent 6 }}

charts/kamaji/templates/etcd_cm.yaml

@@ -0,0 +1,94 @@
+{{- if .Values.etcd.deploy }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  labels:
+    {{- include "etcd.labels" . | nindent 4 }}
+  name: {{ include "etcd.csrConfigMapName" . }}
+  namespace: {{ .Release.Namespace }}
+data:
+  ca-csr.json: |-
+    {
+      "CN": "Clastix CA",
+      "key": {
+        "algo": "rsa",
+        "size": 2048
+      },
+      "names": [
+        {
+          "C": "IT",
+          "ST": "Italy",
+          "L": "Milan"
+        }
+      ]
+    }
+  config.json: |-
+    {
+      "signing": {
+        "default": {
+          "expiry": "8760h"
+        },
+        "profiles": {
+          "server-authentication": {
+            "usages": ["signing", "key encipherment", "server auth"],
+            "expiry": "8760h"
+          },
+          "client-authentication": {
+            "usages": ["signing", "key encipherment", "client auth"],
+            "expiry": "8760h"
+          },
+          "peer-authentication": {
+            "usages": ["signing", "key encipherment", "server auth", "client auth"],
+            "expiry": "8760h"
+          }
+        }
+      }
+    }
+  server-csr.json: |-
+    {
+      "CN": "etcd",
+      "key": {
+        "algo": "rsa",
+        "size": 2048
+      },
+      "hosts": [
+{{- range $count := until 3 -}}
+        {{ printf "\"etcd-%d.%s.%s.svc.cluster.local\"," $count (include "etcd.serviceName" .) $.Release.Namespace }}
+{{- end }}
+        "etcd-server.{{ .Release.Namespace }}.svc.cluster.local",
+        "etcd-server.{{ .Release.Namespace }}.svc",
+        "etcd-server",
+        "127.0.0.1"
+      ]
+    }
+  peer-csr.json: |-
+    {
+      "CN": "etcd",
+      "key": {
+        "algo": "rsa",
+        "size": 2048
+      },
+      "hosts": [
+{{- range $count := until 3 -}}
+        {{ printf "\"etcd-%d\"," $count }}
+        {{ printf "\"etcd-%d.%s\"," $count (include "etcd.serviceName" .) }}
+        {{ printf "\"etcd-%d.%s.%s.svc\"," $count (include "etcd.serviceName" .) $.Release.Namespace }}
+        {{ printf "\"etcd-%d.%s.%s.svc.cluster.local\"," $count (include "etcd.serviceName" .) $.Release.Namespace }}
+{{- end }}
+        "127.0.0.1"
+      ]
+    }
+  root-client-csr.json: |-
+    {
+      "CN": "root",
+      "key": {
+        "algo": "rsa",
+        "size": 2048
+      },
+      "names": [
+        {
+          "O": "system:masters"
+        }
+      ]
+    }
+{{- end }}

charts/kamaji/templates/etcd_job_postdelete.yaml

@@ -0,0 +1,31 @@
+{{- if .Values.etcd.deploy }}
+apiVersion: batch/v1
+kind: Job
+metadata:
+  labels:
+    {{- include "etcd.labels" . | nindent 4 }}
+  annotations:
+    "helm.sh/hook": pre-delete
+    "helm.sh/hook-weight": "-5"
+    "helm.sh/hook-delete-policy": "hook-succeeded,hook-failed"
+  name: "{{ .Release.Name }}-etcd-teardown"
+  namespace: {{ .Release.Namespace }}
+spec:
+  template:
+    metadata:
+      name: "{{ .Release.Name }}"
+    spec:
+      serviceAccountName: {{ include "etcd.serviceAccountName" . }}
+      restartPolicy: Never
+      containers:
+        - name: kubectl
+          image: {{ printf "clastix/kubectl:%s" (include "etcd.jobsTagKubeVersion" .) }}
+          command:
+            - kubectl
+            - --namespace={{ .Release.Namespace }}
+            - delete
+            - secret
+            - --ignore-not-found=true
+            - {{ include "etcd.caSecretName" . }}
+            - {{ include "etcd.clientSecretName" . }}
+{{- end }}

charts/kamaji/templates/etcd_job_postinstall.yaml

@@ -0,0 +1,91 @@
+{{- if .Values.etcd.deploy }}
+apiVersion: batch/v1
+kind: Job
+metadata:
+  labels:
+    {{- include "etcd.labels" . | nindent 4 }}
+  annotations:
+    "helm.sh/hook": post-install
+    "helm.sh/hook-weight": "-5"
+    "helm.sh/hook-delete-policy": "hook-succeeded,hook-failed"
+  name: "{{ .Release.Name }}-etcd-setup"
+  namespace: {{ .Release.Namespace }}
+spec:
+  template:
+    metadata:
+      name: "{{ .Release.Name }}"
+    spec:
+      serviceAccountName: {{ include "etcd.serviceAccountName" . }}
+      restartPolicy: Never
+      initContainers:
+        - name: cfssl
+          image: cfssl/cfssl:latest
+          command:
+            - bash
+            - -c
+            - |-
+              cfssl gencert -initca /csr/ca-csr.json | cfssljson -bare /certs/ca &&
+              mv /certs/ca.pem /certs/ca.crt && mv /certs/ca-key.pem /certs/ca.key &&
+              cfssl gencert -ca=/certs/ca.crt -ca-key=/certs/ca.key -config=/csr/config.json -profile=peer-authentication /csr/peer-csr.json | cfssljson -bare /certs/peer &&
+              cfssl gencert -ca=/certs/ca.crt -ca-key=/certs/ca.key -config=/csr/config.json -profile=peer-authentication /csr/server-csr.json | cfssljson -bare /certs/server &&
+              cfssl gencert -ca=/certs/ca.crt -ca-key=/certs/ca.key -config=/csr/config.json -profile=client-authentication /csr/root-client-csr.json | cfssljson -bare /certs/root-client
+          volumeMounts:
+            - mountPath: /certs
+              name: certs
+            - mountPath: /csr
+              name: csr
+        - name: kubectl
+          image: {{ printf "clastix/kubectl:%s" (include "etcd.jobsTagKubeVersion" .) }}
+          command:
+            - sh
+            - -c
+            - |-
+              kubectl --namespace={{ .Release.Namespace }} delete secret --ignore-not-found=true {{ include "etcd.caSecretName" . }} {{ include "etcd.clientSecretName" . }} &&
+              kubectl --namespace={{ .Release.Namespace }} create secret generic {{ include "etcd.caSecretName" . }} --from-file=/certs/ca.crt --from-file=/certs/ca.key --from-file=/certs/peer-key.pem --from-file=/certs/peer.pem --from-file=/certs/server-key.pem --from-file=/certs/server.pem &&
+              kubectl --namespace={{ .Release.Namespace }} create secret tls {{ include "etcd.clientSecretName" . }} --key=/certs/root-client-key.pem --cert=/certs/root-client.pem &&
+              kubectl --namespace={{ .Release.Namespace }} rollout status sts/etcd --timeout=300s
+          volumeMounts:
+            - mountPath: /certs
+              name: certs
+      containers:
+        - command:
+            - bash
+            - -c
+            - |-
+              etcdctl member list -w table &&
+              etcdctl user add --no-password=true root &&
+              etcdctl role add root &&
+              etcdctl user grant-role root root &&
+              etcdctl auth enable
+          env:
+            - name: ETCDCTL_ENDPOINTS
+              value: https://etcd-0.{{ include "etcd.serviceName" . }}.{{ .Release.Namespace }}.svc.cluster.local:2379
+            - name: ETCDCTL_CACERT
+              value: /opt/certs/ca/ca.crt
+            - name: ETCDCTL_CERT
+              value: /opt/certs/root-certs/tls.crt
+            - name: ETCDCTL_KEY
+              value: /opt/certs/root-certs/tls.key
+          image: quay.io/coreos/etcd:v3.5.1
+          imagePullPolicy: Always
+          name: etcd-client
+          volumeMounts:
+            - name: root-certs
+              mountPath: /opt/certs/root-certs
+            - name: certs
+              mountPath: /opt/certs/ca
+      securityContext:
+        runAsUser: 1000
+        runAsGroup: 1000
+        fsGroup: 1000
+      volumes:
+        - name: root-certs
+          secret:
+            secretName: {{ include "etcd.clientSecretName" . }}
+            optional: true
+        - name: csr
+          configMap:
+            name: {{ include "etcd.csrConfigMapName" . }}
+        - name: certs
+          emptyDir: {}
+{{- end }}

charts/kamaji/templates/etcd_rbac.yaml

@@ -0,0 +1,49 @@
+{{- if .Values.etcd.deploy }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  labels:
+    {{- include "etcd.labels" . | nindent 4 }}
+  name: etcd-gen-certs-role
+  namespace: {{ .Release.Namespace }}
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - secrets
+    verbs:
+      - delete
+    resourceNames:
+      - {{ include "etcd.caSecretName" . }}
+      - {{ include "etcd.clientSecretName" . }}
+  - apiGroups:
+      - ""
+    resources:
+      - secrets
+    verbs:
+      - create
+  - apiGroups:
+      - apps
+    resources:
+      - statefulsets
+    verbs:
+      - get
+      - list
+      - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  labels:
+    {{- include "etcd.labels" . | nindent 4 }}
+  name: etcd-gen-certs-rolebiding
+  namespace: {{ .Release.Namespace }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: etcd-gen-certs-role
+subjects:
+  - kind: ServiceAccount
+    name: {{ include "etcd.serviceAccountName" . }}
+    namespace: {{ .Release.Namespace }}
+{{- end }}

charts/kamaji/templates/etcd_sa.yaml

@@ -0,0 +1,9 @@
+{{- if .Values.etcd.deploy }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    {{- include "etcd.labels" . | nindent 4 }}
+  name: {{ include "etcd.serviceAccountName" . }}
+  namespace: {{ .Release.Namespace }}
+{{- end }}

charts/kamaji/templates/etcd_service.yaml

@@ -0,0 +1,18 @@
+{{- if .Values.etcd.deploy }}
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    {{- include "etcd.labels" . | nindent 4 }}
+  name: {{ include "etcd.serviceName" . }}
+  namespace: {{ .Release.Namespace }}
+spec:
+  clusterIP: None
+  ports:
+    - port: {{ .Values.etcd.port }}
+      name: client
+    - port: {{ .Values.etcd.peerApiPort }}
+      name: peer
+  selector:
+    {{- include "etcd.selectorLabels" . | nindent 4 }}
+{{- end }}

charts/kamaji/templates/etcd_sts.yaml

@@ -0,0 +1,93 @@
+{{- if .Values.etcd.deploy }}
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  labels:
+    {{- include "etcd.labels" . | nindent 4 }}
+  name: {{ include "etcd.fullname" . }}
+  namespace: {{ .Release.Namespace }}
+spec:
+  serviceName: {{ include "etcd.serviceName" . }}
+  selector:
+    matchLabels:
+      {{- include "etcd.selectorLabels" . | nindent 6 }}
+  replicas: 3
+  template:
+    metadata:
+      name: etcd
+      labels:
+        {{- include "etcd.selectorLabels" . | nindent 8 }}
+    spec:
+      volumes:
+        - name: certs
+          secret:
+            secretName: {{ include "etcd.caSecretName" . }}
+      containers:
+        - name: etcd
+          image: {{ .Values.etcd.image.repository }}:{{ .Values.etcd.image.tag | default "v3.5.4" }}
+          imagePullPolicy: {{ .Values.etcd.image.pullPolicy }}
+          ports:
+            - containerPort: 2379
+              name: client
+            - containerPort: 2380
+              name: peer
+          volumeMounts:
+            - name: data
+              mountPath: /var/run/etcd
+            - name: certs
+              mountPath: /etc/etcd/pki
+          command:
+            - etcd
+            - --data-dir=/var/run/etcd
+            - --name=$(POD_NAME)
+            - --initial-cluster-state=new
+            - --initial-cluster={{ include "etcd.initialCluster" . }}
+            - --initial-advertise-peer-urls=https://$(POD_NAME).etcd.$(POD_NAMESPACE).svc.cluster.local:2380
+            - --advertise-client-urls=https://$(POD_NAME).etcd.$(POD_NAMESPACE).svc.cluster.local:2379
+            - --initial-cluster-token=kamaji
+            - --listen-client-urls=https://0.0.0.0:2379
+            - --listen-metrics-urls=http://0.0.0.0:2381
+            - --listen-peer-urls=https://0.0.0.0:2380
+            - --client-cert-auth=true
+            - --peer-client-cert-auth=true
+            - --trusted-ca-file=/etc/etcd/pki/ca.crt
+            - --cert-file=/etc/etcd/pki/server.pem
+            - --key-file=/etc/etcd/pki/server-key.pem
+            - --peer-trusted-ca-file=/etc/etcd/pki/ca.crt
+            - --peer-cert-file=/etc/etcd/pki/peer.pem
+            - --peer-key-file=/etc/etcd/pki/peer-key.pem
+            - --auto-compaction-mode=periodic
+            - --auto-compaction-retention=5m
+            - --snapshot-count=10000
+            - --quota-backend-bytes=8589934592
+            - --v=8
+          env:
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+          {{- with .Values.etcd.livenessProbe }}
+          livenessProbe:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.etcd.startupProbe }}
+          startupProbe:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+  volumeClaimTemplates:
+  - metadata:
+      name: data
+    spec:
+      storageClassName: {{ .Values.etcd.persistence.storageClassName }}
+      accessModes:
+      {{- range .Values.etcd.persistence.accessModes }}
+      - {{ . | quote }}
+      {{- end }}
+      resources:
+        requests:
+          storage: {{ .Values.etcd.persistence.size }}
+{{- end }}

charts/kamaji/templates/rbac.yaml

@@ -102,6 +102,32 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+    - kamaji.clastix.io
+  resources:
+    - datastores
+  verbs:
+    - create
+    - delete
+    - get
+    - list
+    - patch
+    - update
+    - watch
+- apiGroups:
+    - kamaji.clastix.io
+  resources:
+    - datastores/finalizers
+  verbs:
+    - update
+- apiGroups:
+    - kamaji.clastix.io
+  resources:
+    - datastores/status
+  verbs:
+    - get
+    - patch
+    - update
 - apiGroups:
   - kamaji.clastix.io
   resources:

charts/kamaji/values.sample.yaml

@@ -0,0 +1,2 @@
+etcd:
+  endpoints: "https://etcd-0.etcd.kamaji-system.svc.cluster.local:2379,https://etcd-1.etcd.kamaji-system.svc.cluster.local:2379,https://etcd-2.etcd.kamaji-system.svc.cluster.local:2379"

charts/kamaji/values.yaml

@@ -0,0 +1,210 @@
+# Default values for kamaji.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+# -- The number of the pod replicas for the Kamaji controller.
+replicaCount: 1
+
+image:
+  # -- The container image of the Kamaji controller.
+  repository: clastix/kamaji
+  pullPolicy: Always
+  # Overrides the image tag whose default is the chart appVersion.
+  tag: latest
+
+# -- A list of extra arguments to add to the kamaji controller default ones
+extraArgs: []
+
+# -- Configuration file path alternative. (default "./kamaji.yaml")
+configPath: "./kamaji.yaml"
+
+etcd:
+  # -- Install an etcd with enabled multi-tenancy along with Kamaji
+  deploy: true
+
+  # -- The peer API port which servers are listening to.
+  peerApiPort: 2380
+
+  # -- The client request port.
+  port: 2379
+
+  # -- Install specific etcd image
+  image:
+    repository: quay.io/coreos/etcd
+    tag: "v3.5.4"
+    pullPolicy: IfNotPresent
+
+  # -- The livenessProbe for the etcd container
+  livenessProbe:
+    failureThreshold: 8
+    httpGet:
+      path: /health?serializable=true
+      port: 2381
+      scheme: HTTP
+    initialDelaySeconds: 10
+    periodSeconds: 10
+    timeoutSeconds: 15
+
+  serviceAccount:
+    # -- Create a ServiceAccount, required to install and provision the etcd backing storage (default: true)
+    create: true
+    # -- Define the ServiceAccount name to use during the setup and provision of the etcd backing storage (default: "")
+    name: ""
+  persistence:
+    size: 10Gi
+    storageClass: ""
+    accessModes:
+    - ReadWriteOnce
+
+  overrides:
+    caSecret:
+      # -- Name of the secret which contains CA's certificate and private key. (default: "etcd-certs")
+      name: etcd-certs
+      # -- Namespace of the secret which contains CA's certificate and private key. (default: "kamaji-system")
+      namespace: kamaji-system
+    clientSecret:
+      # -- Name of the secret which contains ETCD client certificates. (default: "root-client-certs")
+      name: root-client-certs
+      # -- Name of the namespace where the secret which contains ETCD client certificates is. (default: "kamaji-system")
+      namespace: kamaji-system
+    # -- (map) Dictionary of the endpoints for the etcd cluster's members, key is the name of the etcd server. Don't define any port, inflected from .etcd.peerApiPort value.
+    endpoints:
+      etcd-0: https://etcd-0.etcd.kamaji-system.svc.cluster.local
+      etcd-1: https://etcd-1.etcd.kamaji-system.svc.cluster.local
+      etcd-2: https://etcd-2.etcd.kamaji-system.svc.cluster.local
+  # -- ETCD Compaction interval (e.g. "5m0s"). (default: "0" (disabled))
+  compactionInterval: 0
+
+# -- The address the probe endpoint binds to. (default ":8081")
+healthProbeBindAddress: ":8081"
+
+# -- The livenessProbe for the controller container
+livenessProbe:
+  httpGet:
+    path: /healthz
+    port: healthcheck
+  initialDelaySeconds: 15
+  periodSeconds: 20
+
+# -- The readinessProbe for the controller container
+readinessProbe:
+  httpGet:
+    path: /readyz
+    port: healthcheck
+  initialDelaySeconds: 5
+  periodSeconds: 10
+
+# -- (string) The address the metric endpoint binds to. (default ":8080")
+metricsBindAddress: ":8080"
+
+imagePullSecrets: []
+nameOverride: ""
+fullnameOverride: ""
+
+serviceAccount:
+  # Specifies whether a service account should be created
+  create: true
+  # Annotations to add to the service account
+  annotations: {}
+  # The name of the service account to use.
+  # If not set and create is true, a name is generated using the fullname template
+  name: kamaji-controller-manager
+
+# -- The annotations to apply to the Kamaji controller pods.
+podAnnotations: {}
+
+# -- The securityContext to apply to the Kamaji controller pods.
+podSecurityContext:
+  runAsNonRoot: true
+
+# -- The securityContext to apply to the Kamaji controller container only. It does not apply to the Kamaji RBAC proxy container.
+securityContext:
+  allowPrivilegeEscalation: false
+  # capabilities:
+  #   drop:
+  #   - ALL
+  # readOnlyRootFilesystem: true
+  # runAsNonRoot: true
+  # runAsUser: 1000
+
+service:
+  type: ClusterIP
+  port: 8443
+
+resources:
+  limits:
+    cpu: 200m
+    memory: 100Mi
+  requests:
+    cpu: 100m
+    memory: 20Mi
+
+# -- Kubernetes node selector rules to schedule Kamaji controller
+nodeSelector: {}
+
+# -- Kubernetes node taints that the Kamaji controller pods would tolerate
+tolerations: []
+
+# -- Kubernetes affinity rules to apply to Kamaji controller pods
+affinity: {}
+
+# -- Directory which will be used to work with temporary files. (default "/tmp/kamaji")
+temporaryDirectoryPath: "/tmp/kamaji"
+
+loggingDevel:
+  # -- (string) Development Mode defaults(encoder=consoleEncoder,logLevel=Debug,stackTraceLevel=Warn). Production Mode defaults(encoder=jsonEncoder,logLevel=Info,stackTraceLevel=Error) (default false)
+  enable: false
+
+datastore:
+  # -- (string) The Datastore name override, if empty defaults to `default`
+  nameOverride:
+  # -- (string) The Kamaji Datastore driver, supported: etcd, MySQL, PostgreSQL (defaults=etcd).
+  driver: etcd
+  # -- (array) List of endpoints of the selected Datastore. When letting the Chart install the etcd datastore, this field is populated automatically.
+  endpoints: []
+  basicAuth:
+    usernameSecret:
+      # -- The name of the Secret containing the username used to connect to the relational database.
+      name:
+      # -- The namespace of the Secret containing the username used to connect to the relational database.
+      namespace:
+      # -- The Secret key where the data is stored.
+      keyPath:
+    passwordSecret:
+      # -- The name of the Secret containing the password used to connect to the relational database.
+      name:
+      # -- The namespace of the Secret containing the password used to connect to the relational database.
+      namespace:
+      # -- The Secret key where the data is stored.
+      keyPath:
+  tlsConfig:
+    certificateAuthority:
+      certificate:
+        # -- Name of the Secret containing the CA required to establish the mandatory SSL/TLS connection to the datastore.
+        name:
+        # -- Namespace of the Secret containing the CA required to establish the mandatory SSL/TLS connection to the datastore.
+        namespace:
+        # -- Key of the Secret which contains the content of the certificate.
+        keyPath:
+      privateKey:
+        # -- Name of the Secret containing the CA private key required to establish the mandatory SSL/TLS connection to the datastore.
+        name:
+        # -- Namespace of the Secret containing the CA private key required to establish the mandatory SSL/TLS connection to the datastore.
+        namespace:
+        # -- Key of the Secret which contains the content of the private key.
+        keyPath:
+    clientCertificate:
+      certificate:
+        # -- Name of the Secret containing the client certificate required to establish the mandatory SSL/TLS connection to the datastore.
+        name:
+        # -- Namespace of the Secret containing the client certificate required to establish the mandatory SSL/TLS connection to the datastore.
+        namespace:
+        # -- Key of the Secret which contains the content of the certificate.
+        keyPath:
+      privateKey:
+        # -- Name of the Secret containing the client certificate private key required to establish the mandatory SSL/TLS connection to the datastore.
+        name:
+        # -- Namespace of the Secret containing the client certificate private key required to establish the mandatory SSL/TLS connection to the datastore.
+        namespace:
+        # -- Key of the Secret which contains the content of the private key.
+        keyPath:

config/crd/bases/kamaji.clastix.io_datastores.yaml

@@ -0,0 +1,269 @@
+
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.6.1
+  creationTimestamp: null
+  name: datastores.kamaji.clastix.io
+spec:
+  group: kamaji.clastix.io
+  names:
+    kind: DataStore
+    listKind: DataStoreList
+    plural: datastores
+    singular: datastore
+  scope: Cluster
+  versions:
+  - additionalPrinterColumns:
+    - description: Kamaji data store driver
+      jsonPath: .spec.driver
+      name: Driver
+      type: string
+    - description: Age
+      jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1alpha1
+    schema:
+      openAPIV3Schema:
+        description: DataStore is the Schema for the datastores API.
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: DataStoreSpec defines the desired state of DataStore.
+            properties:
+              basicAuth:
+                description: In case of authentication enabled for the given data
+                  store, specifies the username and password pair. This value is optional.
+                properties:
+                  password:
+                    properties:
+                      content:
+                        description: Bare content of the file, base64 encoded. It
+                          has precedence over the SecretReference value.
+                        format: byte
+                        type: string
+                      secretReference:
+                        properties:
+                          keyPath:
+                            description: Name of the key for the given Secret reference
+                              where the content is stored. This value is mandatory.
+                            type: string
+                          name:
+                            description: Name is unique within a namespace to reference
+                              a secret resource.
+                            type: string
+                          namespace:
+                            description: Namespace defines the space within which
+                              the secret name must be unique.
+                            type: string
+                        required:
+                        - keyPath
+                        type: object
+                    type: object
+                  username:
+                    properties:
+                      content:
+                        description: Bare content of the file, base64 encoded. It
+                          has precedence over the SecretReference value.
+                        format: byte
+                        type: string
+                      secretReference:
+                        properties:
+                          keyPath:
+                            description: Name of the key for the given Secret reference
+                              where the content is stored. This value is mandatory.
+                            type: string
+                          name:
+                            description: Name is unique within a namespace to reference
+                              a secret resource.
+                            type: string
+                          namespace:
+                            description: Namespace defines the space within which
+                              the secret name must be unique.
+                            type: string
+                        required:
+                        - keyPath
+                        type: object
+                    type: object
+                required:
+                - password
+                - username
+                type: object
+              driver:
+                description: The driver to use to connect to the shared datastore.
+                type: string
+              endpoints:
+                description: List of the endpoints to connect to the shared datastore.
+                  No need for protocol, just bare IP/FQDN and port.
+                items:
+                  type: string
+                type: array
+              tlsConfig:
+                description: Defines the TLS/SSL configuration required to connect
+                  to the data store in a secure way.
+                properties:
+                  certificateAuthority:
+                    description: Retrieve the Certificate Authority certificate and
+                      private key, such as bare content of the file, or a SecretReference.
+                      The key reference is required since etcd authentication is based
+                      on certificates, and Kamaji is responsible in creating this.
+                    properties:
+                      certificate:
+                        properties:
+                          content:
+                            description: Bare content of the file, base64 encoded.
+                              It has precedence over the SecretReference value.
+                            format: byte
+                            type: string
+                          secretReference:
+                            properties:
+                              keyPath:
+                                description: Name of the key for the given Secret
+                                  reference where the content is stored. This value
+                                  is mandatory.
+                                type: string
+                              name:
+                                description: Name is unique within a namespace to
+                                  reference a secret resource.
+                                type: string
+                              namespace:
+                                description: Namespace defines the space within which
+                                  the secret name must be unique.
+                                type: string
+                            required:
+                            - keyPath
+                            type: object
+                        type: object
+                      privateKey:
+                        properties:
+                          content:
+                            description: Bare content of the file, base64 encoded.
+                              It has precedence over the SecretReference value.
+                            format: byte
+                            type: string
+                          secretReference:
+                            properties:
+                              keyPath:
+                                description: Name of the key for the given Secret
+                                  reference where the content is stored. This value
+                                  is mandatory.
+                                type: string
+                              name:
+                                description: Name is unique within a namespace to
+                                  reference a secret resource.
+                                type: string
+                              namespace:
+                                description: Namespace defines the space within which
+                                  the secret name must be unique.
+                                type: string
+                            required:
+                            - keyPath
+                            type: object
+                        type: object
+                    required:
+                    - certificate
+                    type: object
+                  clientCertificate:
+                    description: Specifies the SSL/TLS key and private key pair used
+                      to connect to the data store.
+                    properties:
+                      certificate:
+                        properties:
+                          content:
+                            description: Bare content of the file, base64 encoded.
+                              It has precedence over the SecretReference value.
+                            format: byte
+                            type: string
+                          secretReference:
+                            properties:
+                              keyPath:
+                                description: Name of the key for the given Secret
+                                  reference where the content is stored. This value
+                                  is mandatory.
+                                type: string
+                              name:
+                                description: Name is unique within a namespace to
+                                  reference a secret resource.
+                                type: string
+                              namespace:
+                                description: Namespace defines the space within which
+                                  the secret name must be unique.
+                                type: string
+                            required:
+                            - keyPath
+                            type: object
+                        type: object
+                      privateKey:
+                        properties:
+                          content:
+                            description: Bare content of the file, base64 encoded.
+                              It has precedence over the SecretReference value.
+                            format: byte
+                            type: string
+                          secretReference:
+                            properties:
+                              keyPath:
+                                description: Name of the key for the given Secret
+                                  reference where the content is stored. This value
+                                  is mandatory.
+                                type: string
+                              name:
+                                description: Name is unique within a namespace to
+                                  reference a secret resource.
+                                type: string
+                              namespace:
+                                description: Namespace defines the space within which
+                                  the secret name must be unique.
+                                type: string
+                            required:
+                            - keyPath
+                            type: object
+                        type: object
+                    required:
+                    - certificate
+                    - privateKey
+                    type: object
+                required:
+                - certificateAuthority
+                - clientCertificate
+                type: object
+            required:
+            - driver
+            - endpoints
+            - tlsConfig
+            type: object
+          status:
+            description: DataStoreStatus defines the observed state of DataStore.
+            properties:
+              usedBy:
+                description: List of the Tenant Control Planes, namespaced named,
+                  using this data store.
+                items:
+                  type: string
+                type: array
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []

config/crd/bases/kamaji.clastix.io_tenantcontrolplanes.yaml

@@ -60,6 +60,68 @@ spec:
           spec:
             description: TenantControlPlaneSpec defines the desired state of TenantControlPlane.
             properties:
+              addons:
+                description: Addons contain which addons are enabled
+                properties:
+                  coreDNS:
+                    description: AddonSpec defines the spec for every addon.
+                    type: object
+                  konnectivity:
+                    description: KonnectivitySpec defines the spec for Konnectivity.
+                    properties:
+                      agentImage:
+                        default: us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-agent
+                        description: AgentImage defines the container image for Konnectivity's
+                          agent.
+                        type: string
+                      proxyPort:
+                        description: Port of Konnectivity proxy server.
+                        format: int32
+                        type: integer
+                      resources:
+                        description: Resources define the amount of CPU and memory
+                          to allocate to the Konnectivity server.
+                        properties:
+                          limits:
+                            additionalProperties:
+                              anyOf:
+                              - type: integer
+                              - type: string
+                              pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                              x-kubernetes-int-or-string: true
+                            description: 'Limits describes the maximum amount of compute
+                              resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                            type: object
+                          requests:
+                            additionalProperties:
+                              anyOf:
+                              - type: integer
+                              - type: string
+                              pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                              x-kubernetes-int-or-string: true
+                            description: 'Requests describes the minimum amount of
+                              compute resources required. If Requests is omitted for
+                              a container, it defaults to Limits if that is explicitly
+                              specified, otherwise to an implementation-defined value.
+                              More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                            type: object
+                        type: object
+                      serverImage:
+                        default: us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-server
+                        description: ServerImage defines the container image for Konnectivity's
+                          server.
+                        type: string
+                      version:
+                        default: v0.0.31
+                        description: Version for Konnectivity server and agent.
+                        type: string
+                    required:
+                    - proxyPort
+                    type: object
+                  kubeProxy:
+                    description: AddonSpec defines the spec for every addon.
+                    type: object
+                type: object
               controlPlane:
                 description: ControlPlane defines how the Tenant Control Plane Kubernetes
                   resources must be created in the Admin Cluster, such as the number
@@ -83,10 +145,124 @@ spec:
                               type: string
                             type: object
                         type: object
+                      extraArgs:
+                        description: ExtraArgs allows adding additional arguments
+                          to the Control Plane components, such as kube-apiserver,
+                          controller-manager, and scheduler.
+                        properties:
+                          apiServer:
+                            items:
+                              type: string
+                            type: array
+                          controllerManager:
+                            items:
+                              type: string
+                            type: array
+                          kine:
+                            description: Available only if Kamaji is running using
+                              Kine as backing storage.
+                            items:
+                              type: string
+                            type: array
+                          scheduler:
+                            items:
+                              type: string
+                            type: array
+                        type: object
                       replicas:
                         default: 2
                         format: int32
                         type: integer
+                      resources:
+                        description: Resources defines the amount of memory and CPU
+                          to allocate to each component of the Control Plane (kube-apiserver,
+                          controller-manager, and scheduler).
+                        properties:
+                          apiServer:
+                            description: ResourceRequirements describes the compute
+                              resource requirements.
+                            properties:
+                              limits:
+                                additionalProperties:
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
+                                description: 'Limits describes the maximum amount
+                                  of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                                type: object
+                              requests:
+                                additionalProperties:
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
+                                description: 'Requests describes the minimum amount
+                                  of compute resources required. If Requests is omitted
+                                  for a container, it defaults to Limits if that is
+                                  explicitly specified, otherwise to an implementation-defined
+                                  value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                                type: object
+                            type: object
+                          controllerManager:
+                            description: ResourceRequirements describes the compute
+                              resource requirements.
+                            properties:
+                              limits:
+                                additionalProperties:
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
+                                description: 'Limits describes the maximum amount
+                                  of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                                type: object
+                              requests:
+                                additionalProperties:
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
+                                description: 'Requests describes the minimum amount
+                                  of compute resources required. If Requests is omitted
+                                  for a container, it defaults to Limits if that is
+                                  explicitly specified, otherwise to an implementation-defined
+                                  value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                                type: object
+                            type: object
+                          scheduler:
+                            description: ResourceRequirements describes the compute
+                              resource requirements.
+                            properties:
+                              limits:
+                                additionalProperties:
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
+                                description: 'Limits describes the maximum amount
+                                  of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                                type: object
+                              requests:
+                                additionalProperties:
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
+                                description: 'Requests describes the minimum amount
+                                  of compute resources required. If Requests is omitted
+                                  for a container, it defaults to Limits if that is
+                                  explicitly specified, otherwise to an implementation-defined
+                                  value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                                type: object
+                            type: object
+                        type: object
                     type: object
                   ingress:
                     description: Defining the options for an Optional Ingress which
@@ -106,8 +282,6 @@ spec:
                               type: string
                             type: object
                         type: object
-                      enabled:
-                        type: boolean
                       hostname:
                         description: Hostname is an optional field which will be used
                           as Ingress's Host. If it is not defined, Ingress's host
@@ -116,8 +290,6 @@ spec:
                         type: string
                       ingressClassName:
                         type: string
-                    required:
-                    - enabled
                     type: object
                   service:
                     description: Defining the options for the Tenant Control Plane
@@ -246,14 +418,22 @@ spec:
                       in the section of ExternalIPs of the Kubernetes Service (only
                       ClusterIP or NodePort)
                     type: boolean
-                  dnsServiceIPs:
+                  certSANs:
+                    description: CertSANs sets extra Subject Alternative Names (SANs)
+                      for the API Server signing certificate. Use this field to add
+                      additional hostnames when exposing the Tenant Control Plane
+                      with third solutions.
+                    items:
+                      type: string
+                    type: array
+                  dnsServiceIPs:
+                    default:
+                    - 10.96.0.10
                     items:
                       type: string
                     type: array
-                  domain:
-                    description: Domain of the tenant control plane
-                    type: string
                   podCidr:
+                    default: 10.244.0.0/16
                     description: CIDR for Kubernetes Pods
                     type: string
                   port:
@@ -262,14 +442,9 @@ spec:
                     format: int32
                     type: integer
                   serviceCidr:
+                    default: 10.96.0.0/16
                     description: Kubernetes Service
                     type: string
-                required:
-                - dnsServiceIPs
-                - domain
-                - podCidr
-                - port
-                - serviceCidr
                 type: object
             required:
             - controlPlane
@@ -278,13 +453,291 @@ spec:
           status:
             description: TenantControlPlaneStatus defines the observed state of TenantControlPlane.
             properties:
+              addons:
+                description: Addons contains the status of the different Addons
+                properties:
+                  coreDNS:
+                    description: AddonStatus defines the observed state of an Addon.
+                    properties:
+                      checksum:
+                        type: string
+                      enabled:
+                        type: boolean
+                      lastUpdate:
+                        format: date-time
+                        type: string
+                    required:
+                    - enabled
+                    type: object
+                  konnectivity:
+                    description: KonnectivityStatus defines the status of Konnectivity
+                      as Addon.
+                    properties:
+                      agent:
+                        properties:
+                          checksum:
+                            type: string
+                          lastUpdate:
+                            description: Last time when k8s object was updated
+                            format: date-time
+                            type: string
+                          name:
+                            type: string
+                          namespace:
+                            type: string
+                        type: object
+                      certificate:
+                        description: CertificatePrivateKeyPairStatus defines the status.
+                        properties:
+                          checksum:
+                            type: string
+                          lastUpdate:
+                            format: date-time
+                            type: string
+                          secretName:
+                            type: string
+                        type: object
+                      clusterrolebinding:
+                        properties:
+                          checksum:
+                            type: string
+                          lastUpdate:
+                            description: Last time when k8s object was updated
+                            format: date-time
+                            type: string
+                          name:
+                            type: string
+                          namespace:
+                            type: string
+                        type: object
+                      configMap:
+                        properties:
+                          checksum:
+                            type: string
+                          name:
+                            type: string
+                        type: object
+                      enabled:
+                        type: boolean
+                      kubeconfig:
+                        description: KubeconfigStatus contains information about the
+                          generated kubeconfig.
+                        properties:
+                          checksum:
+                            type: string
+                          lastUpdate:
+                            format: date-time
+                            type: string
+                          secretName:
+                            type: string
+                        type: object
+                      sa:
+                        properties:
+                          checksum:
+                            type: string
+                          lastUpdate:
+                            description: Last time when k8s object was updated
+                            format: date-time
+                            type: string
+                          name:
+                            type: string
+                          namespace:
+                            type: string
+                        type: object
+                      service:
+                        description: KubernetesServiceStatus defines the status for
+                          the Tenant Control Plane Service in the management cluster.
+                        properties:
+                          conditions:
+                            description: Current service state
+                            items:
+                              description: "Condition contains details for one aspect
+                                of the current state of this API Resource. --- This
+                                struct is intended for direct use as an array at the
+                                field path .status.conditions.  For example, type
+                                FooStatus struct{     // Represents the observations
+                                of a foo's current state.     // Known .status.conditions.type
+                                are: \"Available\", \"Progressing\", and \"Degraded\"
+                                \    // +patchMergeKey=type     // +patchStrategy=merge
+                                \    // +listType=map     // +listMapKey=type     Conditions
+                                []metav1.Condition `json:\"conditions,omitempty\"
+                                patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"`
+                                \n     // other fields }"
+                              properties:
+                                lastTransitionTime:
+                                  description: lastTransitionTime is the last time
+                                    the condition transitioned from one status to
+                                    another. This should be when the underlying condition
+                                    changed.  If that is not known, then using the
+                                    time when the API field changed is acceptable.
+                                  format: date-time
+                                  type: string
+                                message:
+                                  description: message is a human readable message
+                                    indicating details about the transition. This
+                                    may be an empty string.
+                                  maxLength: 32768
+                                  type: string
+                                observedGeneration:
+                                  description: observedGeneration represents the .metadata.generation
+                                    that the condition was set based upon. For instance,
+                                    if .metadata.generation is currently 12, but the
+                                    .status.conditions[x].observedGeneration is 9,
+                                    the condition is out of date with respect to the
+                                    current state of the instance.
+                                  format: int64
+                                  minimum: 0
+                                  type: integer
+                                reason:
+                                  description: reason contains a programmatic identifier
+                                    indicating the reason for the condition's last
+                                    transition. Producers of specific condition types
+                                    may define expected values and meanings for this
+                                    field, and whether the values are considered a
+                                    guaranteed API. The value should be a CamelCase
+                                    string. This field may not be empty.
+                                  maxLength: 1024
+                                  minLength: 1
+                                  pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
+                                  type: string
+                                status:
+                                  description: status of the condition, one of True,
+                                    False, Unknown.
+                                  enum:
+                                  - "True"
+                                  - "False"
+                                  - Unknown
+                                  type: string
+                                type:
+                                  description: type of condition in CamelCase or in
+                                    foo.example.com/CamelCase. --- Many .condition.type
+                                    values are consistent across resources like Available,
+                                    but because arbitrary conditions can be useful
+                                    (see .node.status.conditions), the ability to
+                                    deconflict is important. The regex it matches
+                                    is (dns1123SubdomainFmt/)?(qualifiedNameFmt)
+                                  maxLength: 316
+                                  pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
+                                  type: string
+                              required:
+                              - lastTransitionTime
+                              - message
+                              - reason
+                              - status
+                              - type
+                              type: object
+                            type: array
+                            x-kubernetes-list-map-keys:
+                            - type
+                            x-kubernetes-list-type: map
+                          loadBalancer:
+                            description: LoadBalancer contains the current status
+                              of the load-balancer, if one is present.
+                            properties:
+                              ingress:
+                                description: Ingress is a list containing ingress
+                                  points for the load-balancer. Traffic intended for
+                                  the service should be sent to these ingress points.
+                                items:
+                                  description: 'LoadBalancerIngress represents the
+                                    status of a load-balancer ingress point: traffic
+                                    intended for the service should be sent to an
+                                    ingress point.'
+                                  properties:
+                                    hostname:
+                                      description: Hostname is set for load-balancer
+                                        ingress points that are DNS based (typically
+                                        AWS load-balancers)
+                                      type: string
+                                    ip:
+                                      description: IP is set for load-balancer ingress
+                                        points that are IP based (typically GCE or
+                                        OpenStack load-balancers)
+                                      type: string
+                                    ports:
+                                      description: Ports is a list of records of service
+                                        ports If used, every port defined in the service
+                                        should have an entry in it
+                                      items:
+                                        properties:
+                                          error:
+                                            description: 'Error is to record the problem
+                                              with the service port The format of
+                                              the error shall comply with the following
+                                              rules: - built-in error values shall
+                                              be specified in this file and those
+                                              shall use   CamelCase names - cloud
+                                              provider specific error values must
+                                              have names that comply with the   format
+                                              foo.example.com/CamelCase. --- The regex
+                                              it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)'
+                                            maxLength: 316
+                                            pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
+                                            type: string
+                                          port:
+                                            description: Port is the port number of
+                                              the service port of which status is
+                                              recorded here
+                                            format: int32
+                                            type: integer
+                                          protocol:
+                                            default: TCP
+                                            description: 'Protocol is the protocol
+                                              of the service port of which status
+                                              is recorded here The supported values
+                                              are: "TCP", "UDP", "SCTP"'
+                                            type: string
+                                        required:
+                                        - port
+                                        - protocol
+                                        type: object
+                                      type: array
+                                      x-kubernetes-list-type: atomic
+                                  type: object
+                                type: array
+                            type: object
+                          name:
+                            description: The name of the Service for the given cluster.
+                            type: string
+                          namespace:
+                            description: The namespace which the Service for the given
+                              cluster is deployed.
+                            type: string
+                          port:
+                            description: The port where the service is running
+                            format: int32
+                            type: integer
+                        required:
+                        - name
+                        - namespace
+                        - port
+                        type: object
+                    required:
+                    - enabled
+                    type: object
+                  kubeProxy:
+                    description: AddonStatus defines the observed state of an Addon.
+                    properties:
+                      checksum:
+                        type: string
+                      enabled:
+                        type: boolean
+                      lastUpdate:
+                        format: date-time
+                        type: string
+                    required:
+                    - enabled
+                    type: object
+                type: object
               certificates:
                 description: Certificates contains information about the different
                   certificates that are necessary to run a kubernetes control plane
                 properties:
                   apiServer:
-                    description: CertificatePrivateKeyPair defines the status.
+                    description: CertificatePrivateKeyPairStatus defines the status.
                     properties:
+                      checksum:
+                        type: string
                       lastUpdate:
                         format: date-time
                         type: string
@@ -292,8 +745,10 @@ spec:
                         type: string
                     type: object
                   apiServerKubeletClient:
-                    description: CertificatePrivateKeyPair defines the status.
+                    description: CertificatePrivateKeyPairStatus defines the status.
                     properties:
+                      checksum:
+                        type: string
                       lastUpdate:
                         format: date-time
                         type: string
@@ -301,8 +756,10 @@ spec:
                         type: string
                     type: object
                   ca:
-                    description: CertificatePrivateKeyPair defines the status.
+                    description: CertificatePrivateKeyPairStatus defines the status.
                     properties:
+                      checksum:
+                        type: string
                       lastUpdate:
                         format: date-time
                         type: string
@@ -310,13 +767,15 @@ spec:
                         type: string
                     type: object
                   etcd:
-                    description: ETCDAPIServerCertificate defines the observed state
+                    description: ETCDCertificatesStatus defines the observed state
                       of ETCD Certificate for API server.
                     properties:
                       apiServer:
-                        description: ETCDAPIServerCertificate defines the observed
+                        description: APIServerCertificatesStatus defines the observed
                           state of ETCD Certificate for API server.
                         properties:
+                          checksum:
+                            type: string
                           lastUpdate:
                             format: date-time
                             type: string
@@ -324,9 +783,11 @@ spec:
                             type: string
                         type: object
                       ca:
-                        description: ETCDAPIServerCertificate defines the observed
-                          state of ETCD Certificate for API server.
+                        description: ETCDCertificateStatus defines the observed state
+                          of ETCD Certificate for API server.
                         properties:
+                          checksum:
+                            type: string
                           lastUpdate:
                             format: date-time
                             type: string
@@ -335,8 +796,10 @@ spec:
                         type: object
                     type: object
                   frontProxyCA:
-                    description: CertificatePrivateKeyPair defines the status.
+                    description: CertificatePrivateKeyPairStatus defines the status.
                     properties:
+                      checksum:
+                        type: string
                       lastUpdate:
                         format: date-time
                         type: string
@@ -344,8 +807,10 @@ spec:
                         type: string
                     type: object
                   frontProxyClient:
-                    description: CertificatePrivateKeyPair defines the status.
+                    description: CertificatePrivateKeyPairStatus defines the status.
                     properties:
+                      checksum:
+                        type: string
                       lastUpdate:
                         format: date-time
                         type: string
@@ -353,8 +818,10 @@ spec:
                         type: string
                     type: object
                   sa:
-                    description: CertificatePrivateKeyPair defines the status.
+                    description: PublicKeyPrivateKeyPairStatus defines the status.
                     properties:
+                      checksum:
+                        type: string
                       lastUpdate:
                         format: date-time
                         type: string
@@ -370,59 +837,37 @@ spec:
                 description: KubeadmPhase contains the status of the kubeadm phases
                   action
                 properties:
-                  addonCoreDNS:
-                    description: KubeadmPhasesStatus contains the status of of a kubeadm
-                      phase action.
-                    properties:
-                      kubeadmConfigResourceVersion:
-                        type: string
-                      lastUpdate:
-                        format: date-time
-                        type: string
-                    type: object
-                  addonKubeProxy:
-                    description: KubeadmPhasesStatus contains the status of of a kubeadm
-                      phase action.
-                    properties:
-                      kubeadmConfigResourceVersion:
-                        type: string
-                      lastUpdate:
-                        format: date-time
-                        type: string
-                    type: object
                   bootstrapToken:
-                    description: KubeadmPhasesStatus contains the status of of a kubeadm
+                    description: KubeadmPhaseStatus contains the status of a kubeadm
                       phase action.
                     properties:
-                      kubeadmConfigResourceVersion:
+                      checksum:
                         type: string
                       lastUpdate:
                         format: date-time
                         type: string
                     type: object
                   uploadConfigKubeadm:
-                    description: KubeadmPhasesStatus contains the status of of a kubeadm
+                    description: KubeadmPhaseStatus contains the status of a kubeadm
                       phase action.
                     properties:
-                      kubeadmConfigResourceVersion:
+                      checksum:
                         type: string
                       lastUpdate:
                         format: date-time
                         type: string
                     type: object
                   uploadConfigKubelet:
-                    description: KubeadmPhasesStatus contains the status of of a kubeadm
+                    description: KubeadmPhaseStatus contains the status of a kubeadm
                       phase action.
                     properties:
-                      kubeadmConfigResourceVersion:
+                      checksum:
                         type: string
                       lastUpdate:
                         format: date-time
                         type: string
                     type: object
                 required:
-                - addonCoreDNS
-                - addonKubeProxy
                 - bootstrapToken
                 - uploadConfigKubeadm
                 - uploadConfigKubelet
@@ -431,34 +876,37 @@ spec:
                 description: KubeadmConfig contains the status of the configuration
                   required by kubeadm
                 properties:
+                  checksum:
+                    description: Checksum of the kubeadm configuration to detect changes
+                    type: string
                   configmapName:
                     type: string
                   lastUpdate:
                     format: date-time
                     type: string
-                  resourceVersion:
-                    type: string
-                required:
-                - resourceVersion
                 type: object
               kubeconfig:
                 description: KubeConfig contains information about the kubenconfigs
                   that control plane pieces need
                 properties:
                   admin:
-                    description: TenantControlPlaneKubeconfigsStatus contains information
-                      about a the generated kubeconfig.
+                    description: KubeconfigStatus contains information about the generated
+                      kubeconfig.
                     properties:
+                      checksum:
+                        type: string
                       lastUpdate:
                         format: date-time
                         type: string
                       secretName:
                         type: string
                     type: object
-                  controlerManager:
-                    description: TenantControlPlaneKubeconfigsStatus contains information
-                      about a the generated kubeconfig.
+                  controllerManager:
+                    description: KubeconfigStatus contains information about the generated
+                      kubeconfig.
                     properties:
+                      checksum:
+                        type: string
                       lastUpdate:
                         format: date-time
                         type: string
@@ -466,9 +914,11 @@ spec:
                         type: string
                     type: object
                   scheduler:
-                    description: TenantControlPlaneKubeconfigsStatus contains information
-                      about a the generated kubeconfig.
+                    description: KubeconfigStatus contains information about the generated
+                      kubeconfig.
                     properties:
+                      checksum:
+                        type: string
                       lastUpdate:
                         format: date-time
                         type: string
@@ -531,6 +981,10 @@ spec:
                           - type
                           type: object
                         type: array
+                      lastUpdate:
+                        description: Last time when deployment was updated
+                        format: date-time
+                        type: string
                       name:
                         description: The name of the Deployment for the given cluster.
                         type: string
@@ -827,52 +1281,42 @@ spec:
                         description: Version is the running Kubernetes version of
                           the Tenant Control Plane.
                         type: string
-                    required:
-                    - status
                     type: object
                 type: object
               storage:
                 description: Storage Status contains information about Kubernetes
                   storage system
                 properties:
-                  etcd:
-                    description: ETCDStatus defines the observed state of ETCDStatus.
+                  certificate:
                     properties:
-                      role:
-                        properties:
-                          exists:
-                            type: boolean
-                          name:
-                            type: string
-                          permissions:
-                            items:
-                              properties:
-                                key:
-                                  type: string
-                                rangeEnd:
-                                  type: string
-                                type:
-                                  type: integer
-                              type: object
-                            type: array
-                        required:
-                        - exists
-                        - name
-                        type: object
+                      checksum:
+                        type: string
+                      lastUpdate:
+                        format: date-time
+                        type: string
+                      secretName:
+                        type: string
+                    type: object
+                  config:
+                    properties:
+                      checksum:
+                        type: string
+                      secretName:
+                        type: string
+                    type: object
+                  driver:
+                    type: string
+                  setup:
+                    properties:
+                      checksum:
+                        type: string
+                      lastUpdate:
+                        format: date-time
+                        type: string
+                      schema:
+                        type: string
                       user:
-                        properties:
-                          exists:
-                            type: boolean
-                          name:
-                            type: string
-                          roles:
-                            items:
-                              type: string
-                            type: array
-                        required:
-                        - exists
-                        - name
-                        type: object
+                        type: string
                     type: object
                 type: object
             type: object

config/crd/patches/cainjection_in_datastores.yaml

@@ -0,0 +1,7 @@
+# The following patch adds a directive for certmanager to inject CA into the CRD
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME)
+  name: datastores.kamaji.clastix.io

config/crd/patches/webhook_in_datastores.yaml

@@ -0,0 +1,16 @@
+# The following patch enables a conversion webhook for the CRD
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: datastores.kamaji.clastix.io
+spec:
+  conversion:
+    strategy: Webhook
+    webhook:
+      clientConfig:
+        service:
+          namespace: system
+          name: webhook-service
+          path: /convert
+      conversionReviewVersions:
+      - v1

config/default/kustomization.yaml

@@ -16,6 +16,7 @@ bases:
 - ../crd
 - ../rbac
 - ../manager
+- ../samples
 # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
 # crd/kustomization.yaml
 #- ../webhook

config/install.yaml

@@ -60,6 +60,60 @@ spec:
           spec:
             description: TenantControlPlaneSpec defines the desired state of TenantControlPlane.
             properties:
+              addons:
+                description: Addons contain which addons are enabled
+                properties:
+                  coreDNS:
+                    description: AddonSpec defines the spec for every addon.
+                    type: object
+                  konnectivity:
+                    description: KonnectivitySpec defines the spec for Konnectivity.
+                    properties:
+                      agentImage:
+                        default: us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-agent
+                        description: AgentImage defines the container image for Konnectivity's agent.
+                        type: string
+                      proxyPort:
+                        description: Port of Konnectivity proxy server.
+                        format: int32
+                        type: integer
+                      resources:
+                        description: Resources define the amount of CPU and memory to allocate to the Konnectivity server.
+                        properties:
+                          limits:
+                            additionalProperties:
+                              anyOf:
+                              - type: integer
+                              - type: string
+                              pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                              x-kubernetes-int-or-string: true
+                            description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                            type: object
+                          requests:
+                            additionalProperties:
+                              anyOf:
+                              - type: integer
+                              - type: string
+                              pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                              x-kubernetes-int-or-string: true
+                            description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                            type: object
+                        type: object
+                      serverImage:
+                        default: us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-server
+                        description: ServerImage defines the container image for Konnectivity's server.
+                        type: string
+                      version:
+                        default: v0.0.31
+                        description: Version for Konnectivity server and agent.
+                        type: string
+                    required:
+                    - proxyPort
+                    type: object
+                  kubeProxy:
+                    description: AddonSpec defines the spec for every addon.
+                    type: object
+                type: object
               controlPlane:
                 description: ControlPlane defines how the Tenant Control Plane Kubernetes resources must be created in the Admin Cluster, such as the number of Pod replicas, the Service resource, or the Ingress.
                 properties:
@@ -78,10 +132,101 @@ spec:
                               type: string
                             type: object
                         type: object
+                      extraArgs:
+                        description: ExtraArgs allows adding additional arguments to the Control Plane components, such as kube-apiserver, controller-manager, and scheduler.
+                        properties:
+                          apiServer:
+                            items:
+                              type: string
+                            type: array
+                          controllerManager:
+                            items:
+                              type: string
+                            type: array
+                          kine:
+                            description: Available only if Kamaji is running using Kine as backing storage.
+                            items:
+                              type: string
+                            type: array
+                          scheduler:
+                            items:
+                              type: string
+                            type: array
+                        type: object
                       replicas:
                         default: 2
                         format: int32
                         type: integer
+                      resources:
+                        description: Resources defines the amount of memory and CPU to allocate to each component of the Control Plane (kube-apiserver, controller-manager, and scheduler).
+                        properties:
+                          apiServer:
+                            description: ResourceRequirements describes the compute resource requirements.
+                            properties:
+                              limits:
+                                additionalProperties:
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
+                                description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                                type: object
+                              requests:
+                                additionalProperties:
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
+                                description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                                type: object
+                            type: object
+                          controllerManager:
+                            description: ResourceRequirements describes the compute resource requirements.
+                            properties:
+                              limits:
+                                additionalProperties:
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
+                                description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                                type: object
+                              requests:
+                                additionalProperties:
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
+                                description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                                type: object
+                            type: object
+                          scheduler:
+                            description: ResourceRequirements describes the compute resource requirements.
+                            properties:
+                              limits:
+                                additionalProperties:
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
+                                description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                                type: object
+                              requests:
+                                additionalProperties:
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
+                                description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                                type: object
+                            type: object
+                        type: object
                     type: object
                   ingress:
                     description: Defining the options for an Optional Ingress which will expose API Server of the Tenant Control Plane
@@ -98,15 +243,11 @@ spec:
                               type: string
                             type: object
                         type: object
-                      enabled:
-                        type: boolean
                       hostname:
                         description: Hostname is an optional field which will be used as Ingress's Host. If it is not defined, Ingress's host will be "<tenant>.<namespace>.<domain>", where domain is specified under NetworkProfileSpec
                         type: string
                       ingressClassName:
                         type: string
-                    required:
-                    - enabled
                     type: object
                   service:
                     description: Defining the options for the Tenant Control Plane Service resource.
@@ -225,14 +366,19 @@ spec:
                   allowAddressAsExternalIP:
                     description: AllowAddressAsExternalIP will include tenantControlPlane.Spec.NetworkProfile.Address in the section of ExternalIPs of the Kubernetes Service (only ClusterIP or NodePort)
                     type: boolean
-                  dnsServiceIPs:
+                  certSANs:
+                    description: CertSANs sets extra Subject Alternative Names (SANs) for the API Server signing certificate. Use this field to add additional hostnames when exposing the Tenant Control Plane with third solutions.
+                    items:
+                      type: string
+                    type: array
+                  dnsServiceIPs:
+                    default:
+                    - 10.96.0.10
                     items:
                       type: string
                     type: array
-                  domain:
-                    description: Domain of the tenant control plane
-                    type: string
                   podCidr:
+                    default: 10.244.0.0/16
                     description: CIDR for Kubernetes Pods
                     type: string
                   port:
@@ -241,14 +387,9 @@ spec:
                     format: int32
                     type: integer
                   serviceCidr:
+                    default: 10.96.0.0/16
                     description: Kubernetes Service
                     type: string
-                required:
-                - dnsServiceIPs
-                - domain
-                - podCidr
-                - port
-                - serviceCidr
                 type: object
             required:
             - controlPlane
@@ -257,12 +398,225 @@ spec:
           status:
             description: TenantControlPlaneStatus defines the observed state of TenantControlPlane.
             properties:
+              addons:
+                description: Addons contains the status of the different Addons
+                properties:
+                  coreDNS:
+                    description: AddonStatus defines the observed state of an Addon.
+                    properties:
+                      checksum:
+                        type: string
+                      enabled:
+                        type: boolean
+                      lastUpdate:
+                        format: date-time
+                        type: string
+                    required:
+                    - enabled
+                    type: object
+                  konnectivity:
+                    description: KonnectivityStatus defines the status of Konnectivity as Addon.
+                    properties:
+                      agent:
+                        properties:
+                          checksum:
+                            type: string
+                          lastUpdate:
+                            description: Last time when k8s object was updated
+                            format: date-time
+                            type: string
+                          name:
+                            type: string
+                          namespace:
+                            type: string
+                        type: object
+                      certificate:
+                        description: CertificatePrivateKeyPairStatus defines the status.
+                        properties:
+                          checksum:
+                            type: string
+                          lastUpdate:
+                            format: date-time
+                            type: string
+                          secretName:
+                            type: string
+                        type: object
+                      clusterrolebinding:
+                        properties:
+                          checksum:
+                            type: string
+                          lastUpdate:
+                            description: Last time when k8s object was updated
+                            format: date-time
+                            type: string
+                          name:
+                            type: string
+                          namespace:
+                            type: string
+                        type: object
+                      configMap:
+                        properties:
+                          checksum:
+                            type: string
+                          name:
+                            type: string
+                        type: object
+                      enabled:
+                        type: boolean
+                      kubeconfig:
+                        description: KubeconfigStatus contains information about the generated kubeconfig.
+                        properties:
+                          checksum:
+                            type: string
+                          lastUpdate:
+                            format: date-time
+                            type: string
+                          secretName:
+                            type: string
+                        type: object
+                      sa:
+                        properties:
+                          checksum:
+                            type: string
+                          lastUpdate:
+                            description: Last time when k8s object was updated
+                            format: date-time
+                            type: string
+                          name:
+                            type: string
+                          namespace:
+                            type: string
+                        type: object
+                      service:
+                        description: KubernetesServiceStatus defines the status for the Tenant Control Plane Service in the management cluster.
+                        properties:
+                          conditions:
+                            description: Current service state
+                            items:
+                              description: "Condition contains details for one aspect of the current state of this API Resource. --- This struct is intended for direct use as an array at the field path .status.conditions.  For example, type FooStatus struct{     // Represents the observations of a foo's current state.     // Known .status.conditions.type are: \"Available\", \"Progressing\", and \"Degraded\"     // +patchMergeKey=type     // +patchStrategy=merge     // +listType=map     // +listMapKey=type     Conditions []metav1.Condition `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"` \n     // other fields }"
+                              properties:
+                                lastTransitionTime:
+                                  description: lastTransitionTime is the last time the condition transitioned from one status to another. This should be when the underlying condition changed.  If that is not known, then using the time when the API field changed is acceptable.
+                                  format: date-time
+                                  type: string
+                                message:
+                                  description: message is a human readable message indicating details about the transition. This may be an empty string.
+                                  maxLength: 32768
+                                  type: string
+                                observedGeneration:
+                                  description: observedGeneration represents the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date with respect to the current state of the instance.
+                                  format: int64
+                                  minimum: 0
+                                  type: integer
+                                reason:
+                                  description: reason contains a programmatic identifier indicating the reason for the condition's last transition. Producers of specific condition types may define expected values and meanings for this field, and whether the values are considered a guaranteed API. The value should be a CamelCase string. This field may not be empty.
+                                  maxLength: 1024
+                                  minLength: 1
+                                  pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
+                                  type: string
+                                status:
+                                  description: status of the condition, one of True, False, Unknown.
+                                  enum:
+                                  - "True"
+                                  - "False"
+                                  - Unknown
+                                  type: string
+                                type:
+                                  description: type of condition in CamelCase or in foo.example.com/CamelCase. --- Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be useful (see .node.status.conditions), the ability to deconflict is important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)
+                                  maxLength: 316
+                                  pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
+                                  type: string
+                              required:
+                              - lastTransitionTime
+                              - message
+                              - reason
+                              - status
+                              - type
+                              type: object
+                            type: array
+                            x-kubernetes-list-map-keys:
+                            - type
+                            x-kubernetes-list-type: map
+                          loadBalancer:
+                            description: LoadBalancer contains the current status of the load-balancer, if one is present.
+                            properties:
+                              ingress:
+                                description: Ingress is a list containing ingress points for the load-balancer. Traffic intended for the service should be sent to these ingress points.
+                                items:
+                                  description: 'LoadBalancerIngress represents the status of a load-balancer ingress point: traffic intended for the service should be sent to an ingress point.'
+                                  properties:
+                                    hostname:
+                                      description: Hostname is set for load-balancer ingress points that are DNS based (typically AWS load-balancers)
+                                      type: string
+                                    ip:
+                                      description: IP is set for load-balancer ingress points that are IP based (typically GCE or OpenStack load-balancers)
+                                      type: string
+                                    ports:
+                                      description: Ports is a list of records of service ports If used, every port defined in the service should have an entry in it
+                                      items:
+                                        properties:
+                                          error:
+                                            description: 'Error is to record the problem with the service port The format of the error shall comply with the following rules: - built-in error values shall be specified in this file and those shall use   CamelCase names - cloud provider specific error values must have names that comply with the   format foo.example.com/CamelCase. --- The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)'
+                                            maxLength: 316
+                                            pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
+                                            type: string
+                                          port:
+                                            description: Port is the port number of the service port of which status is recorded here
+                                            format: int32
+                                            type: integer
+                                          protocol:
+                                            default: TCP
+                                            description: 'Protocol is the protocol of the service port of which status is recorded here The supported values are: "TCP", "UDP", "SCTP"'
+                                            type: string
+                                        required:
+                                        - port
+                                        - protocol
+                                        type: object
+                                      type: array
+                                      x-kubernetes-list-type: atomic
+                                  type: object
+                                type: array
+                            type: object
+                          name:
+                            description: The name of the Service for the given cluster.
+                            type: string
+                          namespace:
+                            description: The namespace which the Service for the given cluster is deployed.
+                            type: string
+                          port:
+                            description: The port where the service is running
+                            format: int32
+                            type: integer
+                        required:
+                        - name
+                        - namespace
+                        - port
+                        type: object
+                    required:
+                    - enabled
+                    type: object
+                  kubeProxy:
+                    description: AddonStatus defines the observed state of an Addon.
+                    properties:
+                      checksum:
+                        type: string
+                      enabled:
+                        type: boolean
+                      lastUpdate:
+                        format: date-time
+                        type: string
+                    required:
+                    - enabled
+                    type: object
+                type: object
               certificates:
                 description: Certificates contains information about the different certificates that are necessary to run a kubernetes control plane
                 properties:
                   apiServer:
-                    description: CertificatePrivateKeyPair defines the status.
+                    description: CertificatePrivateKeyPairStatus defines the status.
                     properties:
+                      checksum:
+                        type: string
                       lastUpdate:
                         format: date-time
                         type: string
@@ -270,8 +624,10 @@ spec:
                         type: string
                     type: object
                   apiServerKubeletClient:
-                    description: CertificatePrivateKeyPair defines the status.
+                    description: CertificatePrivateKeyPairStatus defines the status.
                     properties:
+                      checksum:
+                        type: string
                       lastUpdate:
                         format: date-time
                         type: string
@@ -279,8 +635,10 @@ spec:
                         type: string
                     type: object
                   ca:
-                    description: CertificatePrivateKeyPair defines the status.
+                    description: CertificatePrivateKeyPairStatus defines the status.
                     properties:
+                      checksum:
+                        type: string
                       lastUpdate:
                         format: date-time
                         type: string
@@ -288,11 +646,13 @@ spec:
                         type: string
                     type: object
                   etcd:
-                    description: ETCDAPIServerCertificate defines the observed state of ETCD Certificate for API server.
+                    description: ETCDCertificatesStatus defines the observed state of ETCD Certificate for API server.
                     properties:
                       apiServer:
-                        description: ETCDAPIServerCertificate defines the observed state of ETCD Certificate for API server.
+                        description: APIServerCertificatesStatus defines the observed state of ETCD Certificate for API server.
                         properties:
+                          checksum:
+                            type: string
                           lastUpdate:
                             format: date-time
                             type: string
@@ -300,8 +660,10 @@ spec:
                             type: string
                         type: object
                       ca:
-                        description: ETCDAPIServerCertificate defines the observed state of ETCD Certificate for API server.
+                        description: ETCDCertificateStatus defines the observed state of ETCD Certificate for API server.
                         properties:
+                          checksum:
+                            type: string
                           lastUpdate:
                             format: date-time
                             type: string
@@ -310,8 +672,10 @@ spec:
                         type: object
                     type: object
                   frontProxyCA:
-                    description: CertificatePrivateKeyPair defines the status.
+                    description: CertificatePrivateKeyPairStatus defines the status.
                     properties:
+                      checksum:
+                        type: string
                       lastUpdate:
                         format: date-time
                         type: string
@@ -319,8 +683,10 @@ spec:
                         type: string
                     type: object
                   frontProxyClient:
-                    description: CertificatePrivateKeyPair defines the status.
+                    description: CertificatePrivateKeyPairStatus defines the status.
                     properties:
+                      checksum:
+                        type: string
                       lastUpdate:
                         format: date-time
                         type: string
@@ -328,8 +694,10 @@ spec:
                         type: string
                     type: object
                   sa:
-                    description: CertificatePrivateKeyPair defines the status.
+                    description: PublicKeyPrivateKeyPairStatus defines the status.
                     properties:
+                      checksum:
+                        type: string
                       lastUpdate:
                         format: date-time
                         type: string
@@ -343,54 +711,34 @@ spec:
               kubeadmPhase:
                 description: KubeadmPhase contains the status of the kubeadm phases action
                 properties:
-                  addonCoreDNS:
-                    description: KubeadmPhasesStatus contains the status of of a kubeadm phase action.
-                    properties:
-                      kubeadmConfigResourceVersion:
-                        type: string
-                      lastUpdate:
-                        format: date-time
-                        type: string
-                    type: object
-                  addonKubeProxy:
-                    description: KubeadmPhasesStatus contains the status of of a kubeadm phase action.
-                    properties:
-                      kubeadmConfigResourceVersion:
-                        type: string
-                      lastUpdate:
-                        format: date-time
-                        type: string
-                    type: object
                   bootstrapToken:
-                    description: KubeadmPhasesStatus contains the status of of a kubeadm phase action.
+                    description: KubeadmPhaseStatus contains the status of a kubeadm phase action.
                     properties:
-                      kubeadmConfigResourceVersion:
+                      checksum:
                         type: string
                       lastUpdate:
                         format: date-time
                         type: string
                     type: object
                   uploadConfigKubeadm:
-                    description: KubeadmPhasesStatus contains the status of of a kubeadm phase action.
+                    description: KubeadmPhaseStatus contains the status of a kubeadm phase action.
                     properties:
-                      kubeadmConfigResourceVersion:
+                      checksum:
                         type: string
                       lastUpdate:
                         format: date-time
                         type: string
                     type: object
                   uploadConfigKubelet:
-                    description: KubeadmPhasesStatus contains the status of of a kubeadm phase action.
+                    description: KubeadmPhaseStatus contains the status of a kubeadm phase action.
                     properties:
-                      kubeadmConfigResourceVersion:
+                      checksum:
                         type: string
                       lastUpdate:
                         format: date-time
                         type: string
                     type: object
                 required:
-                - addonCoreDNS
-                - addonKubeProxy
                 - bootstrapToken
                 - uploadConfigKubeadm
                 - uploadConfigKubelet
@@ -398,31 +746,34 @@ spec:
               kubeadmconfig:
                 description: KubeadmConfig contains the status of the configuration required by kubeadm
                 properties:
+                  checksum:
+                    description: Checksum of the kubeadm configuration to detect changes
+                    type: string
                   configmapName:
                     type: string
                   lastUpdate:
                     format: date-time
                     type: string
-                  resourceVersion:
-                    type: string
-                required:
-                - resourceVersion
                 type: object
               kubeconfig:
                 description: KubeConfig contains information about the kubenconfigs that control plane pieces need
                 properties:
                   admin:
-                    description: TenantControlPlaneKubeconfigsStatus contains information about a the generated kubeconfig.
+                    description: KubeconfigStatus contains information about the generated kubeconfig.
                     properties:
+                      checksum:
+                        type: string
                       lastUpdate:
                         format: date-time
                         type: string
                       secretName:
                         type: string
                     type: object
-                  controlerManager:
-                    description: TenantControlPlaneKubeconfigsStatus contains information about a the generated kubeconfig.
+                  controllerManager:
+                    description: KubeconfigStatus contains information about the generated kubeconfig.
                     properties:
+                      checksum:
+                        type: string
                       lastUpdate:
                         format: date-time
                         type: string
@@ -430,8 +781,10 @@ spec:
                         type: string
                     type: object
                   scheduler:
-                    description: TenantControlPlaneKubeconfigsStatus contains information about a the generated kubeconfig.
+                    description: KubeconfigStatus contains information about the generated kubeconfig.
                     properties:
+                      checksum:
+                        type: string
                       lastUpdate:
                         format: date-time
                         type: string
@@ -483,6 +836,10 @@ spec:
                           - type
                           type: object
                         type: array
+                      lastUpdate:
+                        description: Last time when deployment was updated
+                        format: date-time
+                        type: string
                       name:
                         description: The name of the Deployment for the given cluster.
                         type: string
@@ -686,51 +1043,41 @@ spec:
                       version:
                         description: Version is the running Kubernetes version of the Tenant Control Plane.
                         type: string
-                    required:
-                    - status
                     type: object
                 type: object
               storage:
                 description: Storage Status contains information about Kubernetes storage system
                 properties:
-                  etcd:
-                    description: ETCDStatus defines the observed state of ETCDStatus.
+                  certificate:
                     properties:
-                      role:
-                        properties:
-                          exists:
-                            type: boolean
-                          name:
-                            type: string
-                          permissions:
-                            items:
-                              properties:
-                                key:
-                                  type: string
-                                rangeEnd:
-                                  type: string
-                                type:
-                                  type: integer
-                              type: object
-                            type: array
-                        required:
-                        - exists
-                        - name
-                        type: object
+                      checksum:
+                        type: string
+                      lastUpdate:
+                        format: date-time
+                        type: string
+                      secretName:
+                        type: string
+                    type: object
+                  config:
+                    properties:
+                      checksum:
+                        type: string
+                      secretName:
+                        type: string
+                    type: object
+                  driver:
+                    type: string
+                  setup:
+                    properties:
+                      checksum:
+                        type: string
+                      lastUpdate:
+                        format: date-time
+                        type: string
+                      schema:
+                        type: string
                       user:
-                        properties:
-                          exists:
-                            type: boolean
-                          name:
-                            type: string
-                          roles:
-                            items:
-                              type: string
-                            type: array
-                        required:
-                        - exists
-                        - name
-                        type: object
+                        type: string
                     type: object
                 type: object
             type: object
@@ -844,6 +1191,26 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - kamaji.clastix.io
+  resources:
+  - datastores
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - kamaji.clastix.io
+  resources:
+  - datastores/status
+  verbs:
+  - get
+  - patch
+  - update
 - apiGroups:
   - kamaji.clastix.io
   resources:
@@ -1021,8 +1388,8 @@ spec:
         - --leader-elect
         command:
         - /manager
-        image: quay.io/clastix/kamaji:latest
-        imagePullPolicy: IfNotPresent
+        image: clastix/kamaji:latest
+        imagePullPolicy: Always
         livenessProbe:
           httpGet:
             path: /healthz
@@ -1049,3 +1416,39 @@ spec:
         runAsNonRoot: true
       serviceAccountName: kamaji-controller-manager
       terminationGracePeriodSeconds: 10
+---
+apiVersion: kamaji.clastix.io/v1alpha1
+kind: DataStore
+metadata:
+  name: kamaji-etcd
+  namespace: kamaji-system
+spec:
+  basicAuth: null
+  driver: etcd
+  endpoints:
+  - etcd-0.etcd.kamaji-system.svc:2379
+  - etcd-1.etcd.kamaji-system.svc:2379
+  - etcd-2.etcd.kamaji-system.svc:2379
+  tlsConfig:
+    certificateAuthority:
+      certificate:
+        secretReference:
+          keyPath: ca.crt
+          name: etcd-certs
+          namespace: kamaji-system
+      privateKey:
+        secretReference:
+          keyPath: ca.key
+          name: etcd-certs
+          namespace: kamaji-system
+    clientCertificate:
+      certificate:
+        secretReference:
+          keyPath: tls.crt
+          name: root-client-certs
+          namespace: kamaji-system
+      privateKey:
+        secretReference:
+          keyPath: tls.key
+          name: root-client-certs
+          namespace: kamaji-system

config/manager/kustomization.yaml

@@ -12,5 +12,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 images:
 - name: controller
-  newName: quay.io/clastix/kamaji
+  newName: clastix/kamaji
   newTag: latest

config/manager/manager.yaml

@@ -30,7 +30,7 @@ spec:
         args:
         - --leader-elect
         image: controller:latest
-        imagePullPolicy: IfNotPresent
+        imagePullPolicy: Always
         name: manager
         securityContext:
           allowPrivilegeEscalation: false

config/rbac/datastore_editor_role.yaml

@@ -0,0 +1,24 @@
+# permissions for end users to edit datastores.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: datastore-editor-role
+rules:
+- apiGroups:
+  - kamaji.clastix.io
+  resources:
+  - datastores
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - kamaji.clastix.io
+  resources:
+  - datastores/status
+  verbs:
+  - get

config/rbac/datastore_viewer_role.yaml

@@ -0,0 +1,20 @@
+# permissions for end users to view datastores.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: datastore-viewer-role
+rules:
+- apiGroups:
+  - kamaji.clastix.io
+  resources:
+  - datastores
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - kamaji.clastix.io
+  resources:
+  - datastores/status
+  verbs:
+  - get

config/rbac/role.yaml

@@ -54,6 +54,26 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - kamaji.clastix.io
+  resources:
+  - datastores
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - kamaji.clastix.io
+  resources:
+  - datastores/status
+  verbs:
+  - get
+  - patch
+  - update
 - apiGroups:
   - kamaji.clastix.io
   resources:

config/samples/kamaji_v1alpha1_datastore_etcd.yaml

@@ -0,0 +1,34 @@
+apiVersion: kamaji.clastix.io/v1alpha1
+kind: DataStore
+metadata:
+  name: etcd
+spec:
+  driver: etcd
+  endpoints:
+    - etcd-0.etcd.kamaji-system.svc:2379
+    - etcd-1.etcd.kamaji-system.svc:2379
+    - etcd-2.etcd.kamaji-system.svc:2379
+  basicAuth: null
+  tlsConfig:
+    certificateAuthority:
+      certificate:
+        secretReference:
+          name: etcd-certs
+          namespace: kamaji-system
+          keyPath: "ca.crt"
+      privateKey:
+        secretReference:
+          name: etcd-certs
+          namespace: kamaji-system
+          keyPath: "ca.key"
+    clientCertificate:
+      certificate:
+        secretReference:
+          name: root-client-certs
+          namespace: kamaji-system
+          keyPath: "tls.crt"
+      privateKey:
+        secretReference:
+          name: root-client-certs
+          namespace: kamaji-system
+          keyPath: "tls.key"

config/samples/kamaji_v1alpha1_datastore_mysql.yaml

@@ -0,0 +1,34 @@
+apiVersion: kamaji.clastix.io/v1alpha1
+kind: DataStore
+metadata:
+  name: mysql
+spec:
+  driver: MySQL
+  endpoints:
+    - mariadb.kamaji-system.svc:3306
+  basicAuth:
+    username:
+      content: cm9vdA==
+    password:
+      secretReference:
+        name: mysql-config
+        namespace: kamaji-system
+        keyPath: MYSQL_ROOT_PASSWORD
+  tlsConfig:
+    certificateAuthority:
+      certificate:
+        secretReference:
+          name: mysql-config
+          namespace: kamaji-system
+          keyPath: "ca.crt"
+    clientCertificate:
+      certificate:
+        secretReference:
+          name: mysql-config
+          namespace: kamaji-system
+          keyPath: "server.crt"
+      privateKey:
+        secretReference:
+          name: mysql-config
+          namespace: kamaji-system
+          keyPath: "server.key"

config/samples/kamaji_v1alpha1_datastore_postgresql.yaml

@@ -0,0 +1,37 @@
+apiVersion: kamaji.clastix.io/v1alpha1
+kind: DataStore
+metadata:
+  name: postgresql
+spec:
+  driver: PostgreSQL
+  endpoints:
+    - postgresql-rw.kamaji-system.svc:5432
+  basicAuth:
+    username:
+      secretReference:
+        name: postgresql-superuser
+        namespace: kamaji-system
+        keyPath: username
+    password:
+      secretReference:
+        name: postgresql-superuser
+        namespace: kamaji-system
+        keyPath: password
+  tlsConfig:
+    certificateAuthority:
+      certificate:
+        secretReference:
+          name: postgresql-ca
+          namespace: kamaji-system
+          keyPath: ca.crt
+    clientCertificate:
+      certificate:
+        secretReference:
+          name: postgres-root-cert
+          namespace: kamaji-system
+          keyPath: tls.crt
+      privateKey:
+        secretReference:
+          name: postgres-root-cert
+          namespace: kamaji-system
+          keyPath: tls.key

config/samples/kamaji_v1alpha1_tenantcontrolplane.yaml

@@ -5,44 +5,18 @@ metadata:
 spec:
   controlPlane:
     deployment:
-      replicas: 2
-      additionalMetadata:
-        annotations:
-          environment.clastix.io: test
-          tier.clastix.io: "0"
-        labels:
-          tenant.clastix.io: test
-          kind.clastix.io: deployment
+      replicas: 1
     service:
-      additionalMetadata:
-        annotations:
-          environment.clastix.io: test
-          tier.clastix.io: "0"
-        labels:
-          tenant.clastix.io: test
-          kind.clastix.io: service
-      serviceType: ClusterIP
-    ingress:
-      enabled: true
-      hostname: kamaji.local
-      ingressClassName: nginx
-      additionalMetadata:
-        annotations:
-          kubernetes.io/ingress.allow-http: "false"
-          nginx.ingress.kubernetes.io/secure-backends: "true"
-          nginx.ingress.kubernetes.io/ssl-passthrough: "true"      
+      serviceType: LoadBalancer
   kubernetes:
     version: "v1.23.1"
     kubelet:
-      cgroupfs: systemd
+      cgroupfs: cgroupfs
     admissionControllers:
       - ResourceQuota
       - LimitRanger
   networkProfile:
-    address: "127.0.0.1"
     port: 6443
-    domain: "clastix.labs"
-    serviceCidr: "10.96.0.0/16"
-    podCidr: "10.244.0.0/16"
-    dnsServiceIPs:
-      - "10.96.0.10"
+  addons:
+    coreDNS: {}
+    kubeProxy: {}

config/samples/kustomization.yaml

@@ -1,4 +1,4 @@
 ## Append samples you want in your CSV to this file as resources ##
 resources:
-- kamaji_v1alpha1_tenantcontrolplane.yaml
+- kamaji_v1alpha1_datastore_etcd.yaml
 #+kubebuilder:scaffold:manifestskustomizesamples

controllers/datastore_controller.go

@@ -0,0 +1,96 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package controllers
+
+import (
+	"context"
+
+	"github.com/pkg/errors"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/util/sets"
+	controllerruntime "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/builder"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/event"
+	"sigs.k8s.io/controller-runtime/pkg/predicate"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+)
+
+type DataStore struct {
+	client client.Client
+	// TenantControlPlaneTrigger is the channel used to communicate across the controllers:
+	// if a Data Source is updated we have to be sure that the reconciliation of the certificates content
+	// for each Tenant Control Plane is put in place properly.
+	TenantControlPlaneTrigger TenantControlPlaneChannel
+	// ResourceName is the DataStore object that should be watched for changes.
+	ResourceName string
+}
+
+//+kubebuilder:rbac:groups=kamaji.clastix.io,resources=datastores,verbs=get;list;watch;create;update;patch;delete
+//+kubebuilder:rbac:groups=kamaji.clastix.io,resources=datastores/status,verbs=get;update;patch
+
+func (r *DataStore) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) {
+	ds := kamajiv1alpha1.DataStore{}
+	if err := r.client.Get(ctx, request.NamespacedName, &ds); err != nil {
+		if k8serrors.IsNotFound(err) {
+			return reconcile.Result{}, nil
+		}
+
+		return reconcile.Result{}, err
+	}
+	// A Data Source can trigger several Tenant Control Planes and requires a minimum validation:
+	// we have to ensure the data provided by the Data Source is valid and referencing an existing Secret object.
+	if _, err := ds.Spec.TLSConfig.CertificateAuthority.Certificate.GetContent(ctx, r.client); err != nil {
+		return reconcile.Result{}, errors.Wrap(err, "invalid Certificate Authority data")
+	}
+
+	if _, err := ds.Spec.TLSConfig.ClientCertificate.Certificate.GetContent(ctx, r.client); err != nil {
+		return reconcile.Result{}, errors.Wrap(err, "invalid Client Certificate data")
+	}
+
+	if _, err := ds.Spec.TLSConfig.ClientCertificate.PrivateKey.GetContent(ctx, r.client); err != nil {
+		return reconcile.Result{}, errors.Wrap(err, "invalid Client Certificate data")
+	}
+
+	tcpList := kamajiv1alpha1.TenantControlPlaneList{}
+
+	if err := r.client.List(ctx, &tcpList); err != nil {
+		return reconcile.Result{}, err
+	}
+	// Updating the status with the list of Tenant Control Plane using the following Data Source
+	tcpSets := sets.NewString()
+	for _, tcp := range tcpList.Items {
+		tcpSets.Insert(getNamespacedName(tcp.GetNamespace(), tcp.GetName()).String())
+	}
+
+	ds.Status.UsedBy = tcpSets.List()
+
+	if err := r.client.Status().Update(ctx, &ds); err != nil {
+		return reconcile.Result{}, err
+	}
+	// Triggering the reconciliation of the Tenant Control Plane upon a Secret change
+	for _, i := range tcpList.Items {
+		tcp := i
+
+		r.TenantControlPlaneTrigger <- event.GenericEvent{Object: &tcp}
+	}
+
+	return reconcile.Result{}, nil
+}
+
+func (r *DataStore) InjectClient(client client.Client) error {
+	r.client = client
+
+	return nil
+}
+
+func (r *DataStore) SetupWithManager(mgr controllerruntime.Manager) error {
+	return controllerruntime.NewControllerManagedBy(mgr).
+		For(&kamajiv1alpha1.DataStore{}, builder.WithPredicates(predicate.NewPredicateFuncs(func(object client.Object) bool {
+			return object.GetName() == r.ResourceName
+		}))).
+		Complete(r)
+}

controllers/resources.go

@@ -0,0 +1,299 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package controllers
+
+import (
+	"fmt"
+
+	"github.com/go-logr/logr"
+	"github.com/google/uuid"
+	k8stypes "k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+	"github.com/clastix/kamaji/internal/datastore"
+	"github.com/clastix/kamaji/internal/resources"
+	ds "github.com/clastix/kamaji/internal/resources/datastore"
+	"github.com/clastix/kamaji/internal/resources/konnectivity"
+)
+
+type GroupResourceBuilderConfiguration struct {
+	client              client.Client
+	log                 logr.Logger
+	tcpReconcilerConfig TenantControlPlaneReconcilerConfig
+	tenantControlPlane  kamajiv1alpha1.TenantControlPlane
+	Connection          datastore.Connection
+	DataStore           kamajiv1alpha1.DataStore
+}
+
+type GroupDeleteableResourceBuilderConfiguration struct {
+	client              client.Client
+	log                 logr.Logger
+	tcpReconcilerConfig TenantControlPlaneReconcilerConfig
+	tenantControlPlane  kamajiv1alpha1.TenantControlPlane
+	connection          datastore.Connection
+}
+
+// GetResources returns a list of resources that will be used to provide tenant control planes
+// Currently there is only a default approach
+// TODO: the idea of this function is to become a factory to return the group of resources according to the given configuration.
+func GetResources(config GroupResourceBuilderConfiguration) []resources.Resource {
+	return getDefaultResources(config)
+}
+
+// GetDeletableResources returns a list of resources that have to be deleted when tenant control planes are deleted
+// Currently there is only a default approach
+// TODO: the idea of this function is to become a factory to return the group of deleteable resources according to the given configuration.
+func GetDeletableResources(config GroupDeleteableResourceBuilderConfiguration) []resources.DeleteableResource {
+	return getDefaultDeleteableResources(config)
+}
+
+func getDefaultResources(config GroupResourceBuilderConfiguration) []resources.Resource {
+	resources := append(getUpgradeResources(config.client), getKubernetesServiceResources(config.client)...)
+	resources = append(resources, getKubeadmConfigResources(config.client, getTmpDirectory(config.tcpReconcilerConfig.TmpBaseDirectory, config.tenantControlPlane), config.DataStore)...)
+	resources = append(resources, getKubernetesCertificatesResources(config.client, config.log, config.tcpReconcilerConfig, config.tenantControlPlane)...)
+	resources = append(resources, getKubeconfigResources(config.client, config.log, config.tcpReconcilerConfig, config.tenantControlPlane)...)
+	resources = append(resources, getKubernetesStorageResources(config.client, config.Connection, config.DataStore)...)
+	resources = append(resources, getInternalKonnectivityResources(config.client, config.log)...)
+	resources = append(resources, getKubernetesDeploymentResources(config.client, config.tcpReconcilerConfig, config.DataStore)...)
+	resources = append(resources, getKubernetesIngressResources(config.client)...)
+	resources = append(resources, getKubeadmPhaseResources(config.client, config.log)...)
+	resources = append(resources, getKubeadmAddonResources(config.client, config.log)...)
+	resources = append(resources, getExternalKonnectivityResources(config.client)...)
+
+	return resources
+}
+
+func getDefaultDeleteableResources(config GroupDeleteableResourceBuilderConfiguration) []resources.DeleteableResource {
+	return []resources.DeleteableResource{
+		&ds.Setup{
+			Client:     config.client,
+			Connection: config.connection,
+		},
+	}
+}
+
+func getUpgradeResources(c client.Client) []resources.Resource {
+	return []resources.Resource{
+		&resources.KubernetesUpgrade{
+			Client: c,
+		},
+	}
+}
+
+func getKubernetesServiceResources(c client.Client) []resources.Resource {
+	return []resources.Resource{
+		&resources.KubernetesServiceResource{
+			Client: c,
+		},
+	}
+}
+
+func getKubeadmConfigResources(c client.Client, tmpDirectory string, dataStore kamajiv1alpha1.DataStore) []resources.Resource {
+	var endpoints []string
+
+	switch dataStore.Spec.Driver {
+	case kamajiv1alpha1.EtcdDriver:
+		endpoints = dataStore.Spec.Endpoints
+	default:
+		endpoints = []string{"127.0.0.1:2379"}
+	}
+
+	return []resources.Resource{
+		&resources.KubeadmConfigResource{
+			ETCDs:        endpoints,
+			Client:       c,
+			TmpDirectory: tmpDirectory,
+		},
+	}
+}
+
+func getKubernetesCertificatesResources(c client.Client, log logr.Logger, tcpReconcilerConfig TenantControlPlaneReconcilerConfig, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource {
+	return []resources.Resource{
+		&resources.CACertificate{
+			Client:       c,
+			Log:          log,
+			TmpDirectory: getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
+		},
+		&resources.FrontProxyCACertificate{
+			Client:       c,
+			Log:          log,
+			TmpDirectory: getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
+		},
+		&resources.SACertificate{
+			Client:       c,
+			Log:          log,
+			TmpDirectory: getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
+		},
+		&resources.APIServerCertificate{
+			Client:       c,
+			Log:          log,
+			TmpDirectory: getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
+		},
+		&resources.APIServerKubeletClientCertificate{
+			Client:       c,
+			Log:          log,
+			TmpDirectory: getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
+		},
+		&resources.FrontProxyClientCertificate{
+			Client:       c,
+			Log:          log,
+			TmpDirectory: getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
+		},
+	}
+}
+
+func getKubeconfigResources(c client.Client, log logr.Logger, tcpReconcilerConfig TenantControlPlaneReconcilerConfig, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource {
+	return []resources.Resource{
+		&resources.KubeconfigResource{
+			Name:               "admin-kubeconfig",
+			Client:             c,
+			Log:                log,
+			KubeConfigFileName: resources.AdminKubeConfigFileName,
+			TmpDirectory:       getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
+		},
+		&resources.KubeconfigResource{
+			Name:               "controller-manager-kubeconfig",
+			Client:             c,
+			Log:                log,
+			KubeConfigFileName: resources.ControllerManagerKubeConfigFileName,
+			TmpDirectory:       getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
+		},
+		&resources.KubeconfigResource{
+			Name:               "scheduler-kubeconfig",
+			Client:             c,
+			Log:                log,
+			KubeConfigFileName: resources.SchedulerKubeConfigFileName,
+			TmpDirectory:       getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
+		},
+	}
+}
+
+func getKubernetesStorageResources(c client.Client, dbConnection datastore.Connection, datastore kamajiv1alpha1.DataStore) []resources.Resource {
+	return []resources.Resource{
+		&ds.Config{
+			Client:     c,
+			ConnString: dbConnection.GetConnectionString(),
+			Driver:     dbConnection.Driver(),
+		},
+		&ds.Setup{
+			Client:     c,
+			Connection: dbConnection,
+			DataStore:  datastore,
+		},
+		&ds.Certificate{
+			Client:    c,
+			DataStore: datastore,
+		},
+	}
+}
+
+func getKubernetesDeploymentResources(c client.Client, tcpReconcilerConfig TenantControlPlaneReconcilerConfig, dataStore kamajiv1alpha1.DataStore) []resources.Resource {
+	return []resources.Resource{
+		&resources.KubernetesDeploymentResource{
+			Client:             c,
+			DataStore:          dataStore,
+			KineContainerImage: tcpReconcilerConfig.KineContainerImage,
+		},
+	}
+}
+
+func getKubernetesIngressResources(c client.Client) []resources.Resource {
+	return []resources.Resource{
+		&resources.KubernetesIngressResource{
+			Client: c,
+		},
+	}
+}
+
+func getKubeadmPhaseResources(c client.Client, log logr.Logger) []resources.Resource {
+	return []resources.Resource{
+		&resources.KubeadmPhase{
+			Name:   "upload-config-kubeadm",
+			Client: c,
+			Log:    log,
+			Phase:  resources.PhaseUploadConfigKubeadm,
+		},
+		&resources.KubeadmPhase{
+			Name:   "upload-config-kubelet",
+			Client: c,
+			Log:    log,
+			Phase:  resources.PhaseUploadConfigKubelet,
+		},
+		&resources.KubeadmPhase{
+			Name:   "bootstrap-token",
+			Client: c,
+			Log:    log,
+			Phase:  resources.PhaseBootstrapToken,
+		},
+	}
+}
+
+func getKubeadmAddonResources(c client.Client, log logr.Logger) []resources.Resource {
+	return []resources.Resource{
+		&resources.KubeadmAddonResource{
+			Name:         "coredns",
+			Client:       c,
+			Log:          log,
+			KubeadmAddon: resources.AddonCoreDNS,
+		},
+		&resources.KubeadmAddonResource{
+			Name:         "kubeproxy",
+			Client:       c,
+			Log:          log,
+			KubeadmAddon: resources.AddonKubeProxy,
+		},
+	}
+}
+
+func getExternalKonnectivityResources(c client.Client) []resources.Resource {
+	return []resources.Resource{
+		&konnectivity.ServiceAccountResource{
+			Client: c,
+			Name:   "konnectivity-sa",
+		},
+		&konnectivity.ClusterRoleBindingResource{
+			Client: c,
+			Name:   "konnectivity-clusterrolebinding",
+		},
+		&konnectivity.KubernetesDeploymentResource{
+			Client: c,
+			Name:   "konnectivity-deployment",
+		},
+		&konnectivity.ServiceResource{
+			Client: c,
+			Name:   "konnectivity-service",
+		},
+		&konnectivity.Agent{
+			Client: c,
+			Name:   "konnectivity-agent",
+		},
+	}
+}
+
+func getInternalKonnectivityResources(c client.Client, log logr.Logger) []resources.Resource {
+	return []resources.Resource{
+		&konnectivity.EgressSelectorConfigurationResource{
+			Client: c,
+			Name:   "konnectivity-egress-selector-configuration",
+		},
+		&konnectivity.CertificateResource{
+			Client: c,
+			Log:    log,
+			Name:   "konnectivity-certificate",
+		},
+		&konnectivity.KubeconfigResource{
+			Client: c,
+			Name:   "konnectivity-kubeconfig",
+		},
+	}
+}
+
+func getNamespacedName(namespace string, name string) k8stypes.NamespacedName {
+	return k8stypes.NamespacedName{Namespace: namespace, Name: name}
+}
+
+func getTmpDirectory(base string, tenantControlPlane kamajiv1alpha1.TenantControlPlane) string {
+	return fmt.Sprintf("%s/%s/%s", base, tenantControlPlane.GetName(), uuid.New())
+}

controllers/storage.go

@@ -0,0 +1,104 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package controllers
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"fmt"
+	"net"
+	"strconv"
+
+	"github.com/pkg/errors"
+
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+	"github.com/clastix/kamaji/internal/datastore"
+)
+
+func (r *TenantControlPlaneReconciler) getStorageConnection(ctx context.Context, ds kamajiv1alpha1.DataStore) (datastore.Connection, error) {
+	ca, err := ds.Spec.TLSConfig.CertificateAuthority.Certificate.GetContent(ctx, r.Client)
+	if err != nil {
+		return nil, err
+	}
+
+	crt, err := ds.Spec.TLSConfig.ClientCertificate.Certificate.GetContent(ctx, r.Client)
+	if err != nil {
+		return nil, err
+	}
+
+	key, err := ds.Spec.TLSConfig.ClientCertificate.PrivateKey.GetContent(ctx, r.Client)
+	if err != nil {
+		return nil, err
+	}
+
+	rootCAs := x509.NewCertPool()
+	if ok := rootCAs.AppendCertsFromPEM(ca); !ok {
+		return nil, fmt.Errorf("error create root CA for the DB connector")
+	}
+
+	certificate, err := tls.X509KeyPair(crt, key)
+	if err != nil {
+		return nil, errors.Wrap(err, "cannot retrieve x.509 key pair from the Kine Secret")
+	}
+
+	var user, password string
+	if auth := ds.Spec.BasicAuth; auth != nil {
+		u, err := auth.Username.GetContent(ctx, r.Client)
+		if err != nil {
+			return nil, err
+		}
+		user = string(u)
+
+		p, err := auth.Password.GetContent(ctx, r.Client)
+		if err != nil {
+			return nil, err
+		}
+		password = string(p)
+	}
+
+	eps := make([]datastore.ConnectionEndpoint, 0, len(ds.Spec.Endpoints))
+
+	for _, ep := range ds.Spec.Endpoints {
+		host, stringPort, err := net.SplitHostPort(ep)
+		if err != nil {
+			return nil, errors.Wrap(err, "cannot retrieve host-port pair from DataStore endpoints")
+		}
+
+		port, err := strconv.Atoi(stringPort)
+		if err != nil {
+			return nil, errors.Wrap(err, "cannot convert port from string for the given DataStore")
+		}
+
+		eps = append(eps, datastore.ConnectionEndpoint{
+			Host: host,
+			Port: port,
+		})
+	}
+
+	cc := datastore.ConnectionConfig{
+		User:      user,
+		Password:  password,
+		Endpoints: eps,
+		TLSConfig: &tls.Config{
+			RootCAs:      rootCAs,
+			Certificates: []tls.Certificate{certificate},
+		},
+	}
+
+	switch ds.Spec.Driver {
+	case kamajiv1alpha1.KineMySQLDriver:
+		cc.TLSConfig.ServerName = cc.Endpoints[0].Host
+
+		return datastore.NewMySQLConnection(cc)
+	case kamajiv1alpha1.KinePostgreSQLDriver:
+		cc.TLSConfig.ServerName = cc.Endpoints[0].Host
+		//nolint:contextcheck
+		return datastore.NewPostgreSQLConnection(cc)
+	case kamajiv1alpha1.EtcdDriver:
+		return datastore.NewETCDConnection(cc)
+	default:
+		return nil, fmt.Errorf("%s is not a valid driver", ds.Spec.Driver)
+	}
+}

controllers/tcp_channel.go

@@ -0,0 +1,8 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package controllers
+
+import "sigs.k8s.io/controller-runtime/pkg/event"
+
+type TenantControlPlaneChannel chan event.GenericEvent

controllers/tenantcontrolplane_controller.go

@@ -6,55 +6,55 @@ package controllers
 import (
 	"context"
 	"fmt"
-	"strings"
 
-	"github.com/google/uuid"
+	"github.com/pkg/errors"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	networkingv1 "k8s.io/api/networking/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime"
 	k8stypes "k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/util/workqueue"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+	"sigs.k8s.io/controller-runtime/pkg/event"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
 	"sigs.k8s.io/controller-runtime/pkg/log"
+	"sigs.k8s.io/controller-runtime/pkg/source"
 
 	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+	kamajierrors "github.com/clastix/kamaji/internal/errors"
 	"github.com/clastix/kamaji/internal/resources"
 )
 
 const (
-	separator = ","
 	finalizer = "finalizer.kamaji.clastix.io"
 )
 
 // TenantControlPlaneReconciler reconciles a TenantControlPlane object.
 type TenantControlPlaneReconciler struct {
 	client.Client
-	Scheme *runtime.Scheme
-	Config TenantControlPlaneReconcilerConfig
+	Scheme      *runtime.Scheme
+	Config      TenantControlPlaneReconcilerConfig
+	TriggerChan TenantControlPlaneChannel
 }
 
 // TenantControlPlaneReconcilerConfig gives the necessary configuration for TenantControlPlaneReconciler.
 type TenantControlPlaneReconcilerConfig struct {
-	ETCDCASecretName          string
-	ETCDCASecretNamespace     string
-	ETCDClientSecretName      string
-	ETCDClientSecretNamespace string
-	ETCDEndpoints             string
-	ETCDCompactionInterval    string
-	TmpBaseDirectory          string
+	DataStoreName      string
+	KineContainerImage string
+	TmpBaseDirectory   string
 }
 
 //+kubebuilder:rbac:groups=kamaji.clastix.io,resources=tenantcontrolplanes,verbs=get;list;watch;create;update;patch;delete
 //+kubebuilder:rbac:groups=kamaji.clastix.io,resources=tenantcontrolplanes/status,verbs=get;update;patch
 //+kubebuilder:rbac:groups=kamaji.clastix.io,resources=tenantcontrolplanes/finalizers,verbs=update
-// +kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch;create;update;patch;delete
-// +kubebuilder:rbac:groups=core,resources=configmaps,verbs=get;list;watch;create;update;patch;delete
-// +kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;update;patch;delete
-// +kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete
-// +kubebuilder:rbac:groups=networking.k8s.io,resources=ingresses,verbs=get;list;watch;create;update;patch;delete
+//+kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch;create;update;patch;delete
+//+kubebuilder:rbac:groups=core,resources=configmaps,verbs=get;list;watch;create;update;patch;delete
+//+kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;update;patch;delete
+//+kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete
+//+kubebuilder:rbac:groups=networking.k8s.io,resources=ingresses,verbs=get;list;watch;create;update;patch;delete
 
 func (r *TenantControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
 	log := log.FromContext(ctx)
@@ -75,190 +75,71 @@ func (r *TenantControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R
 		return ctrl.Result{}, nil
 	}
 
-	if markedToBeDeleted {
-		registeredDeleteableResources := []resources.DeleteableResource{
-			&resources.ETCDSetupResource{
-				Name:                  "etcd-setup",
-				Client:                r.Client,
-				Scheme:                r.Scheme,
-				Log:                   log,
-				ETCDClientCertsSecret: getNamespacedName(r.Config.ETCDClientSecretNamespace, r.Config.ETCDClientSecretName),
-				ETCDCACertsSecret:     getNamespacedName(r.Config.ETCDCASecretNamespace, r.Config.ETCDCASecretName),
-				Endpoints:             getArrayFromString(r.Config.ETCDEndpoints),
-			},
-		}
+	ds := kamajiv1alpha1.DataStore{}
+	if err = r.Client.Get(ctx, k8stypes.NamespacedName{Name: r.Config.DataStoreName}, &ds); err != nil {
+		return ctrl.Result{}, errors.Wrap(err, "cannot retrieve kamajiv1alpha.DataStore object")
+	}
 
-		for _, resource := range registeredDeleteableResources {
-			if err := resource.Delete(ctx, tenantControlPlane); err != nil {
+	dsConnection, err := r.getStorageConnection(ctx, ds)
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+	defer dsConnection.Close()
+
+	if markedToBeDeleted {
+		log.Info("marked for deletion, performing clean-up")
+
+		groupDeleteableResourceBuilderConfiguration := GroupDeleteableResourceBuilderConfiguration{
+			client:              r.Client,
+			log:                 log,
+			tcpReconcilerConfig: r.Config,
+			tenantControlPlane:  *tenantControlPlane,
+			connection:          dsConnection,
+		}
+		registeredDeletableResources := GetDeletableResources(groupDeleteableResourceBuilderConfiguration)
+
+		for _, resource := range registeredDeletableResources {
+			if err = resources.HandleDeletion(ctx, resource, tenantControlPlane); err != nil {
 				return ctrl.Result{}, err
 			}
 		}
 
 		if hasFinalizer {
-			controllerutil.RemoveFinalizer(tenantControlPlane, finalizer)
-			if err := r.Update(ctx, tenantControlPlane); err != nil {
+			log.Info("removing finalizer")
+
+			if err = r.RemoveFinalizer(ctx, tenantControlPlane); err != nil {
 				return ctrl.Result{}, err
 			}
 		}
 
+		log.Info("resource deletion has been completed")
+
 		return ctrl.Result{}, nil
 	}
 
 	if !hasFinalizer {
-		controllerutil.AddFinalizer(tenantControlPlane, finalizer)
-		if err := r.Update(ctx, tenantControlPlane); err != nil {
-			return ctrl.Result{}, err
-		}
-
-		return ctrl.Result{}, nil
+		return ctrl.Result{}, r.AddFinalizer(ctx, tenantControlPlane)
 	}
 
-	registeredResources := []resources.Resource{
-		&resources.KubernetesServiceResource{
-			Client: r.Client,
-		},
-		&resources.KubeadmConfigResource{
-			Name:                   "kubeadmconfig",
-			Port:                   tenantControlPlane.Spec.NetworkProfile.Port,
-			KubernetesVersion:      tenantControlPlane.Spec.Kubernetes.Version,
-			PodCIDR:                tenantControlPlane.Spec.NetworkProfile.PodCIDR,
-			ServiceCIDR:            tenantControlPlane.Spec.NetworkProfile.ServiceCIDR,
-			Domain:                 tenantControlPlane.Spec.NetworkProfile.Domain,
-			ETCDs:                  getArrayFromString(r.Config.ETCDEndpoints),
-			ETCDCompactionInterval: r.Config.ETCDCompactionInterval,
-			Client:                 r.Client,
-			Scheme:                 r.Scheme,
-			Log:                    log,
-			TmpDirectory:           getTmpDirectory(r.Config.TmpBaseDirectory, *tenantControlPlane),
-		},
-		&resources.CACertificate{
-			Name:         "ca",
-			Client:       r.Client,
-			Log:          log,
-			TmpDirectory: getTmpDirectory(r.Config.TmpBaseDirectory, *tenantControlPlane),
-		},
-		&resources.FrontProxyCACertificate{
-			Name:         "front-proxy-ca-certificate",
-			Client:       r.Client,
-			Log:          log,
-			TmpDirectory: getTmpDirectory(r.Config.TmpBaseDirectory, *tenantControlPlane),
-		},
-		&resources.SACertificate{
-			Name:         "sa-certificate",
-			Client:       r.Client,
-			Log:          log,
-			TmpDirectory: getTmpDirectory(r.Config.TmpBaseDirectory, *tenantControlPlane),
-		},
-		&resources.APIServerCertificate{
-			Name:         "api-server-certificate",
-			Client:       r.Client,
-			Log:          log,
-			TmpDirectory: getTmpDirectory(r.Config.TmpBaseDirectory, *tenantControlPlane),
-		},
-		&resources.APIServerKubeletClientCertificate{
-			Name:         "api-server-kubelet-client-certificate",
-			Client:       r.Client,
-			Log:          log,
-			TmpDirectory: getTmpDirectory(r.Config.TmpBaseDirectory, *tenantControlPlane),
-		},
-		&resources.FrontProxyClientCertificate{
-			Name:         "front-proxy-client-certificate",
-			Client:       r.Client,
-			Log:          log,
-			TmpDirectory: getTmpDirectory(r.Config.TmpBaseDirectory, *tenantControlPlane),
-		},
-		&resources.KubeconfigResource{
-			Name:               "admin-kubeconfig",
-			Client:             r.Client,
-			Scheme:             r.Scheme,
-			Log:                log,
-			KubeConfigFileName: resources.AdminKubeConfigFileName,
-			TmpDirectory:       getTmpDirectory(r.Config.TmpBaseDirectory, *tenantControlPlane),
-		},
-		&resources.KubeconfigResource{
-			Name:               "controller-manager-kubeconfig",
-			Client:             r.Client,
-			Scheme:             r.Scheme,
-			Log:                log,
-			KubeConfigFileName: resources.ControllerManagerKubeConfigFileName,
-			TmpDirectory:       getTmpDirectory(r.Config.TmpBaseDirectory, *tenantControlPlane),
-		},
-		&resources.KubeconfigResource{
-			Name:               "scheduler-kubeconfig",
-			Client:             r.Client,
-			Scheme:             r.Scheme,
-			Log:                log,
-			KubeConfigFileName: resources.SchedulerKubeConfigFileName,
-			TmpDirectory:       getTmpDirectory(r.Config.TmpBaseDirectory, *tenantControlPlane),
-		},
-		&resources.ETCDCACertificatesResource{
-			Name:                  "etcd-ca-certificates",
-			Client:                r.Client,
-			Log:                   log,
-			ETCDCASecretName:      r.Config.ETCDCASecretName,
-			ETCDCASecretNamespace: r.Config.ETCDCASecretNamespace,
-		},
-		&resources.ETCDCertificatesResource{
-			Name:   "etcd-certificates",
-			Client: r.Client,
-			Log:    log,
-		},
-		&resources.ETCDSetupResource{
-			Name:                  "etcd-setup",
-			Client:                r.Client,
-			Scheme:                r.Scheme,
-			Log:                   log,
-			ETCDClientCertsSecret: getNamespacedName(r.Config.ETCDClientSecretNamespace, r.Config.ETCDClientSecretName),
-			ETCDCACertsSecret:     getNamespacedName(r.Config.ETCDCASecretNamespace, r.Config.ETCDCASecretName),
-			Endpoints:             getArrayFromString(r.Config.ETCDEndpoints),
-		},
-		&resources.KubernetesUpgrade{
-			Name:   "upgrade",
-			Client: r.Client,
-		},
-		&resources.KubernetesDeploymentResource{
-			Client:                 r.Client,
-			ETCDEndpoints:          getArrayFromString(r.Config.ETCDEndpoints),
-			ETCDCompactionInterval: r.Config.ETCDCompactionInterval,
-		},
-		&resources.KubernetesIngressResource{
-			Client: r.Client,
-		},
-		&resources.KubeadmPhaseResource{
-			Name:         "upload-config-kubeadm",
-			Client:       r.Client,
-			Log:          log,
-			KubeadmPhase: resources.PhaseUploadConfigKubeadm,
-		},
-		&resources.KubeadmPhaseResource{
-			Name:         "upload-config-kubelet",
-			Client:       r.Client,
-			Log:          log,
-			KubeadmPhase: resources.PhaseUploadConfigKubelet,
-		},
-		&resources.KubeadmPhaseResource{
-			Name:         "addon-coredns",
-			Client:       r.Client,
-			Log:          log,
-			KubeadmPhase: resources.PhaseAddonCoreDNS,
-		},
-		&resources.KubeadmPhaseResource{
-			Name:         "addon-kubeproxy",
-			Client:       r.Client,
-			Log:          log,
-			KubeadmPhase: resources.PhaseAddonKubeProxy,
-		},
-		&resources.KubeadmPhaseResource{
-			Name:         "bootstrap-token",
-			Client:       r.Client,
-			Log:          log,
-			KubeadmPhase: resources.PhaseBootstrapToken,
-		},
+	groupResourceBuilderConfiguration := GroupResourceBuilderConfiguration{
+		client:              r.Client,
+		log:                 log,
+		tcpReconcilerConfig: r.Config,
+		tenantControlPlane:  *tenantControlPlane,
+		DataStore:           ds,
+		Connection:          dsConnection,
 	}
+	registeredResources := GetResources(groupResourceBuilderConfiguration)
 
 	for _, resource := range registeredResources {
 		result, err := resources.Handle(ctx, resource, tenantControlPlane)
 		if err != nil {
+			if kamajierrors.ShouldReconcileErrorBeIgnored(err) {
+				log.V(1).Info("sentinel error, enqueuing back request", "error", err.Error())
+
+				return ctrl.Result{Requeue: true}, nil
+			}
+
 			return ctrl.Result{}, err
 		}
 
@@ -275,12 +156,22 @@ func (r *TenantControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R
 		return ctrl.Result{}, nil
 	}
 
+	log.Info(fmt.Sprintf("%s has been reconciled", tenantControlPlane.GetName()))
+
 	return ctrl.Result{}, nil
 }
 
 // SetupWithManager sets up the controller with the Manager.
 func (r *TenantControlPlaneReconciler) SetupWithManager(mgr ctrl.Manager) error {
 	return ctrl.NewControllerManagedBy(mgr).
+		Watches(&source.Channel{Source: r.TriggerChan}, handler.Funcs{GenericFunc: func(genericEvent event.GenericEvent, limitingInterface workqueue.RateLimitingInterface) {
+			limitingInterface.AddRateLimited(ctrl.Request{
+				NamespacedName: k8stypes.NamespacedName{
+					Namespace: genericEvent.Object.GetNamespace(),
+					Name:      genericEvent.Object.GetName(),
+				},
+			})
+		}}).
 		For(&kamajiv1alpha1.TenantControlPlane{}).
 		Owns(&corev1.Secret{}).
 		Owns(&corev1.ConfigMap{}).
@@ -324,21 +215,6 @@ func (r *TenantControlPlaneReconciler) updateStatus(ctx context.Context, namespa
 	return nil
 }
 
-func getArrayFromString(s string) []string {
-	var a []string
-	a = append(a, strings.Split(s, separator)...)
-
-	return a
-}
-
-func getNamespacedName(namespace string, name string) k8stypes.NamespacedName {
-	return k8stypes.NamespacedName{Namespace: namespace, Name: name}
-}
-
-func getTmpDirectory(base string, tenantControlPlane kamajiv1alpha1.TenantControlPlane) string {
-	return fmt.Sprintf("%s/%s/%s", base, tenantControlPlane.GetName(), uuid.New())
-}
-
 func hasFinalizer(tenantControlPlane kamajiv1alpha1.TenantControlPlane) bool {
 	for _, f := range tenantControlPlane.GetFinalizers() {
 		if f == finalizer {
@@ -348,3 +224,15 @@ func hasFinalizer(tenantControlPlane kamajiv1alpha1.TenantControlPlane) bool {
 
 	return false
 }
+
+func (r *TenantControlPlaneReconciler) AddFinalizer(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+	controllerutil.AddFinalizer(tenantControlPlane, finalizer)
+
+	return r.Update(ctx, tenantControlPlane)
+}
+
+func (r *TenantControlPlaneReconciler) RemoveFinalizer(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+	controllerutil.RemoveFinalizer(tenantControlPlane, finalizer)
+
+	return r.Update(ctx, tenantControlPlane)
+}

deploy/Makefile

@@ -0,0 +1,16 @@
+include etcd/Makefile
+
+deploy_path := $(patsubst %/,%,$(dir $(abspath $(lastword $(MAKEFILE_LIST)))))
+
+.DEFAULT_GOAL := kamaji
+
+.PHONY: etcd-cluster
+reqs: etcd-cluster
+
+.PHONY: kamaji
+kamaji: reqs
+	@kubectl apply -f $(deploy_path)/../../config/install.yaml
+
+.PHONY: destroy
+destroy: etcd-certificates/cleanup
+	@kubectl delete -f $(deploy_path)/../../config/install.yaml

deploy/README.md

@@ -0,0 +1,26 @@
+# Deploy Kamaji
+
+## Quickstart with KinD
+
+```sh
+make -C kind
+```
+
+## Multi-tenant etcd cluster
+
+> This assumes you already have a running Kubernetes cluster and kubeconfig.
+
+```sh
+make -C etcd
+```
+
+## Multi-tenant cluster using Kine
+
+`kine` is an `etcd` shim that allows using different datastore.
+
+Kamaji actually support the following backends:
+
+- [MySQL](kine/mysql/README.md)
+- [PostgreSQL](kine/postgresql/README.md)
+
+> This assumes you already have a running Kubernetes cluster and kubeconfig.

deploy/etcd/etcd-cluster.yaml

@@ -54,6 +54,13 @@ spec:
         app: etcd
     spec:
       serviceAccountName: etcd
+      topologySpreadConstraints:
+      - maxSkew: 1
+        topologyKey: topology.kubernetes.io/zone
+        whenUnsatisfiable: DoNotSchedule
+        labelSelector:
+          matchLabels:
+            app: etcd
       volumes:
       - name: certs
         secret:
@@ -90,9 +97,10 @@ spec:
           - --peer-trusted-ca-file=/etc/etcd/pki/ca.crt
           - --peer-cert-file=/etc/etcd/pki/peer.pem 
           - --peer-key-file=/etc/etcd/pki/peer-key.pem
+          - --snapshot-count=8000
           - --auto-compaction-mode=periodic
           - --auto-compaction-retention=5m
-          - --v=8
+          - --quota-backend-bytes=8589934592
           env:
           - name: POD_NAME
             valueFrom:
@@ -109,5 +117,4 @@ spec:
       accessModes: ["ReadWriteOnce"]
       resources:
         requests:
-          storage: 2Gi
-
+          storage: 8Gi

deploy/kamaji-azure.env

@@ -1,5 +1,30 @@
+# azure parameters
 export KAMAJI_REGION=westeurope
 export KAMAJI_RG=Kamaji
 # https://docs.microsoft.com/en-us/azure/aks/faq#why-are-two-resource-groups-created-with-aks
 export KAMAJI_CLUSTER=kamaji
 export KAMAJI_NODE_RG=MC_${KAMAJI_RG}_${KAMAJI_CLUSTER}_${KAMAJI_REGION}
+
+# kamaji parameters
+export KAMAJI_NAMESPACE=kamaji-system
+
+# tenant cluster parameters
+export TENANT_NAMESPACE=tenants
+export TENANT_NAME=tenant-00
+export TENANT_DOMAIN=$KAMAJI_REGION.cloudapp.azure.com
+export TENANT_VERSION=v1.23.5
+export TENANT_PORT=6443 # port used to expose the tenant api server
+export TENANT_PROXY_PORT=8132 # port used to expose the konnectivity server
+export TENANT_POD_CIDR=10.36.0.0/16
+export TENANT_SVC_CIDR=10.96.0.0/16
+export TENANT_DNS_SERVICE=10.96.0.10
+
+export TENANT_VM_SIZE=Standard_D2ds_v4
+export TENANT_VM_IMAGE=UbuntuLTS
+export TENANT_RG=$TENANT_NAME
+export TENANT_NSG=$TENANT_NAME-nsg
+export TENANT_VNET_NAME=$TENANT_NAME
+export TENANT_VNET_ADDRESS=172.12.0.0/16
+export TENANT_SUBNET_NAME=$TENANT_NAME-subnet
+export TENANT_SUBNET_ADDRESS=172.12.10.0/24
+export TENANT_VMSS=$TENANT_NAME-vmss

deploy/kamaji-external-etcd.env

@@ -1,5 +0,0 @@
-# etcd machine addresses
-export ETCD0=192.168.32.10
-export ETCD1=192.168.32.11
-export ETCD2=192.168.32.12
-export ETCDHOSTS=($ETCD0 $ETCD1 $ETCD2)

deploy/kamaji-internal-etcd.env

@@ -1,5 +0,0 @@
-# etcd endpoints
-export ETCD_NAMESPACE=etcd-system
-export ETCD0=etcd-0.etcd.${ETCD_NAMESPACE}.svc.cluster.local
-export ETCD1=etcd-1.etcd.${ETCD_NAMESPACE}.svc.cluster.local
-export ETCD2=etcd-2.etcd.${ETCD_NAMESPACE}.svc.cluster.local

deploy/kamaji-tenant-azure.env

@@ -1,22 +0,0 @@
-export KAMAJI_REGION=westeurope
-
-# tenant cluster parameters
-export TENANT_NAMESPACE=tenants
-export TENANT_NAME=tenant-00
-export TENANT_DOMAIN=$KAMAJI_REGION.cloudapp.azure.com
-export TENANT_VERSION=v1.23.4
-export TENANT_ADDR=10.240.0.100 # IP used to expose the tenant control plane
-export TENANT_PORT=6443 # PORT used to expose the tenant control plane
-export TENANT_POD_CIDR=10.36.0.0/16
-export TENANT_SVC_CIDR=10.96.0.0/16
-export TENANT_DNS_SERVICE=10.96.0.10
-
-export TENANT_VM_SIZE=Standard_D2ds_v4
-export TENANT_VM_IMAGE=UbuntuLTS
-export TENANT_RG=$TENANT_NAME
-export TENANT_NSG=$TENANT_NAME-nsg
-export TENANT_VNET_NAME=$TENANT_NAME
-export TENANT_VNET_ADDRESS=192.168.0.0/16
-export TENANT_SUBNET_NAME=$TENANT_NAME-subnet
-export TENANT_SUBNET_ADDRESS=192.168.10.0/24
-export TENANT_VMSS=$TENANT_NAME-vmss

deploy/kamaji-tenant.env

@@ -1,17 +0,0 @@
-
-# tenant cluster parameters
-export TENANT_NAMESPACE=tenants
-export TENANT_NAME=tenant-00
-export TENANT_DOMAIN=clastix.labs
-export TENANT_VERSION=v1.23.1
-export TENANT_ADDR=192.168.32.150 # IP used to expose the tenant control plane
-export TENANT_PORT=6443 # PORT used to expose the tenant control plane
-export TENANT_POD_CIDR=10.36.0.0/16
-export TENANT_SVC_CIDR=10.96.0.0/16
-export TENANT_DNS_SERVICE=10.96.0.10
-
-# tenant node addresses
-export WORKER0=192.168.32.90
-export WORKER1=192.168.32.91
-export WORKER2=192.168.32.92
-export WORKER3=192.168.32.93

deploy/kamaji.env

@@ -0,0 +1,18 @@
+# kamaji parameters
+export KAMAJI_NAMESPACE=kamaji-system
+
+# tenant cluster parameters
+export TENANT_NAMESPACE=tenants
+export TENANT_NAME=tenant-00
+export TENANT_DOMAIN=clastix.labs
+export TENANT_VERSION=v1.23.5
+export TENANT_PORT=6443 # port used to expose the tenant api server
+export TENANT_PROXY_PORT=8132 # port used to expose the konnectivity server
+export TENANT_POD_CIDR=10.36.0.0/16
+export TENANT_SVC_CIDR=10.96.0.0/16
+export TENANT_DNS_SERVICE=10.96.0.10
+
+# tenant node addresses
+export WORKER0=172.12.0.10
+export WORKER1=172.12.0.11
+export WORKER2=172.12.0.12

deploy/kind/Makefile

@@ -2,7 +2,7 @@ kind_path := $(patsubst %/,%,$(dir $(abspath $(lastword $(MAKEFILE_LIST)))))
 
 include ../etcd/Makefile
 
-.PHONY: kind ingress-nginx kamaji-kind-worker-build
+.PHONY: kind ingress-nginx
 
 .DEFAULT_GOAL := kamaji
 
@@ -11,7 +11,10 @@ prometheus-stack:
 	helm repo update
 	helm install prometheus-stack --create-namespace -n monitoring prometheus-community/kube-prometheus-stack
 
-kamaji: kind ingress-nginx etcd-cluster
+reqs: kind ingress-nginx etcd-cluster
+
+kamaji: reqs
+	@kubectl apply -f $(kind_path)/../../config/install.yaml
 
 destroy: kind/destroy etcd-certificates/cleanup
 
@@ -26,11 +29,5 @@ ingress-nginx: ingress-nginx-install
 ingress-nginx-install:
 	kubectl apply -f $(kind_path)/nginx-deploy.yaml
 
-kamaji-kind-worker-build:
-	docker build -f $(kind_path)/kamaji-kind-worker.dockerfile -t quay.io/clastix/kamaji-kind-worker:$${WORKER_VERSION:-latest} .
-
-kamaji-kind-worker-push: kamaji-kind-worker-build
-	docker push quay.io/clastix/kamaji-kind-worker:$${WORKER_VERSION:-latest}
-
 kamaji-kind-worker-join:
 	$(kind_path)/join-node.bash

deploy/kind/README.md

@@ -1,138 +0,0 @@
-# Setup a minimal Kamaji for development
-
-This document explains how to deploy a minimal Kamaji setup on [KinD](https://kind.sigs.k8s.io/) for development scopes. Please refer to the [Kamaji documentation](../../README.md) for understanding all the terms used in this guide, as for example: `admin cluster` and `tenant control plane`.
-
-## Tools
-
-We assume you have installed on your workstation:
-
-- [Docker](https://docs.docker.com/engine/install/)
-- [KinD](https://kind.sigs.k8s.io/)
-- [kubectl](https://kubernetes.io/docs/tasks/tools/)
-- [kubeadm](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/install-kubeadm/)
-- [jq](https://stedolan.github.io/jq/)
-- [openssl](https://www.openssl.org/)
-- [cfssl](https://github.com/cloudflare/cfssl)
-- [cfssljson](https://github.com/cloudflare/cfssl)
-
-## Setup Kamaji on KinD
-
-The instance of Kamaji is made of a single node hosting:
-
-- admin control-plane
-- admin worker
-- multi-tenant etcd cluster
-
-The multi-tenant etcd cluster is deployed as statefulset into the Kamaji node.
-
-Run `make kamaji` to setup Kamaji on KinD.
-
-```bash
-cd ./deploy/kind
-make kamaji
-```
-
-At this moment you will have your KinD up and running and ETCD cluster in multitenant mode. 
-
-### Install Kamaji
-
-```bash
-$ kubectl apply -f ../../config/install.yaml
-```
-
-### Deploy Tenant Control Plane
-
-Now it is the moment of deploying your first tenant control plane.
-
-```bash
-$ kubectl apply -f - <<EOF
-apiVersion: kamaji.clastix.io/v1alpha1
-kind: TenantControlPlane
-metadata:
-  name: tenant1
-spec:
-  controlPlane:
-    deployment:
-      replicas: 2
-      additionalMetadata:
-        annotations:
-          environment.clastix.io: tenant1
-          tier.clastix.io: "0"
-        labels:
-          tenant.clastix.io: tenant1
-          kind.clastix.io: deployment
-    service:
-      additionalMetadata:
-        annotations:
-          environment.clastix.io: tenant1
-          tier.clastix.io: "0"
-        labels:
-          tenant.clastix.io: tenant1
-          kind.clastix.io: service
-      serviceType: NodePort
-    ingress:
-      enabled: false
-  kubernetes:
-    version: "v1.23.4"
-    kubelet:
-      cgroupfs: cgroupfs
-    admissionControllers:
-    - LimitRanger
-    - ResourceQuota
-  networkProfile:
-    address: "172.18.0.2"
-    port: 31443
-    domain: "clastix.labs"
-    serviceCidr: "10.96.0.0/16"
-    podCidr: "10.244.0.0/16"
-    dnsServiceIPs: 
-    - "10.96.0.10"
-EOF
-```
-
-> Check networkProfile fields according to your installation
-> To let Kamaji works in kind, you have indicate that the service must be [NodePort](https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport)
-
-### Get Kubeconfig
-
-Let's retrieve kubeconfig and store in `/tmp/kubeconfig`
-
-```bash
-$ kubectl get secrets tenant1-admin-kubeconfig -o json \
- | jq -r '.data["admin.conf"]' \
- | base64 -d > /tmp/kubeconfig
- ```
-
-It can be export it, to facilitate the next tasks:
-
-```bash
-$ export KUBECONFIG=/tmp/kubeconfig
-```
-
-### Install CNI
-
-We highly recommend to install [kindnet](https://github.com/aojea/kindnet) as CNI for your kamaji TCP.
-
-```bash
-$ kubectl create -f https://raw.githubusercontent.com/aojea/kindnet/master/install-kindnet.yaml
-```
-
-### Join worker nodes
-
-```bash
-$ make kamaji-kind-worker-join
-```
-
-> To add more worker nodes, run again the command above.
-
-Check out the node:
-
-```bash
-$ kubectl get nodes
-NAME           STATUS   ROLES    AGE   VERSION
-d2d4b468c9de   Ready    <none>   44s   v1.23.4
-```
-
-> For more complex scenarios (exposing port, different version and so on), run `join-node.bash`
-
-Tenant control plane provision has been finished in a minimal Kamaji setup based on KinD. Therefore, you could develop, test and make your own experiments with Kamaji.

deploy/kind/cni-kindnet-config.json

@@ -1,33 +0,0 @@
-{
-    "cniVersion": "0.3.1",
-    "name": "kindnet",
-    "plugins": [
-      {
-        "type": "ptp",
-        "ipMasq": false,
-        "ipam": {
-          "type": "host-local",
-          "dataDir": "/run/cni-ipam-state",
-          "routes": [
-            {
-              "dst": "0.0.0.0/0"
-            }
-          ],
-          "ranges": [
-            [
-              {
-                "subnet": "10.244.0.0/24"
-              }
-            ]
-          ]
-        },
-        "mtu": 1500
-      },
-      {
-        "type": "portmap",
-        "capabilities": {
-          "portMappings": true
-        }
-      }
-    ]
-  }

deploy/kind/etcd-cluster.yaml

@@ -1,177 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  creationTimestamp: "2021-12-08T17:49:38Z"
-  name: etcd
-  namespace: kamaji-system
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: etcd-server
-  namespace: kamaji-system
-spec:
-  type: ClusterIP
-  ports:
-    - name: client
-      port: 2379
-      protocol: TCP
-      targetPort: 2379
-  selector:
-    app: etcd
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: etcd
-  namespace: kamaji-system
-spec:
-  clusterIP: None
-  ports:
-    - port: 2379
-      name: client
-    - port: 2380
-      name: peer
-  selector:
-    app: etcd
----
-apiVersion: apps/v1
-kind: StatefulSet
-metadata:
-  name: etcd
-  labels:
-    app: etcd
-  namespace: kamaji-system
-spec:
-  serviceName: etcd
-  selector:
-    matchLabels:
-      app: etcd
-  replicas: 3
-  template:
-    metadata:
-      name: etcd
-      labels:
-        app: etcd
-    spec:
-      serviceAccountName: etcd
-      volumes:
-      - name: certs
-        secret:
-          secretName: etcd-certs
-      containers:
-        - name: etcd
-          image: quay.io/coreos/etcd:v3.5.1
-          ports:
-            - containerPort: 2379
-              name: client
-            - containerPort: 2380
-              name: peer
-          volumeMounts:
-            - name: data
-              mountPath: /var/run/etcd
-            - name: certs
-              mountPath: /etc/etcd/pki
-          command:
-          - etcd 
-          - --data-dir=/var/run/etcd
-          - --name=$(POD_NAME)
-          - --initial-cluster-state=new
-          - --initial-cluster=etcd-0=https://etcd-0.etcd.$(POD_NAMESPACE).svc.cluster.local:2380,etcd-1=https://etcd-1.etcd.$(POD_NAMESPACE).svc.cluster.local:2380,etcd-2=https://etcd-2.etcd.$(POD_NAMESPACE).svc.cluster.local:2380
-          - --initial-advertise-peer-urls=https://$(POD_NAME).etcd.$(POD_NAMESPACE).svc.cluster.local:2380
-          - --initial-cluster-token=kamaji
-          - --listen-client-urls=https://0.0.0.0:2379
-          - --advertise-client-urls=https://etcd-0.etcd.$(POD_NAMESPACE).svc.cluster.local:2379,https://etcd-1.etcd.$(POD_NAMESPACE).svc.cluster.local:2379,https://etcd-2.etcd.$(POD_NAMESPACE).svc.cluster.local:2379,https://etcd-server.$(POD_NAMESPACE).svc.cluster.local:2379
-          - --client-cert-auth=true
-          - --trusted-ca-file=/etc/etcd/pki/ca.crt
-          - --cert-file=/etc/etcd/pki/server.pem
-          - --key-file=/etc/etcd/pki/server-key.pem
-          - --listen-peer-urls=https://0.0.0.0:2380
-          - --peer-client-cert-auth=true
-          - --peer-trusted-ca-file=/etc/etcd/pki/ca.crt
-          - --peer-cert-file=/etc/etcd/pki/peer.pem 
-          - --peer-key-file=/etc/etcd/pki/peer-key.pem
-          - --auto-compaction-mode=periodic
-          - --auto-compaction-retention=5m
-          - --v=8
-          env:
-          - name: POD_NAME
-            valueFrom:
-              fieldRef:
-                fieldPath: metadata.name
-          - name: POD_NAMESPACE
-            valueFrom:
-              fieldRef:
-                fieldPath: metadata.namespace
-          livenessProbe:
-            failureThreshold: 8
-            httpGet:
-              host: 127.0.0.1
-              path: /health
-              port: 2381
-              scheme: HTTP
-            initialDelaySeconds: 10
-            periodSeconds: 10
-            timeoutSeconds: 15
-          startupProbe:
-            failureThreshold: 24
-            httpGet:
-              host: 127.0.0.1
-              path: /health
-              port: 2381
-              scheme: HTTP
-            initialDelaySeconds: 10
-            periodSeconds: 10
-            timeoutSeconds: 15
-  volumeClaimTemplates:
-    - metadata:
-        name: data
-      spec:
-        accessModes: ["ReadWriteOnce"]
-        resources:
-          requests:
-            storage: 8Gi
----
-apiVersion: v1
-kind: Pod
-metadata:
-  labels:
-    app: etcd
-  name: etcd-root-client
-  namespace: kamaji-system
-spec:
-  serviceAccountName: etcd
-  containers:
-  - command:
-    - sleep
-    - infinity
-    env:
-    - name: POD_NAMESPACE
-      valueFrom:
-        fieldRef:
-          fieldPath: metadata.namespace    
-    - name: ETCDCTL_ENDPOINTS
-      value: https://etcd-server.$(POD_NAMESPACE).svc.cluster.local:2379
-    - name: ETCDCTL_CACERT
-      value: /opt/certs/ca/ca.crt
-    - name: ETCDCTL_CERT
-      value: /opt/certs/root-client-certs/tls.crt
-    - name: ETCDCTL_KEY
-      value: /opt/certs/root-client-certs/tls.key
-    image: quay.io/coreos/etcd:v3.5.1
-    imagePullPolicy: IfNotPresent
-    name: etcd-client
-    resources: {}
-    volumeMounts:
-    - name: root-client-certs
-      mountPath: /opt/certs/root-client-certs
-    - name: ca
-      mountPath: /opt/certs/ca     
-  volumes:
-  - name: root-client-certs
-    secret:
-      secretName: root-client-certs
-  - name: ca
-    secret:
-      secretName: etcd-certs     
-

deploy/kind/join-node.bash

@@ -3,11 +3,12 @@
 set -e
 
 # Constants
-export DOCKER_IMAGE_NAME="quay.io/clastix/kamaji-kind-worker"
+export DOCKER_IMAGE_NAME="kindest/node"
 export DOCKER_NETWORK="kind"
 
 # Variables
-export KUBERNETES_VERSION=${1:-latest}
+export KUBERNETES_VERSION=${1:-v1.23.5}
+export KUBECONFIG="${KUBECONFIG:-/tmp/kubeconfig}"
 
 if [ -z $2 ]
 then
@@ -24,9 +25,12 @@ kubectl cluster-info
 echo "Are you pointing to the right tenant control plane? (Type return to continue)"
 read
 
-JOIN_CMD="$(kubeadm --kubeconfig=/tmp/kubeconfig token create --print-join-command) --ignore-preflight-errors=SystemVerification"
+JOIN_CMD="$(kubeadm --kubeconfig=${KUBECONFIG} token create --print-join-command) --ignore-preflight-errors=SystemVerification"
 echo "Deploying new node..."
 NODE=$(docker run -d --privileged -v /lib/modules:/lib/modules:ro -v /var --net $DOCKER_NETWORK $MAPPING_PORT $DOCKER_IMAGE_NAME:$KUBERNETES_VERSION)
 sleep 10
 echo "Joining new node..."
 docker exec -e JOIN_CMD="$JOIN_CMD" $NODE /bin/bash -c "$JOIN_CMD"
+
+echo "Node has joined! Remember to install the kind-net CNI by issuing the following command:"
+echo "  $: kubectl apply -f https://raw.githubusercontent.com/aojea/kindnet/master/install-kindnet.yaml"

deploy/kind/kamaji-kind-worker.dockerfile

@@ -1,4 +0,0 @@
-ARG KUBERNETES_VERSION=v1.23.4
-FROM kindest/node:$KUBERNETES_VERSION
-
-COPY ./cni-kindnet-config.json /etc/cni/net.d/10-kindnet.conflist

deploy/kind/kind-kamaji.yaml

@@ -10,6 +10,10 @@ nodes:
     nodeRegistration:
       kubeletExtraArgs:
         node-labels: "ingress-ready=true"
+  ## required for Cluster API local development
+  extraMounts:
+    - hostPath: /var/run/docker.sock
+      containerPath: /var/run/docker.sock
   extraPortMappings:
     ## expose port 80 of the node to port 80 on the host
   - containerPort: 80
@@ -19,12 +23,15 @@ nodes:
   - containerPort: 443
     hostPort: 443
     protocol: TCP
+    ## expose port 31132 of the node to port 31132 on the host for konnectivity
+  - containerPort: 31132
+    hostPort: 31132
+    protocol: TCP   
     ## expose port 31443 of the node to port 31443 on the host
   - containerPort: 31443
     hostPort: 31443
-    protocol: TCP    
+    protocol: TCP
     ## expose port 6443 of the node to port 8443 on the host
   - containerPort: 6443
     hostPort: 8443
     protocol: TCP
-

deploy/kine/README.md

@@ -0,0 +1,26 @@
+# Kine integration
+
+[kine](https://github.com/k3s-io/kine) is an `etcd` shim that allows to use a different datastore for your Kubernetes cluster.
+
+Kamaji actually allows to run a shared datastore using different MySQL and PostgreSQL schemas per Tenant.
+This can help in overcoming the `etcd` limitation regarding scalability and cluster size, as well with HA and replication.
+
+## Kamaji additional CLI flags
+
+Kamaji read the data store configuration from a cluster-scoped resource named `DataStore`, containing all tha required details to secure a connection using a specific driver. 
+
+- [Example of a `etcd` DataStore](./../../config/samples/kamaji_v1alpha1_datastore_etcd.yaml)
+- [Example of a `MySQL` DataStore](./../../config/samples/kamaji_v1alpha1_datastore_mysql.yaml)
+- [Example of a `PostgreSQL` DataStore](./../../config/samples/kamaji_v1alpha1_datastore_postgresql.yaml)
+
+Once the datastore is running, and the `DataStore` has been created with the required details, we need to provide information about it to Kamaji by using the following flag and pointing to the resource name:
+
+```
+--datastore={.metadata.name}
+```
+
+## Drivers
+
+Further details on the setup for each driver are available here:
+- [MySQL/MariaDB](../deploy/kine/mysql/README.md)
+- [PostgreSQL](../deploy/kine/postgresql/README.md)

deploy/kine/mysql/Makefile

@@ -0,0 +1,31 @@
+ROOT_DIR:=$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))
+
+mariadb: mariadb-certificates mariadb-secret mariadb-deployment
+
+mariadb-certificates:
+	rm -rf $(ROOT_DIR)/certs && mkdir $(ROOT_DIR)/certs
+	cfssl gencert -initca $(ROOT_DIR)/ca-csr.json | cfssljson -bare $(ROOT_DIR)/certs/ca
+	@mv $(ROOT_DIR)/certs/ca.pem $(ROOT_DIR)/certs/ca.crt
+	@mv $(ROOT_DIR)/certs/ca-key.pem $(ROOT_DIR)/certs/ca.key
+	cfssl gencert -ca=$(ROOT_DIR)/certs/ca.crt -ca-key=$(ROOT_DIR)/certs/ca.key \
+		-config=$(ROOT_DIR)/config.json -profile=server \
+		$(ROOT_DIR)/server-csr.json | cfssljson -bare $(ROOT_DIR)/certs/server
+	@mv $(ROOT_DIR)/certs/server.pem $(ROOT_DIR)/certs/server.crt
+	@mv $(ROOT_DIR)/certs/server-key.pem $(ROOT_DIR)/certs/server.key
+	chmod 644 $(ROOT_DIR)/certs/*
+
+mariadb-secret:
+	@kubectl -n kamaji-system create secret generic mysql-config \
+		--from-file=$(ROOT_DIR)/certs/ca.crt --from-file=$(ROOT_DIR)/certs/ca.key \
+		--from-file=$(ROOT_DIR)/certs/server.key --from-file=$(ROOT_DIR)/certs/server.crt \
+		--from-file=$(ROOT_DIR)/mysql-ssl.cnf \
+		--from-literal=MYSQL_ROOT_PASSWORD=root \
+		--dry-run=client -o yaml | kubectl apply -f -
+
+mariadb-deployment:
+	@kubectl -n kamaji-system apply -f $(ROOT_DIR)/mariadb.yaml
+
+mariadb-destroy:
+	@kubectl delete -n kamaji-system -f $(ROOT_DIR)/mariadb.yaml --ignore-not-found
+	@kubectl delete -n kamaji-system secret mysql-config --ignore-not-found
+	@kubectl delete -n kamaji-system secret kine-secret --ignore-not-found

deploy/kine/mysql/README.md

@@ -0,0 +1,74 @@
+# MySQL as Kubernetes Storage
+
+Kamaji offers the possibility of having a different storage system than `ETCD` thanks to [kine](https://github.com/k3s-io/kine). One of the implementations is [MySQL](https://www.mysql.com/).
+
+Kamaji project is developed using [kind](https://kind.sigs.k8s.io), therefore, MySQL (or [MariaDB](https://mariadb.org/) in this case) will be deployed into the local kubernetes cluster in order to be used as storage for the tenants.
+
+There is a Makefile to help with the process:
+
+# Setup
+
+Setup of the MySQL/MariaDB backend can be easily issued with a single command.
+
+```bash
+$ make mariadb
+```
+
+This action will perform all the necessary stuffs to have MariaDB as Kubernetes storage backend using kine.
+
+```shell
+rm -rf /home/prometherion/Documents/clastix/kamaji/deploy/mysql/certs && mkdir /home/prometherion/Documents/clastix/kamaji/deploy/mysql/certs
+cfssl gencert -initca /home/prometherion/Documents/clastix/kamaji/deploy/mysql/ca-csr.json | cfssljson -bare /home/prometherion/Documents/clastix/kamaji/deploy/mysql/certs/ca
+2022/08/18 23:52:56 [INFO] generating a new CA key and certificate from CSR
+2022/08/18 23:52:56 [INFO] generate received request
+2022/08/18 23:52:56 [INFO] received CSR
+2022/08/18 23:52:56 [INFO] generating key: rsa-2048
+2022/08/18 23:52:56 [INFO] encoded CSR
+2022/08/18 23:52:56 [INFO] signed certificate with serial number 310428005543054656774215122317606431230766314770
+cfssl gencert -ca=/home/prometherion/Documents/clastix/kamaji/deploy/mysql/certs/ca.crt -ca-key=/home/prometherion/Documents/clastix/kamaji/deploy/mysql/certs/ca.key \
+        -config=/home/prometherion/Documents/clastix/kamaji/deploy/mysql/config.json -profile=server \
+        /home/prometherion/Documents/clastix/kamaji/deploy/mysql/server-csr.json | cfssljson -bare /home/prometherion/Documents/clastix/kamaji/deploy/mysql/certs/server
+2022/08/18 23:52:56 [INFO] generate received request
+2022/08/18 23:52:56 [INFO] received CSR
+2022/08/18 23:52:56 [INFO] generating key: rsa-2048
+2022/08/18 23:52:56 [INFO] encoded CSR
+2022/08/18 23:52:56 [INFO] signed certificate with serial number 582698914718104852311252458344736030793138969927
+chmod 644 /home/prometherion/Documents/clastix/kamaji/deploy/mysql/certs/*
+secret/mysql-config created
+secret/kine-secret created
+serviceaccount/mariadb created
+service/mariadb created
+deployment.apps/mariadb created
+persistentvolumeclaim/pvc-mariadb created
+```
+
+## Certificate creation
+
+```bash
+$ make mariadb-certificates
+```
+
+Communication between kine and the backend is encrypted, therefore, a CA and a certificate from it must be created.
+
+## Secret Deployment
+
+```bash
+$ make mariadb-secrets
+```
+
+Previous certificates and MySQL configuration have to be available in order to be used.
+They will be under the secret `kamaji-system:mysql-config`, used by the MySQL/MariaDB instance.
+
+## Deployment
+
+```bash
+$ make mariadb-deployment
+```
+
+Finally, starts the MySQL/MariaDB installation with all the required settings, such as SSL connection, and configuration.
+
+# Cleanup
+
+```bash
+$ make mariadb-destroy
+```

deploy/kine/mysql/ca-csr.json

@@ -0,0 +1,18 @@
+{
+  "CN": "Clastix CA",
+  "key": {
+    "algo": "rsa",
+    "size": 2048
+  },
+  "names": [
+    {
+      "C": "IT",
+      "ST": "Italy",
+      "L": "Milan"
+    }
+  ],
+  "hosts": [
+    "127.0.0.1",
+    "localhost"
+  ]
+}

deploy/kine/mysql/config.json

@@ -0,0 +1,18 @@
+{
+  "signing": {
+      "default": {
+          "expiry": "8760h"
+      },
+      "profiles": {
+          "server": {
+              "expiry": "8760h",
+              "usages": [
+                  "signing",
+                  "key encipherment",
+                  "server auth",
+                  "client auth"
+              ]
+          }
+      }
+  }
+}

deploy/kine/mysql/kine.yaml

@@ -0,0 +1,61 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kine-tenant
+  namespace:
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: kine-tenant
+  namespace:
+spec:
+  type: ClusterIP
+  ports:
+    - name: server
+      port: 2379
+      protocol: TCP
+      targetPort: 2379
+  selector:
+    app: kine-tenant
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: kine-tenant
+  labels:
+    app: kine-tenant
+  namespace:
+spec:
+  selector:
+    matchLabels:
+      app: kine-tenant
+  replicas: 1
+  template:
+    metadata:
+      name: kine-tenant
+      labels:
+        app: kine-tenant
+    spec:
+      serviceAccountName: kine-tenant
+      volumes:
+      - name: certs
+        secret:
+          secretName: mysql-certs
+      containers:
+        - name: kine-tenant
+          image: rancher/kine:v0.9.2-amd64
+          ports:
+            - containerPort: 2379
+              name: server
+          volumeMounts:
+          - name: certs
+            mountPath: /kine
+          env:    
+          - name: GODEBUG
+            value: "x509ignoreCN=0"
+          args:
+          - --endpoint=mysql://tenant1:tenant1@tcp(mysql:3306)/tenant1
+          - --ca-file=/kine/ca.crt
+          - --cert-file=/kine/server.crt
+          - --key-file=/kine/server.key

deploy/kine/mysql/mariadb.yaml

@@ -0,0 +1,77 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: mariadb
+  namespace:
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: mariadb
+  namespace:
+spec:
+  type: ClusterIP
+  ports:
+    - name: server
+      port: 3306
+      protocol: TCP
+      targetPort: 3306
+  selector:
+    app: mariadb
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: mariadb
+  labels:
+    app: mariadb
+  namespace:
+spec:
+  selector:
+    matchLabels:
+      app: mariadb
+  replicas: 1
+  template:
+    metadata:
+      name: mariadb
+      labels:
+        app: mariadb
+    spec:
+      serviceAccountName: mariadb
+      volumes:
+      - name: certs
+        secret:
+          secretName: mysql-config
+      - name: data
+        persistentVolumeClaim:
+          claimName: pvc-mariadb
+      containers:
+        - name: mariadb
+          image: mariadb:10.7.4
+          ports:
+            - containerPort: 3306
+              name: server
+          volumeMounts:
+          - name: data
+            mountPath: /var/lib/mariadb
+          - name: certs
+            mountPath: /etc/mysql/conf.d/
+          env:
+          - name: MYSQL_ROOT_PASSWORD
+            valueFrom:
+              secretKeyRef:
+                name: mysql-config
+                key: MYSQL_ROOT_PASSWORD                
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: pvc-mariadb
+  namespace:
+spec:
+  accessModes:
+  - ReadWriteOnce
+  resources:
+    requests:
+      storage: 2Gi
+  storageClassName: standard

deploy/kine/mysql/mysql-ssl.cnf

@@ -0,0 +1,5 @@
+[mysqld]
+ssl-ca=/etc/mysql/conf.d/ca.crt
+ssl-cert=/etc/mysql/conf.d/server.crt
+ssl-key=/etc/mysql/conf.d/server.key
+require_secure_transport=ON

deploy/kine/mysql/server-csr.json

@@ -0,0 +1,19 @@
+{
+  "CN": "mariadb.kamaji-system.svc.cluster.local",
+  "key": {
+    "algo": "rsa",
+    "size": 2048
+  },
+  "hosts": [
+    "127.0.0.1",
+    "localhost",
+    "mariadb",
+    "mariadb.kamaji-system",
+    "mariadb.kamaji-system.svc",
+    "mariadb.kamaji-system.svc.cluster.local",
+    "mysql",
+    "mysql.kamaji-system",
+    "mysql.kamaji-system.svc",
+    "mysql.kamaji-system.svc.cluster.local"
+  ]
+}

deploy/kine/postgresql/Makefile

@@ -0,0 +1,27 @@
+ROOT_DIR:=$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))
+
+postgresql: cnpg-setup cnpg-deploy postgresql-secret
+
+cnpg-setup:
+	@kubectl apply -f https://raw.githubusercontent.com/cloudnative-pg/cloudnative-pg/main/releases/cnpg-1.16.0.yaml
+
+cnpg-deploy:
+	@kubectl -n cnpg-system rollout status deployment/cnpg-controller-manager
+	@kubectl -n kamaji-system apply -f postgresql.yaml
+	@while ! kubectl -n kamaji-system get secret postgresql-superuser > /dev/null 2>&1; do sleep 1; done
+
+CNPG = $(shell git rev-parse --show-toplevel)/bin/kubectl-cnpg
+cnpg:
+	@test -f $(shell git rev-parse --show-toplevel)/bin/kubectl-cnpg || curl -sSfL \
+      https://github.com/cloudnative-pg/cloudnative-pg/raw/main/hack/install-cnpg-plugin.sh | \
+      sh -s -- -b $(shell git rev-parse --show-toplevel)/bin
+
+postgresql-secret: cnpg
+	@kubectl -n kamaji-system get secret postgres-root-cert > /dev/null 2>&1 || $(CNPG) certificate postgres-root-cert \
+    		--cnpg-cluster postgresql \
+    		--cnpg-user $$(kubectl -n kamaji-system get secret postgresql-superuser -o jsonpath='{.data.username}' | base64 -d)
+
+postgresql-destroy:
+	@kubectl delete -f https://raw.githubusercontent.com/cloudnative-pg/cloudnative-pg/main/releases/cnpg-1.16.0.yaml --ignore-not-found && \
+	kubectl delete secret postgres-root-cert --ignore-not-found && \
+	kubectl delete secret kine-secret --ignore-not-found

deploy/kine/postgresql/README.md

@@ -0,0 +1,74 @@
+# PostgreSQL as Kubernetes Storage
+
+Kamaji offers the possibility of having a different storage system than `etcd` thanks to [kine](https://github.com/k3s-io/kine).
+One of the implementations is [PostgreSQL](https://www.postgresql.org/).
+
+Kamaji project is developed using [kind](https://kind.sigs.k8s.io), therefore, a PostgreSQL instance must be deployed in advance into the local kubernetes cluster in order to be used as storage for the tenants.
+For the sake of simplicity, the [cloudnative-pg](https://cloudnative-pg.io/) Operator will be used to simplify the setup of it.
+
+There is a Makefile to help with the process:
+
+## Setup
+
+```bash
+$ make postgresql
+```
+
+This target will install the `cloudnative-pg`, creating the PostgreSQL instance in the Kamaji Namespace, along with the generation of the required Secret resource for the kine integration.
+
+This action is idempotent and doesn't overwrite values if they already exist.
+
+```shell
+namespace/cnpg-system unchanged
+customresourcedefinition.apiextensions.k8s.io/backups.postgresql.cnpg.io configured
+customresourcedefinition.apiextensions.k8s.io/clusters.postgresql.cnpg.io configured
+customresourcedefinition.apiextensions.k8s.io/poolers.postgresql.cnpg.io configured
+customresourcedefinition.apiextensions.k8s.io/scheduledbackups.postgresql.cnpg.io configured
+serviceaccount/cnpg-manager unchanged
+clusterrole.rbac.authorization.k8s.io/cnpg-manager configured
+clusterrolebinding.rbac.authorization.k8s.io/cnpg-manager-rolebinding unchanged
+configmap/cnpg-default-monitoring unchanged
+service/cnpg-webhook-service unchanged
+deployment.apps/cnpg-controller-manager unchanged
+mutatingwebhookconfiguration.admissionregistration.k8s.io/cnpg-mutating-webhook-configuration configured
+validatingwebhookconfiguration.admissionregistration.k8s.io/cnpg-validating-webhook-configuration configured
+deployment "cnpg-controller-manager" successfully rolled out
+cluster.postgresql.cnpg.io/postgresql unchanged
+secret/postgres-root-cert created
+```
+
+## Operator setup
+
+```bash
+$ make cnpg-setup
+```
+
+This target will apply all the required manifests with the `cloudnative-pg` CRD, and required RBAC, and Deployment.
+
+Release [v1.16.0](https://github.com/cloudnative-pg/cloudnative-pg/releases/tag/v1.16.0) has been tested successfully.
+
+## SSL certificate Secret generation
+
+```bash
+$ make postgresql-secret
+```
+
+This target will download locally the `kubectl-cnpg` utility to generate an SSL certificate required to secure the connection to the PostgreSQL instance.
+
+## Certificate generation
+
+```bash
+$ make postgresql-secret
+```
+
+Generate the Certificate required to connect to the DataStore.
+
+## Teardown
+
+```bash
+$ make postgresql-destroy
+```
+
+This will lead to the deletion of the `cloudnative-pg` Operator, along with any instance, and related secrets.
+
+This action is idempotent.

deploy/kine/postgresql/postgresql.yaml

@@ -0,0 +1,13 @@
+apiVersion: postgresql.cnpg.io/v1
+kind: Cluster
+metadata:
+  name: postgresql
+spec:
+  description: PostgreSQL cluster used by Kamaji along with kine
+  instances: 3
+  postgresql:
+    pg_hba:
+      - hostssl app all all cert  # makes authentication entirely based on certificates
+  primaryUpdateStrategy: unsupervised
+  storage:
+    size: 1Gi

deploy/nodes-prerequisites.sh

@@ -18,7 +18,9 @@ EOF
 for i in "${!HOSTS[@]}"; do
   HOST=${HOSTS[$i]}
   ssh ${USER}@${HOST} -t 'sudo apt update && sudo apt install -y containerd'
-  ssh ${USER}@${HOST} -t 'sudo systemctl start containerd && sudo systemctl enable containerd'
+  ssh ${USER}@${HOST} -t 'sudo mkdir -p /etc/containerd'
+  ssh ${USER}@${HOST} -t 'containerd config default | sed -e "s#SystemdCgroup = false#SystemdCgroup = true#g" | sudo tee -a /etc/containerd/config.toml'
+  ssh ${USER}@${HOST} -t 'sudo systemctl restart containerd && sudo systemctl enable containerd'
   scp containerd.conf ${USER}@${HOST}:
   ssh ${USER}@${HOST} -t 'sudo chown -R root:root containerd.conf && sudo mv containerd.conf /etc/modules-load.d/containerd.conf'
   ssh ${USER}@${HOST} -t 'sudo modprobe overlay && sudo modprobe br_netfilter'

deploy/tenant-cloudinit.yaml

@@ -21,10 +21,12 @@ runcmd:
   - sudo modprobe overlay
   - sudo modprobe br_netfilter
   - sudo sysctl --system
-  - sudo systemctl start containerd
+  - sudo mkdir -p /etc/containerd
+  - containerd config default | sed -e 's#SystemdCgroup = false#SystemdCgroup = true#g' | sudo tee -a /etc/containerd/config.toml
+  - sudo systemctl restart containerd
   - sudo systemctl enable containerd
   - sudo curl -fsSLo /usr/share/keyrings/kubernetes-archive-keyring.gpg https://packages.cloud.google.com/apt/doc/apt-key.gpg
   - echo "deb [signed-by=/usr/share/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" | sudo tee /etc/apt/sources.list.d/kubernetes.list
   - sudo apt update
-  - sudo apt install -y kubelet kubeadm kubectl containerd
-  - sudo apt-mark hold kubelet kubeadm kubectl
+  - sudo apt install -y kubelet=1.23.5-00 kubeadm=1.23.5-00 kubectl=1.23.5-00
+  - sudo apt-mark hold kubelet kubeadm kubectl

docs/README.md

@@ -0,0 +1,13 @@
+# Kamaji documentation
+
+- [Core Concepts](./concepts.md)
+- [Architecture](./architecture.md)
+- [Getting started](./getting-started-with-kamaji.md)
+- Guides:
+  - [Deploy Kamaji](./kamaji-deployment-guide.md)
+  - [Deploy Kamaji on Azure](./kamaji-azure-deployment-guide.md)
+  - Deploy Kamaji on AWS
+  - Deploy Kamaji on GCP
+  - Deploy Kamaji on OpenStack
+- [Reference](./reference.md)
+- [Versioning](./versioning.md)

docs/concepts.md

@@ -1 +1,31 @@
-# Kamaji concepts
+# Core Concepts
+
+Kamaji is a Kubernetes Operator. It turns any Kubernetes cluster into an _“admin cluster”_ to orchestrate other Kubernetes clusters called _“tenant clusters”_. 
+
+## Tenant Control Plane
+What makes Kamaji special is that the Control Plane of a _“tenant cluster”_ is just one or more regular pods running in a namespace of the _“admin cluster”_ instead of a dedicated set of Virtual Machines. This solution makes running control planes at scale cheaper and easier to deploy and operate. The Tenant Control Plane components are packaged in the same way they are running in bare metal or virtual nodes. We leverage the `kubeadm` code to set up the control plane components as they were running on their own server. The unchanged images of upstream `kube-apiserver`, `kube-scheduler`, and `kube-controller-manager` are used.
+
+High Availability and rolling updates of the Tenant Control Plane pods are provided by a regular Deployment. Autoscaling based on the metrics is available. A Service is used to espose the Tenant Control Plane outside of the _“admin cluster”_. The `LoadBalancer` service type is used, `NodePort` and `ClusterIP` with an Ingress Controller are still viable options, depending on the case. 
+
+Kamaji offers a [Custom Resource Definition](https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/) to provide a declarative approach of managing a Tenant Control Plane. This *CRD* is called `TenantControlPlane`, or `tcp` in short.
+
+## Tenant worker nodes
+And what about the tenant worker nodes? They are just _"worker nodes"_, i.e. regular virtual or bare metal machines, connecting to the APIs server of the Tenant Control Plane. Kamaji's goal is to manage the lifecycle of hundreds of these _“tenant clusters”_, not only one, so how to add another tenant cluster to Kamaji? As you could expect, you have just deploys a new Tenant Control Plane in one of the _“admin cluster”_ namespace, and then joins the tenant worker nodes to it.
+
+All the tenant clusters built with Kamaji are fully compliant CNCF Kubernetes clusters and are compatible with the standard Kubernetes toolchains everybody knows and loves.
+
+## Save the state
+Putting the Tenant Control Plane in a pod is the easiest part. Also, we have to make sure each tenant cluster saves the state to be able to store and retrieve data. A dedicated `etcd` cluster for each tenant cluster doesn’t scale well for a managed service because `etcd` data persistence can be cumbersome at scale, rising the operational effort to mitigate it. So we have to find an alternative keeping in mind our goal for a resilient and cost-optimized solution at the same time. As we can deploy any Kubernetes cluster with an external `etcd` cluster, we explored this option for the tenant control planes. On the admin cluster, we deploy a multi-tenant `etcd` cluster storing the state of multiple tenant clusters.
+
+With this solution, the resiliency is guaranteed by the usual `etcd` mechanism, and the pods' count remains under control, so it solves the main goal of resiliency and costs optimization. The trade-off here is that we have to operate an external `etcd` cluster, in addition to `etcd` of the _“admin cluster”_ and manage the access to be sure that each _“tenant cluster”_ uses only its data. Also, there are limits in size in `etcd`, defaulted to 2GB and configurable to a maximum of 8GB. We’re solving this issue by pooling multiple `etcd` togheter and sharding the Tenant Control Planes.
+
+Optionally, Kamaji offers the possibility of using a different storage system than `etcd` to save the state of the tenants' clusters, like MySQL or PostgreSQL compatible databases, thanks to the [kine](https://github.com/k3s-io/kine) integration. 
+
+## Requirements of design
+These are requirements of design behind Kamaji:
+
+- Communication between the _“admin cluster”_ and a _“tenant cluster”_ is unidirectional. The _“admin cluster”_ manages a _“tenant cluster”_, but a _“tenant cluster”_ has no awareness of the _“admin cluster”_.
+- Communication between different _“tenant clusters”_ is not allowed.
+- The worker nodes of tenant should not run anything beyond tenant's workloads.
+
+Goals and scope may vary as the project evolves.