chore(gh): trigger helm release on tag

Signed-off-by: Viktor Oreshkin <imselfish@stek29.rocks>

.github/workflows/ci.yaml

@@ -12,6 +12,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
+      - uses: actions/setup-go@v2
+        with:
+          go-version: '1.18'
       - name: Run golangci-lint
         uses: golangci/golangci-lint-action@v2.3.0
         with:
@@ -27,7 +30,7 @@ jobs:
           fetch-depth: 0
       - uses: actions/setup-go@v2
         with:
-          go-version: '1.17'
+          go-version: '1.18'
       - run: make yaml-installation-file
       - name: Checking if YAML installer file is not aligned
         run: if [[ $(git diff | wc -l) -gt 0 ]]; then echo ">>> Untracked generated files have not been committed" && git --no-pager diff && exit 1; fi

.github/workflows/docker-ci.yml

@@ -0,0 +1,74 @@
+name: docker-ci
+
+on:
+  push:
+    tags:        
+      - "v*"
+
+jobs:
+  docker-ci:
+    runs-on: ubuntu-20.04
+    steps:
+
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v3
+        with:
+          images: |
+             quay.io/${{ github.repository }}
+             docker.io/${{ github.repository }}
+          tags: |
+            type=semver,pattern={{raw}}
+          flavor: |
+            latest=false
+
+      - name: Set up QEMU
+        id: qemu
+        uses: docker/setup-qemu-action@v1
+        with:
+          platforms: arm64,arm
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v1
+        with:
+          install: true
+
+      - name: Inspect builder
+        run: |
+          echo "Name:      ${{ steps.buildx.outputs.name }}"
+          echo "Endpoint:  ${{ steps.buildx.outputs.endpoint }}"
+          echo "Status:    ${{ steps.buildx.outputs.status }}"
+          echo "Flags:     ${{ steps.buildx.outputs.flags }}"
+          echo "Platforms: ${{ steps.buildx.outputs.platforms }}"
+
+      - name: Login to quay.io Container Registry
+        uses: docker/login-action@v1 
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_IO_USERNAME }}
+          password: ${{ secrets.QUAY_IO_TOKEN }}
+
+      - name: Login to docker.io Container Registry
+        uses: docker/login-action@v1 
+        with:
+          registry: docker.io
+          username: ${{ secrets.DOCKER_IO_USERNAME }}
+          password: ${{ secrets.DOCKER_IO_TOKEN }}
+
+      - name: Build and push
+        id: build-release
+        uses: docker/build-push-action@v2
+        with:
+          file: Dockerfile
+          context: .
+          platforms: linux/amd64,linux/arm64,linux/arm
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          build-args:
+
+      - name: Image digest
+        run: echo ${{ steps.build-release.outputs.digest }}

.github/workflows/e2e.yaml

@@ -0,0 +1,44 @@
+name: e2e
+
+on:
+  push:
+    branches: [ "*" ]
+    paths:
+      - '.github/workflows/e2e.yml'
+      - 'api/**'
+      - 'controllers/**'
+      - 'e2e/*'
+      - 'Dockerfile'
+      - 'go.*'
+      - 'main.go'
+      - 'Makefile'
+      - 'internal/**'
+  pull_request:
+    branches: [ "*" ]
+    paths:
+      - '.github/workflows/e2e.yml'
+      - 'api/**'
+      - 'controllers/**'
+      - 'e2e/*'
+      - 'Dockerfile'
+      - 'go.*'
+      - 'main.go'
+      - 'Makefile'
+      - 'internal/**'
+
+jobs:
+  kind:
+    name: Kubernetes
+    runs-on: ubuntu-18.04
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-go@v2
+        with:
+          go-version: '1.18'
+      - run: |
+          sudo apt-get update
+          sudo apt-get install -y golang-cfssl
+      - name: e2e testing
+        run: make e2e

.github/workflows/helm.yaml

@@ -0,0 +1,36 @@
+name: Helm Chart
+
+on:
+  push:
+    branches: [ "*" ]
+    tags: [ "helm-v*" ]
+  pull_request:
+    branches: [ "*" ]
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: azure/setup-helm@v1
+        with:
+          version: 3.3.4
+      - name: Linting Chart
+        run: helm lint ./charts/kamaji
+  release:
+    if: startsWith(github.ref, 'refs/tags/helm-v')
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Publish Helm chart
+        uses: stefanprodan/helm-gh-pages@master
+        with:
+          token: ${{ secrets.BOT_GITHUB_TOKEN }}
+          charts_dir: charts
+          charts_url: https://clastix.github.io/charts
+          owner: clastix
+          repository: charts
+          branch: gh-pages
+          target_dir: .
+          commit_username: prometherion
+          commit_email: dario@tranchitella.eu

.golangci.yml

@@ -4,6 +4,10 @@ linters-settings:
       - standard
       - default
       - prefix(github.com/clastix/kamaji)
+  goheader:
+    template: |-
+      Copyright 2022 Clastix Labs
+      SPDX-License-Identifier: Apache-2.0
 
 linters:
   disable:

Dockerfile

@@ -1,5 +1,13 @@
 # Build the manager binary
-FROM golang:1.17 as builder
+FROM golang:1.18 as builder
+
+ARG TARGETARCH
+ARG GIT_HEAD_COMMIT
+ARG GIT_TAG_COMMIT
+ARG GIT_LAST_TAG
+ARG GIT_MODIFIED
+ARG GIT_REPO
+ARG BUILD_DATE
 
 WORKDIR /workspace
 # Copy the Go Modules manifests
@@ -16,7 +24,9 @@ COPY controllers/ controllers/
 COPY internal/ internal/
 
 # Build
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o manager main.go
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=$TARGETARCH go build \
+    -ldflags "-X github.com/clastix/kamaji/internal.GitRepo=$GIT_REPO -X github.com/clastix/kamaji/internal.GitTag=$GIT_LAST_TAG -X github.com/clastix/kamaji/internal.GitCommit=$GIT_HEAD_COMMIT -X github.com/clastix/kamaji/internal.GitDirty=$GIT_MODIFIED -X github.com/clastix/kamaji/internal.BuildTime=$BUILD_DATE" \
+    -a -o manager main.go
 
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details

Makefile

@@ -36,11 +36,9 @@ IMAGE_TAG_BASE ?= clastix.io/operator
 BUNDLE_IMG ?= $(IMAGE_TAG_BASE)-bundle:v$(VERSION)
 
 # Image URL to use all building/pushing image targets
-IMG ?= controller:latest
+IMG ?= clastix/kamaji:latest
 # Produce CRDs that work back to Kubernetes 1.11 (no version conversion)
 CRD_OPTIONS ?= "crd:trivialVersions=true,preserveUnknownFields=false"
-# ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary.
-ENVTEST_K8S_VERSION = 1.21
 
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
 ifeq (,$(shell go env GOBIN))
@@ -50,7 +48,6 @@ GOBIN=$(shell go env GOBIN)
 endif
 
 # Setting SHELL to bash allows bash commands to be executed by recipes.
-# This is a requirement for 'setup-envtest.sh' in the test target.
 # Options are set to exit when a recipe line exits non-zero or a piped command fails.
 SHELL = /usr/bin/env bash -o pipefail
 .SHELLFLAGS = -ec
@@ -73,33 +70,65 @@ all: build
 help: ## Display this help.
 	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf "  \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
 
+##@ Binary
+
+.PHONY: helm
+HELM = $(shell pwd)/bin/helm
+helm: ## Download helm locally if necessary.
+	$(call go-install-tool,$(HELM),helm.sh/helm/v3/cmd/helm@v3.9.0)
+
+GINKGO = $(shell pwd)/bin/ginkgo
+ginkgo: ## Download ginkgo locally if necessary.
+	$(call go-install-tool,$(GINKGO),github.com/onsi/ginkgo/ginkgo@v1.16.5)
+
+KIND = $(shell pwd)/bin/kind
+kind: ## Download kind locally if necessary.
+	$(call go-install-tool,$(KIND),sigs.k8s.io/kind/cmd/kind@v0.14.0)
+
+CONTROLLER_GEN = $(shell pwd)/bin/controller-gen
+controller-gen: ## Download controller-gen locally if necessary.
+	$(call go-install-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.6.1)
+
+KUSTOMIZE = $(shell pwd)/bin/kustomize
+kustomize: ## Download kustomize locally if necessary.
+	$(call install-kustomize,$(KUSTOMIZE),3.8.7)
+
 ##@ Development
 
 manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects.
 	$(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=manager-role webhook paths="./..." output:crd:artifacts:config=config/crd/bases
+	cp config/crd/bases/kamaji.clastix.io_tenantcontrolplanes.yaml charts/kamaji/crds/tenantcontrolplane.yaml
 
 generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations.
 	$(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..."
 
-fmt: ## Run go fmt against code.
-	go fmt ./...
-
-vet: ## Run go vet against code.
-	go vet ./...
-
-test: manifests generate fmt vet envtest ## Run tests.
-	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" go test ./... -coverprofile cover.out
+test:
+	go test ./... -coverprofile cover.out
 
 ##@ Build
 
+# Get information about git current status
+GIT_HEAD_COMMIT ?= $$(git rev-parse --short HEAD)
+GIT_TAG_COMMIT  ?= $$(git rev-parse --short $(VERSION))
+GIT_MODIFIED_1  ?= $$(git diff $(GIT_HEAD_COMMIT) $(GIT_TAG_COMMIT) --quiet && echo "" || echo ".dev")
+GIT_MODIFIED_2  ?= $$(git diff --quiet && echo "" || echo ".dirty")
+GIT_MODIFIED    ?= $$(echo "$(GIT_MODIFIED_1)$(GIT_MODIFIED_2)")
+GIT_REPO        ?= $$(git config --get remote.origin.url)
+BUILD_DATE      ?= $$(git log -1 --format="%at" | xargs -I{} date -d @{} +%Y-%m-%dT%H:%M:%S)
+
 build: generate fmt vet ## Build manager binary.
 	go build -o bin/manager main.go
 
 run: manifests generate fmt vet ## Run a controller from your host.
 	go run ./main.go
 
-docker-build: test ## Build docker image with the manager.
-	docker build -t ${IMG} .
+docker-build: ## Build docker image with the manager.
+	docker build -t ${IMG} . --build-arg GIT_HEAD_COMMIT=$(GIT_HEAD_COMMIT) \
+		--build-arg GIT_TAG_COMMIT=$(GIT_TAG_COMMIT) \
+		--build-arg GIT_MODIFIED=$(GIT_MODIFIED) \
+		--build-arg GIT_REPO=$(GIT_REPO) \
+		--build-arg GIT_LAST_TAG=$(VERSION) \
+		--build-arg BUILD_DATE=$(BUILD_DATE)
 
 docker-push: ## Push docker image with the manager.
 	docker push ${IMG}
@@ -109,8 +138,8 @@ docker-push: ## Push docker image with the manager.
 dev: generate manifests uninstall install rbac ## Full installation for development purposes
 	go fmt ./...
 
-load: dev docker-build
-	kind load docker-image --name kamaji ${IMG}
+load: docker-build kind
+	$(KIND) load docker-image --name kamaji ${IMG}
 
 rbac: manifests kustomize ## Install RBAC into the K8s cluster specified in ~/.kube/config.
 	$(KUSTOMIZE) build config/rbac | kubectl apply -f -
@@ -131,33 +160,6 @@ undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/confi
 yaml-installation-file: manifests kustomize ## Create yaml installation file
 	$(KUSTOMIZE) build config/default > config/install.yaml
 
-
-CONTROLLER_GEN = $(shell pwd)/bin/controller-gen
-controller-gen: ## Download controller-gen locally if necessary.
-	$(call go-get-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.6.1)
-
-KUSTOMIZE = $(shell pwd)/bin/kustomize
-kustomize: ## Download kustomize locally if necessary.
-	$(call go-get-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/v3@v3.8.7)
-
-ENVTEST = $(shell pwd)/bin/setup-envtest
-envtest: ## Download envtest-setup locally if necessary.
-	$(call go-get-tool,$(ENVTEST),sigs.k8s.io/controller-runtime/tools/setup-envtest@latest)
-
-# go-get-tool will 'go get' any package $2 and install it to $1.
-PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST))))
-define go-get-tool
-@[ -f $(1) ] || { \
-set -e ;\
-TMP_DIR=$$(mktemp -d) ;\
-cd $$TMP_DIR ;\
-go mod init tmp ;\
-echo "Downloading $(2)" ;\
-GOBIN=$(PROJECT_DIR)/bin go get $(2) ;\
-rm -rf $$TMP_DIR ;\
-}
-endef
-
 .PHONY: bundle
 bundle: manifests kustomize ## Generate bundle manifests and metadata, then validate generated files.
 	operator-sdk generate kustomize manifests -q
@@ -213,3 +215,34 @@ catalog-build: opm ## Build a catalog image.
 .PHONY: catalog-push
 catalog-push: ## Push a catalog image.
 	$(MAKE) docker-push IMG=$(CATALOG_IMG)
+
+define install-kustomize
+@[ -f $(1) ] || { \
+set -e ;\
+echo "Installing v$(2)" ;\
+cd bin ;\
+wget "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" ;\
+bash ./install_kustomize.sh $(2) ;\
+}
+endef
+
+# go-install-tool will 'go install' any package $2 and install it to $1.
+PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST))))
+define go-install-tool
+@[ -f $(1) ] || { \
+set -e ;\
+echo "Installing $(2)" ;\
+GOBIN=$(PROJECT_DIR)/bin go install $(2) ;\
+}
+endef
+
+.PHONY: env
+env:
+	@make -C deploy/kind kind ingress-nginx
+
+##@ e2e
+
+.PHONY: e2e
+e2e: env load helm ginkgo ## Create a KinD cluster, install Kamaji on it and run the test suite.
+	$(HELM) upgrade --debug --install kamaji ./charts/kamaji --create-namespace --namespace kamaji-system --set "image.pullPolicy=Never"
+	$(GINKGO) -v ./e2e

README.md

@@ -8,25 +8,26 @@
   </a>
 </p>
 
-**Kamaji** is a tool aimed to build and operate a **Managed Kubernetes Service** with a fraction of the operational burden. With **Kamaji**, you can deploy and operate hundreds of Kubernetes clusters as a hyper-scale cloud provider.
+**Kamaji** deploys and operates **Kubernetes** at scale with a fraction of the operational burden.
 
 <p align="center" style="padding: 6px 6px">
   <img src="assets/kamaji-logo.png" />
 </p>
 
-> This project is still in the early development stage which means it's not ready for production as APIs, commands, flags, etc. are subject to change, but also that your feedback can still help to shape it. Please try it out and let us know what you like, dislike, what works, what doesn't, etc.
-
 ## Why we are building it?
-Global hyper-scalers are leading the Managed Kubernetes space, while regional cloud providers, as well as large corporations, are struggling to offer the same level of experience to their developers because of the lack of the right tools. Also, Kubernetes solutions for the on-premises are designed with an enterprise-first approach and they are too costly when deployed at scale. Project Kamaji aims to solve this pain by leveraging multi-tenancy and simplifying how to run Kubernetes clusters at scale with a fraction of the operational burden.
+Global hyper-scalers are leading the Managed Kubernetes space, while other cloud providers, as well as large corporations, are struggling to offer the same experience to their DevOps teams because of the lack of the right tools. Also, current Kubernetes solutions are mainly designed with an enterprise-first approach and they are too costly when deployed at scale.
+
+**Kamaji** aims to solve these pains by leveraging multi-tenancy and simplifying how to run multiple control planes on the same infrastructure with a fraction of the operational burden.
 
 ## How it works
-Kamaji turns any CNCF conformant Kubernetes cluster into an _“admin cluster”_ to orchestrate other Kubernetes clusters we're calling _“tenant clusters”_. As with every Kubernetes cluster, a tenant cluster has a set of nodes and a control plane, composed of several components: `APIs server`, `scheduler`, `controller manager`. What Kamaji does is deploy those components as a pod running in the admin cluster.
-
-
-And what about the tenant worker nodes? They are just worker nodes: regular instances, e.g. virtual or bare metal, connecting to the APIs server of the tenant cluster. Kamaji's goal is to manage the lifecycle of hundreds of these clusters, not only one, so how can we add another tenant cluster? As you could expect, Kamaji just deploys a new tenant control plane as a new pod in the admin cluster, and then it joins the new tenant worker nodes.
+Kamaji turns any Kubernetes cluster into an _“admin cluster”_ to orchestrate other Kubernetes clusters called _“tenant clusters”_. What makes Kamaji special is that Control Planes of _“tenant clusters”_ are just regular pods running in the _“admin cluster”_ instead of dedicated Virtual Machines. This solution makes running control planes at scale cheaper and easier to deploy and operate. View [Core Concepts](./docs/concepts.md) for a deeper understanding of principles behind Kamaji's design.
 
 <p align="center">
-  <img src="assets/kamaji-light.png" />
+  <img src="assets/kamaji-light.png#gh-light-mode-only" />
+</p>
+
+<p align="center">
+  <img src="assets/kamaji-dark.png#gh-dark-mode-only" />
 </p>
 
 All the tenant clusters built with Kamaji are fully compliant CNCF Kubernetes clusters and are compatible with the standard Kubernetes toolchains everybody knows and loves.
@@ -35,60 +36,48 @@ All the tenant clusters built with Kamaji are fully compliant CNCF Kubernetes cl
   <img src="assets/screenshot.png" />
 </p>
 
-## Save the state
-Putting the tenant cluster control plane in a pod is the easiest part. Also, we have to make sure each tenant cluster saves the state to be able to store and retrieve data.
+## Getting started
 
-A dedicated `etcd` cluster for each tenant cluster doesn’t scale well for a managed service because `etcd` data persistence can be cumbersome at scale, rising the operational effort to mitigate it. So we have to find an alternative keeping in mind our goal for a resilient and cost-optimized solution at the same time. As we can deploy any Kubernetes cluster with an external `etcd` cluster, we explored this option for the tenant control planes. On the admin cluster, we deploy a multi-tenant `etcd` cluster storing the state of multiple tenant clusters.
+Please refer to the [Getting Started guide](./docs/getting-started-with-kamaji.md) to deploy a minimal setup of Kamaji on KinD.
 
-With this solution, the resiliency is guaranteed by the usual `etcd` mechanism, and the pods' count remains under control, so it solves the main goal of resiliency and costs optimization. The trade-off here is that we have to operate an external `etcd` cluster and manage the access to be sure that each tenant cluster uses only its data. Also, there are limits in size in `etcd`, defaulted to 2GB and configurable to a maximum of 8GB. We’re solving this issue by pooling multiple `etcd` and sharding the tenant control planes.
+> This project is still in the early development stage which means it's not ready for production as APIs, commands, flags, etc. are subject to change, but also that your feedback can still help to shape it. Please try it out and let us know what you like, dislike, what works, what doesn't, etc.
 
 ## Use cases
-Kamaji project has been initially started as a solution for actual and common problems such as minimizing the Total Cost of Ownership while running Kubernetes at scale. However, it can open a wider range of use cases. Here are a few:
+Kamaji project has been initially started as a solution for actual and common problems such as minimizing the Total Cost of Ownership while running Kubernetes at large scale. However, it can open a wider range of use cases.
 
-### Managed Kubernetes
-Enabling companies to provide Cloud Native Infrastructure with ease by introducing a strong separation of concerns between management and workloads. Centralize clusters management, monitoring, and observability by leaving developers to focus on the applications, increase productivity and reduce operational costs. 
+Here are a few:
 
-### Control Plane as a Service
-Provide Kubernetes control plane in a self-service fashion by running management and workloads on different infrastructures and cost centers with the option of Bring Your Own Device - BYOD.
-
-### Edge Computing
-Distribute Kubernetes workloads across edge computing locations without having to manage multiple clusters across various providers. Centralize management of hundreds of control planes while leaving workloads to run isolated on their own dedicated infrastructure.
-
-### Cluster Simulations
-Test a new Kubernetes API or experimental flag or a new tool without impacting production operations. Kamaji will let you simulate such things in a safe and controlled environment.
+- **Managed Kubernetes:** enable companies to provide Cloud Native Infrastructure with ease by introducing a strong separation of concerns between management and workloads. Centralize clusters management, monitoring, and observability by leaving developers to focus on applications, increase productivity and reduce operational costs.
+- **Kubernetes as a Service:** provide Kubernetes clusters in a self-service fashion by running management and workloads on different infrastructures with the option of Bring Your Own Device, BYOD.
+- **Control Plane as a Service:** provide multiple Kubernetes control planes running on top of a single Kubernetes cluster. Tenants who use namespaces based isolation often still need access to cluster wide resources like Cluster Roles, Admission Webhooks, or Custom Resource Definitions.
+- **Edge Computing:** distribute Kubernetes workloads across edge computing locations without having to manage multiple clusters across various providers. Centralize management of hundreds of control planes while leaving workloads to run isolated on their own dedicated infrastructure.
+- **Cluster Simulation:** check new Kubernetes API or experimental flag or a new tool without impacting production operations. Kamaji will let you simulate such things in a safe and controlled environment.
+- **Workloads Testing:** check the behaviour of your workloads on different and multiple versions of Kubernetes with ease by deploying multiple Control Planes in a single cluster.
 
 ## Features
 
-### Self Service Kubernetes
-Leave users the freedom to self-provision their Kubernetes clusters according to the assigned boundaries.
-
-### Multi-cluster Management
-Centrally manage multiple tenant clusters from a single admin cluster. Happy SREs. 
-
-### Cheaper Control Planes
-Place multiple tenant control planes on a single node, instead of having three nodes for a single control plane.
-
-### Stronger Multi-Tenancy
-Leave tenants to access the control plane with admin permissions while keeping the tenant isolated at the infrastructure level.
-
-### Kubernetes Inception
-Use Kubernetes to manage Kubernetes by re-using all the Kubernetes goodies you already know and love.
-
-### Full APIs compliant
-Tenant clusters are fully CNCF compliant built with upstream Kubernetes binaries. A client does not see differences between a Kamaji provisioned cluster and a dedicated cluster.
+- **Self Service Kubernetes:** leave users the freedom to self-provision their Kubernetes clusters according to the assigned boundaries.
+- **Multi-cluster Management:** centrally manage multiple tenant clusters from a single admin cluster. Happy SREs. 
+- **Cheaper Control Planes:** place multiple tenant control planes on a single node, instead of having three nodes for a single control plane.
+- **Stronger Multi-Tenancy:** leave tenants to access the control plane with admin permissions while keeping the tenant isolated at the infrastructure level.
+- **Kubernetes Inception:** use Kubernetes to manage Kubernetes by re-using all the Kubernetes goodies you already know and love.
+- **Full APIs compliant:** tenant clusters are fully CNCF compliant built with upstream Kubernetes binaries. A user does not see differences between a Kamaji provisioned cluster and a dedicated cluster.
 
 ## Roadmap
 
 - [ ] Benchmarking and stress-test
 - [x] Support for dynamic address allocation on native Load Balancer
 - [x] Zero Downtime Tenant Control Plane upgrade
-- [ ] `konnectivity` integration
+- [x] `konnectivity` integration
 - [ ] Provisioning of Tenant Control Plane through Cluster APIs
-- [ ] Prometheus metrics for monitoring and alerting
-- [ ] `kine` integration, i.e. use MySQL, SQLite, PostgreSQL as datastore
+- [ ] Terraform provider
+- [ ] Custom Prometheus metrics for monitoring and alerting
+- [x] `kine` integration for MySQL as datastore
+- [x] `kine` integration for PostgreSQL as datastore
 - [ ] Deeper `kubeadm` integration
-- [ ] `etcd` pooling
-- [ ] Tenant Control Planes sharding
+- [ ] Pooling of multiple `etcd` datastores
+- [ ] Autoscaling of Tenant Control Plane pods
+
 
 ## Documentation
 Please, check the project's [documentation](./docs/) for getting started with Kamaji.
@@ -96,27 +85,26 @@ Please, check the project's [documentation](./docs/) for getting started with Ka
 ## Contributions
 Kamaji is Open Source with Apache 2 license and any contribution is welcome.
 
-## FAQ
+## Community
+Join the [Kubernetes Slack Workspace](https://slack.k8s.io/) and the [`#kamaji`](https://kubernetes.slack.com/archives/C03GLTTMWNN) channel to meet end-users and contributors.
+
+## FAQs
 Q. What does Kamaji means?
 
 A. Kamaji is named as the character _Kamaji_ from the Japanese movie [_Spirited Away_](https://en.wikipedia.org/wiki/Spirited_Away).
 
 Q. Is Kamaji another Kubernetes distribution?
 
-A. No, Kamaji is a tool you can install on top of any CNCF conformant Kubernetes to provide hundreds of managed Tenant clusters as a service. We tested Kamaji on vanilla Kubernetes 1.23+, KinD, and MS Azure AKS. We expect it to work smoothly on other Kubernetes distributions. The tenant clusters made with Kamaji are conformant CNCF Kubernetes vanilla clusters built with `kubeadm`.
+A. No, Kamaji is a Kubernetes Operator you can install on top of any Kubernetes cluster to provide hundreds of managed Kubernetes clusters as a service. We tested Kamaji on vanilla Kubernetes 1.22+, KinD, and Azure AKS. We expect it to work smoothly on other Kubernetes distributions. The tenant clusters made with Kamaji are conformant CNCF Kubernetes clusters as we leverage on [`kubeadm`](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/).
 
-Q. Is it safe to run Kubernetes control plane components in a pod?
+Q. Is it safe to run Kubernetes control plane components in a pod instead of dedicated virtual machines?
 
-A. Yes, the tenant control plane components are packaged in the same way they are running in bare metal or virtual nodes. We leverage the `kubeadm` code to set up the control plane components as they were running on a server. The same unchanged images of upstream `APIs server`, `scheduler`, `controller manager` are used.
+A. Yes, the tenant control plane components are packaged in the same way they are running in bare metal or virtual nodes. We leverage the `kubeadm` code to set up the control plane components as they were running on their own server. The unchanged images of upstream `kube-apiserver`, `kube-scheduler`, and `kube-controller-manager` are used.
 
-Q. And what about multi-tenant `etcd`? I never heard of it.
+Q. You already provide a Kubernetes multi-tenancy solution with [Capsule](https://capsule.clastix.io). Why does Kamaji matter?
 
-A. Even if multi-tenant deployment for `etcd` is not a common practice, multi-tenancy, RBAC, and client authentication has been [supported](https://etcd.io/docs/v3.5/op-guide/authentication/) in `etcd` from a long time. 
+A. A multi-tenancy solution, like Capsule shares the Kubernetes control plane among all tenants keeping tenant namespaces isolated by policies. While the solution is the right choice by balancing between features and ease of usage, there are cases where a tenant user requires access to the control plane, for example, when a tenant requires to manage CRDs on his own. With Kamaji, you can provide cluster admin permissions to the tenant.
 
-Q. You already provide a Kubernetes multi-tenancy solution with [Capsule](capsule.clastix.io). Why does Kamaji matter?
+Q. Well you convinced me, how to get a try?
 
-A. Lighter Multi-Tenancy solutions, like Capsule shares the Kubernetes control plane among all tenants keeping tenant namespaces isolated by policies. While these solutions are the right choice by balancing between features and ease of usage, there are cases where a tenant user requires access to the control plane, for example, when a tenant requires to manage CRDs on his own. With Kamaji, you can provide admin permissions to the tenants.
-
-Q. So I need a costly cloud infrastructure to try Kamaji?
-
-A. No, it is possible to try Kamaji on your laptop with [KinD](./deploy/kind/README.md).
+A. It is possible to get started with Kamaji on a laptop with [KinD](./docs/getting-started-with-kamaji.md) installed.

api/v1alpha1/tenantcontrolplane_addons_funcs.go

@@ -0,0 +1,17 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package v1alpha1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func (in AddonStatus) GetChecksum() string {
+	return in.Checksum
+}
+
+func (in *AddonStatus) SetChecksum(checksum string) {
+	in.LastUpdate = metav1.Now()
+	in.Checksum = checksum
+}

api/v1alpha1/tenantcontrolplane_funcs.go

@@ -6,14 +6,41 @@ package v1alpha1
 import (
 	"context"
 	"fmt"
+	"net"
+	"strconv"
 
 	"github.com/pkg/errors"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	kamajierrors "github.com/clastix/kamaji/internal/errors"
 )
 
-func (in *TenantControlPlane) GetAddress(ctx context.Context, client client.Client) (string, error) {
+// AssignedControlPlaneAddress returns the announced address and port of a Tenant Control Plane.
+// In case of non-well formed values, or missing announcement, an error is returned.
+func (in *TenantControlPlane) AssignedControlPlaneAddress() (string, int32, error) {
+	if len(in.Status.ControlPlaneEndpoint) == 0 {
+		return "", 0, fmt.Errorf("the Tenant Control Plane is not yet exposed")
+	}
+
+	address, portString, err := net.SplitHostPort(in.Status.ControlPlaneEndpoint)
+	if err != nil {
+		return "", 0, errors.Wrap(err, "cannot split host port from Tenant Control Plane endpoint")
+	}
+
+	port, err := strconv.Atoi(portString)
+	if err != nil {
+		return "", 0, errors.Wrap(err, "cannot convert Tenant Control Plane port from endpoint")
+	}
+
+	return address, int32(port), nil
+}
+
+// DeclaredControlPlaneAddress returns the desired Tenant Control Plane address.
+// In case of dynamic allocation, e.g. using a Load Balancer, it queries the API Server looking for the allocated IP.
+// When an IP has not been yet assigned, or it is expected, an error is returned.
+func (in *TenantControlPlane) DeclaredControlPlaneAddress(ctx context.Context, client client.Client) (string, error) {
 	var loadBalancerStatus corev1.LoadBalancerStatus
 
 	svc := &corev1.Service{}
@@ -26,10 +53,12 @@ func (in *TenantControlPlane) GetAddress(ctx context.Context, client client.Clie
 	case len(in.Spec.NetworkProfile.Address) > 0:
 		// Returning the hard-coded value in the specification in case of non LoadBalanced resources
 		return in.Spec.NetworkProfile.Address, nil
+	case svc.Spec.Type == corev1.ServiceTypeClusterIP:
+		return svc.Spec.ClusterIP, nil
 	case svc.Spec.Type == corev1.ServiceTypeLoadBalancer:
 		loadBalancerStatus = svc.Status.LoadBalancer
 		if len(loadBalancerStatus.Ingress) == 0 {
-			return "", fmt.Errorf("cannot retrieve the TenantControlPlane address, Service resource is not yet exposed as LoadBalancer")
+			return "", kamajierrors.NonExposedLoadBalancerError{}
 		}
 
 		for _, lb := range loadBalancerStatus.Ingress {
@@ -39,5 +68,5 @@ func (in *TenantControlPlane) GetAddress(ctx context.Context, client client.Clie
 		}
 	}
 
-	return "", fmt.Errorf("the actual resource doesn't have yet a valid IP address")
+	return "", kamajierrors.MissingValidIPError{}
 }

api/v1alpha1/tenantcontrolplane_interfaces.go

@@ -0,0 +1,12 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package v1alpha1
+
+// KubeadmConfigChecksumDependant is the interface used to retrieve the checksum of the kubeadm phases and addons
+// configuration, required to validate the changes and, upon from that, perform the required reconciliation.
+// +kubebuilder:object:generate=false
+type KubeadmConfigChecksumDependant interface {
+	GetChecksum() string
+	SetChecksum(string)
+}

api/v1alpha1/tenantcontrolplane_kubeadmphase_funcs.go

@@ -0,0 +1,17 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package v1alpha1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func (in KubeadmPhaseStatus) GetChecksum() string {
+	return in.Checksum
+}
+
+func (in *KubeadmPhaseStatus) SetChecksum(checksum string) {
+	in.LastUpdate = metav1.Now()
+	in.Checksum = checksum
+}

api/v1alpha1/tenantcontrolplane_status.go

@@ -0,0 +1,250 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package v1alpha1
+
+import (
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	networkingv1 "k8s.io/api/networking/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/clastix/kamaji/internal/etcd"
+)
+
+// APIServerCertificatesStatus defines the observed state of ETCD Certificate for API server.
+type APIServerCertificatesStatus struct {
+	SecretName string      `json:"secretName,omitempty"`
+	LastUpdate metav1.Time `json:"lastUpdate,omitempty"`
+	Checksum   string      `json:"checksum,omitempty"`
+}
+
+// ETCDCertificateStatus defines the observed state of ETCD Certificate for API server.
+type ETCDCertificateStatus struct {
+	SecretName string      `json:"secretName,omitempty"`
+	LastUpdate metav1.Time `json:"lastUpdate,omitempty"`
+	Checksum   string      `json:"checksum,omitempty"`
+}
+
+// ETCDCertificatesStatus defines the observed state of ETCD Certificate for API server.
+type ETCDCertificatesStatus struct {
+	APIServer APIServerCertificatesStatus `json:"apiServer,omitempty"`
+	CA        ETCDCertificateStatus       `json:"ca,omitempty"`
+}
+
+// CertificatePrivateKeyPairStatus defines the status.
+type CertificatePrivateKeyPairStatus struct {
+	SecretName string      `json:"secretName,omitempty"`
+	LastUpdate metav1.Time `json:"lastUpdate,omitempty"`
+	Checksum   string      `json:"checksum,omitempty"`
+}
+
+// PublicKeyPrivateKeyPairStatus defines the status.
+type PublicKeyPrivateKeyPairStatus struct {
+	SecretName string      `json:"secretName,omitempty"`
+	LastUpdate metav1.Time `json:"lastUpdate,omitempty"`
+	Checksum   string      `json:"checksum,omitempty"`
+}
+
+// CertificatesStatus defines the observed state of ETCD Certificates.
+type CertificatesStatus struct {
+	CA                     CertificatePrivateKeyPairStatus `json:"ca,omitempty"`
+	APIServer              CertificatePrivateKeyPairStatus `json:"apiServer,omitempty"`
+	APIServerKubeletClient CertificatePrivateKeyPairStatus `json:"apiServerKubeletClient,omitempty"`
+	FrontProxyCA           CertificatePrivateKeyPairStatus `json:"frontProxyCA,omitempty"`
+	FrontProxyClient       CertificatePrivateKeyPairStatus `json:"frontProxyClient,omitempty"`
+	SA                     PublicKeyPrivateKeyPairStatus   `json:"sa,omitempty"`
+	ETCD                   *ETCDCertificatesStatus         `json:"etcd,omitempty"`
+}
+
+// ETCDStatus defines the observed state of ETCDStatus.
+type ETCDStatus struct {
+	Role etcd.Role `json:"role,omitempty"`
+	User etcd.User `json:"user,omitempty"`
+}
+
+type SQLCertificateStatus struct {
+	SecretName string      `json:"secretName,omitempty"`
+	Checksum   string      `json:"checksum,omitempty"`
+	LastUpdate metav1.Time `json:"lastUpdate,omitempty"`
+}
+
+type SQLConfigStatus struct {
+	SecretName string `json:"secretName,omitempty"`
+	Checksum   string `json:"checksum,omitempty"`
+}
+
+type SQLSetupStatus struct {
+	Schema     string      `json:"schema,omitempty"`
+	User       string      `json:"user,omitempty"`
+	LastUpdate metav1.Time `json:"lastUpdate,omitempty"`
+	Checksum   string      `json:"checksum,omitempty"`
+}
+
+type KineStatus struct {
+	Driver      string               `json:"driver,omitempty"`
+	Config      SQLConfigStatus      `json:"config,omitempty"`
+	Setup       SQLSetupStatus       `json:"setup,omitempty"`
+	Certificate SQLCertificateStatus `json:"certificate,omitempty"`
+}
+
+// StorageStatus defines the observed state of StorageStatus.
+type StorageStatus struct {
+	ETCD *ETCDStatus `json:"etcd,omitempty"`
+	Kine *KineStatus `json:"kine,omitempty"`
+}
+
+// KubeconfigStatus contains information about the generated kubeconfig.
+type KubeconfigStatus struct {
+	SecretName string      `json:"secretName,omitempty"`
+	LastUpdate metav1.Time `json:"lastUpdate,omitempty"`
+	Checksum   string      `json:"checksum,omitempty"`
+}
+
+// KubeconfigsStatus stores information about all the generated kubeconfig resources.
+type KubeconfigsStatus struct {
+	Admin             KubeconfigStatus `json:"admin,omitempty"`
+	ControllerManager KubeconfigStatus `json:"controllerManager,omitempty"`
+	Scheduler         KubeconfigStatus `json:"scheduler,omitempty"`
+}
+
+// KubeadmConfigStatus contains the status of the configuration required by kubeadm.
+type KubeadmConfigStatus struct {
+	ConfigmapName string      `json:"configmapName,omitempty"`
+	LastUpdate    metav1.Time `json:"lastUpdate,omitempty"`
+	// Checksum of the kubeadm configuration to detect changes
+	Checksum string `json:"checksum,omitempty"`
+}
+
+// KubeadmPhaseStatus contains the status of a kubeadm phase action.
+type KubeadmPhaseStatus struct {
+	Checksum   string      `json:"checksum,omitempty"`
+	LastUpdate metav1.Time `json:"lastUpdate,omitempty"`
+}
+
+// KubeadmPhasesStatus contains the status of the different kubeadm phases action.
+type KubeadmPhasesStatus struct {
+	UploadConfigKubeadm KubeadmPhaseStatus `json:"uploadConfigKubeadm"`
+	UploadConfigKubelet KubeadmPhaseStatus `json:"uploadConfigKubelet"`
+	BootstrapToken      KubeadmPhaseStatus `json:"bootstrapToken"`
+}
+
+type ExternalKubernetesObjectStatus struct {
+	Name      string `json:"name,omitempty"`
+	Namespace string `json:"namespace,omitempty"`
+	Checksum  string `json:"checksum,omitempty"`
+	// Last time when k8s object was updated
+	LastUpdate metav1.Time `json:"lastUpdate,omitempty"`
+}
+
+// KonnectivityStatus defines the status of Konnectivity as Addon.
+type KonnectivityStatus struct {
+	Enabled            bool                            `json:"enabled"`
+	ConfigMap          KonnectivityConfigMap           `json:"configMap,omitempty"`
+	Certificate        CertificatePrivateKeyPairStatus `json:"certificate,omitempty"`
+	Kubeconfig         KubeconfigStatus                `json:"kubeconfig,omitempty"`
+	ServiceAccount     ExternalKubernetesObjectStatus  `json:"sa,omitempty"`
+	ClusterRoleBinding ExternalKubernetesObjectStatus  `json:"clusterrolebinding,omitempty"`
+	Agent              ExternalKubernetesObjectStatus  `json:"agent,omitempty"`
+	Service            KubernetesServiceStatus         `json:"service,omitempty"`
+}
+
+type KonnectivityConfigMap struct {
+	Name     string `json:"name,omitempty"`
+	Checksum string `json:"checksum,omitempty"`
+}
+
+// AddonStatus defines the observed state of an Addon.
+type AddonStatus struct {
+	Enabled    bool        `json:"enabled"`
+	Checksum   string      `json:"checksum,omitempty"`
+	LastUpdate metav1.Time `json:"lastUpdate,omitempty"`
+}
+
+// AddonsStatus defines the observed state of the different Addons.
+type AddonsStatus struct {
+	CoreDNS   AddonStatus `json:"coreDNS,omitempty"`
+	KubeProxy AddonStatus `json:"kubeProxy,omitempty"`
+
+	Konnectivity KonnectivityStatus `json:"konnectivity,omitempty"`
+}
+
+// TenantControlPlaneStatus defines the observed state of TenantControlPlane.
+type TenantControlPlaneStatus struct {
+	// Storage Status contains information about Kubernetes storage system
+	Storage StorageStatus `json:"storage,omitempty"`
+	// Certificates contains information about the different certificates
+	// that are necessary to run a kubernetes control plane
+	Certificates CertificatesStatus `json:"certificates,omitempty"`
+	// KubeConfig contains information about the kubenconfigs that control plane pieces need
+	KubeConfig KubeconfigsStatus `json:"kubeconfig,omitempty"`
+	// Kubernetes contains information about the reconciliation of the required Kubernetes resources deployed in the admin cluster
+	Kubernetes KubernetesStatus `json:"kubernetesResources,omitempty"`
+	// KubeadmConfig contains the status of the configuration required by kubeadm
+	KubeadmConfig KubeadmConfigStatus `json:"kubeadmconfig,omitempty"`
+	// KubeadmPhase contains the status of the kubeadm phases action
+	KubeadmPhase KubeadmPhasesStatus `json:"kubeadmPhase,omitempty"`
+	// ControlPlaneEndpoint contains the status of the kubernetes control plane
+	ControlPlaneEndpoint string `json:"controlPlaneEndpoint,omitempty"`
+	// Addons contains the status of the different Addons
+	Addons AddonsStatus `json:"addons,omitempty"`
+}
+
+// KubernetesStatus defines the status of the resources deployed in the management cluster,
+// such as Deployment and Service.
+type KubernetesStatus struct {
+	// KubernetesVersion contains the information regarding the running Kubernetes version, and its upgrade status.
+	Version    KubernetesVersion          `json:"version,omitempty"`
+	Deployment KubernetesDeploymentStatus `json:"deployment,omitempty"`
+	Service    KubernetesServiceStatus    `json:"service,omitempty"`
+	Ingress    *KubernetesIngressStatus   `json:"ingress,omitempty"`
+}
+
+// +kubebuilder:validation:Enum=Provisioning;Upgrading;Ready;NotReady
+type KubernetesVersionStatus string
+
+var (
+	VersionProvisioning KubernetesVersionStatus = "Provisioning"
+	VersionUpgrading    KubernetesVersionStatus = "Upgrading"
+	VersionReady        KubernetesVersionStatus = "Ready"
+	VersionNotReady     KubernetesVersionStatus = "NotReady"
+)
+
+type KubernetesVersion struct {
+	// Version is the running Kubernetes version of the Tenant Control Plane.
+	Version string `json:"version,omitempty"`
+	// +kubebuilder:default=Provisioning
+	// Status returns the current status of the Kubernetes version, such as its provisioning state, or completed upgrade.
+	Status *KubernetesVersionStatus `json:"status,omitempty"`
+}
+
+// KubernetesDeploymentStatus defines the status for the Tenant Control Plane Deployment in the management cluster.
+type KubernetesDeploymentStatus struct {
+	appsv1.DeploymentStatus `json:",inline"`
+	// The name of the Deployment for the given cluster.
+	Name string `json:"name"`
+	// The namespace which the Deployment for the given cluster is deployed.
+	Namespace string `json:"namespace"`
+	// Last time when deployment was updated
+	LastUpdate metav1.Time `json:"lastUpdate,omitempty"`
+}
+
+// KubernetesServiceStatus defines the status for the Tenant Control Plane Service in the management cluster.
+type KubernetesServiceStatus struct {
+	corev1.ServiceStatus `json:",inline"`
+	// The name of the Service for the given cluster.
+	Name string `json:"name"`
+	// The namespace which the Service for the given cluster is deployed.
+	Namespace string `json:"namespace"`
+	// The port where the service is running
+	Port int32 `json:"port"`
+}
+
+// KubernetesIngressStatus defines the status for the Tenant Control Plane Ingress in the management cluster.
+type KubernetesIngressStatus struct {
+	networkingv1.IngressStatus `json:",inline"`
+	// The name of the Ingress for the given cluster.
+	Name string `json:"name"`
+	// The namespace which the Ingress for the given cluster is deployed.
+	Namespace string `json:"namespace"`
+}

api/v1alpha1/tenantcontrolplane_types.go

@@ -4,12 +4,8 @@
 package v1alpha1
 
 import (
-	appv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
-	networkingv1 "k8s.io/api/networking/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-
-	"github.com/clastix/kamaji/internal/etcd"
 )
 
 // NetworkProfileSpec defines the desired state of NetworkProfile.
@@ -22,15 +18,18 @@ type NetworkProfileSpec struct {
 	AllowAddressAsExternalIP bool `json:"allowAddressAsExternalIP,omitempty"`
 	// Port where API server of will be exposed
 	// +kubebuilder:default=6443
-	Port int32 `json:"port"`
-
-	// Domain of the tenant control plane
-	Domain string `json:"domain"`
+	Port int32 `json:"port,omitempty"`
+	// CertSANs sets extra Subject Alternative Names (SANs) for the API Server signing certificate.
+	// Use this field to add additional hostnames when exposing the Tenant Control Plane with third solutions.
+	CertSANs []string `json:"certSANs,omitempty"`
 	// Kubernetes Service
-	ServiceCIDR string `json:"serviceCidr"`
+	// +kubebuilder:default="10.96.0.0/16"
+	ServiceCIDR string `json:"serviceCidr,omitempty"`
 	// CIDR for Kubernetes Pods
-	PodCIDR       string   `json:"podCidr"`
-	DNSServiceIPs []string `json:"dnsServiceIPs"`
+	// +kubebuilder:default="10.244.0.0/16"
+	PodCIDR string `json:"podCidr,omitempty"`
+	// +kubebuilder:default={"10.96.0.10"}
+	DNSServiceIPs []string `json:"dnsServiceIPs,omitempty"`
 }
 
 type KubeletSpec struct {
@@ -65,23 +64,43 @@ type ControlPlane struct {
 	// Defining the options for the Tenant Control Plane Service resource.
 	Service ServiceSpec `json:"service"`
 	// Defining the options for an Optional Ingress which will expose API Server of the Tenant Control Plane
-	Ingress IngressSpec `json:"ingress,omitempty"`
+	Ingress *IngressSpec `json:"ingress,omitempty"`
 }
 
 // IngressSpec defines the options for the ingress which will expose API Server of the Tenant Control Plane.
 type IngressSpec struct {
 	AdditionalMetadata AdditionalMetadata `json:"additionalMetadata,omitempty"`
-	Enabled            bool               `json:"enabled"`
 	IngressClassName   string             `json:"ingressClassName,omitempty"`
 	// Hostname is an optional field which will be used as Ingress's Host. If it is not defined,
 	// Ingress's host will be "<tenant>.<namespace>.<domain>", where domain is specified under NetworkProfileSpec
 	Hostname string `json:"hostname,omitempty"`
 }
 
+type ControlPlaneComponentsResources struct {
+	APIServer         *corev1.ResourceRequirements `json:"apiServer,omitempty"`
+	ControllerManager *corev1.ResourceRequirements `json:"controllerManager,omitempty"`
+	Scheduler         *corev1.ResourceRequirements `json:"scheduler,omitempty"`
+}
+
 type DeploymentSpec struct {
 	// +kubebuilder:default=2
-	Replicas           int32              `json:"replicas,omitempty"`
-	AdditionalMetadata AdditionalMetadata `json:"additionalMetadata,omitempty"`
+	Replicas int32 `json:"replicas,omitempty"`
+	// Resources defines the amount of memory and CPU to allocate to each component of the Control Plane
+	// (kube-apiserver, controller-manager, and scheduler).
+	Resources *ControlPlaneComponentsResources `json:"resources,omitempty"`
+	// ExtraArgs allows adding additional arguments to the Control Plane components,
+	// such as kube-apiserver, controller-manager, and scheduler.
+	ExtraArgs          *ControlPlaneExtraArgs `json:"extraArgs,omitempty"`
+	AdditionalMetadata AdditionalMetadata     `json:"additionalMetadata,omitempty"`
+}
+
+// ControlPlaneExtraArgs allows specifying additional arguments to the Control Plane components.
+type ControlPlaneExtraArgs struct {
+	APIServer         []string `json:"apiServer,omitempty"`
+	ControllerManager []string `json:"controllerManager,omitempty"`
+	Scheduler         []string `json:"scheduler,omitempty"`
+	// Available only if Kamaji is running using Kine as backing storage.
+	Kine []string `json:"kine,omitempty"`
 }
 
 type ServiceSpec struct {
@@ -90,178 +109,42 @@ type ServiceSpec struct {
 	ServiceType ServiceType `json:"serviceType"`
 }
 
+// AddonSpec defines the spec for every addon.
+type AddonSpec struct{}
+
+// KonnectivitySpec defines the spec for Konnectivity.
+type KonnectivitySpec struct {
+	// Port of Konnectivity proxy server.
+	ProxyPort int32 `json:"proxyPort"`
+	// Version for Konnectivity server and agent.
+	// +kubebuilder:default=v0.0.31
+	Version string `json:"version,omitempty"`
+	// ServerImage defines the container image for Konnectivity's server.
+	// +kubebuilder:default=us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-server
+	ServerImage string `json:"serverImage,omitempty"`
+	// AgentImage defines the container image for Konnectivity's agent.
+	// +kubebuilder:default=us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-agent
+	AgentImage string `json:"agentImage,omitempty"`
+	// Resources define the amount of CPU and memory to allocate to the Konnectivity server.
+	Resources *corev1.ResourceRequirements `json:"resources,omitempty"`
+}
+
+// AddonsSpec defines the enabled addons and their features.
+type AddonsSpec struct {
+	CoreDNS      *AddonSpec        `json:"coreDNS,omitempty"`
+	Konnectivity *KonnectivitySpec `json:"konnectivity,omitempty"`
+	KubeProxy    *AddonSpec        `json:"kubeProxy,omitempty"`
+}
+
 // TenantControlPlaneSpec defines the desired state of TenantControlPlane.
 type TenantControlPlaneSpec struct {
 	ControlPlane ControlPlane `json:"controlPlane"`
-
 	// Kubernetes specification for tenant control plane
 	Kubernetes KubernetesSpec `json:"kubernetes"`
-
 	// NetworkProfile specifies how the network is
 	NetworkProfile NetworkProfileSpec `json:"networkProfile,omitempty"`
-}
-
-// ETCDAPIServerCertificate defines the observed state of ETCD Certificate for API server.
-type APIServerCertificatesStatus struct {
-	SecretName string      `json:"secretName,omitempty"`
-	LastUpdate metav1.Time `json:"lastUpdate,omitempty"`
-}
-
-// ETCDAPIServerCertificate defines the observed state of ETCD Certificate for API server.
-type ETCDCertificateStatus struct {
-	SecretName string      `json:"secretName,omitempty"`
-	LastUpdate metav1.Time `json:"lastUpdate,omitempty"`
-}
-
-// ETCDAPIServerCertificate defines the observed state of ETCD Certificate for API server.
-type ETCDCertificatesStatus struct {
-	APIServer ETCDCertificateStatus `json:"apiServer,omitempty"`
-	CA        ETCDCertificateStatus `json:"ca,omitempty"`
-}
-
-// CertificatePrivateKeyPair defines the status.
-type CertificatePrivateKeyPairStatus struct {
-	SecretName string      `json:"secretName,omitempty"`
-	LastUpdate metav1.Time `json:"lastUpdate,omitempty"`
-}
-
-// CertificatePrivateKeyPair defines the status.
-type PublicKeyPrivateKeyPairStatus struct {
-	SecretName string      `json:"secretName,omitempty"`
-	LastUpdate metav1.Time `json:"lastUpdate,omitempty"`
-}
-
-// ETCDCertificates defines the observed state of ETCD Certificates.
-type CertificatesStatus struct {
-	CA                     CertificatePrivateKeyPairStatus `json:"ca,omitempty"`
-	APIServer              CertificatePrivateKeyPairStatus `json:"apiServer,omitempty"`
-	APIServerKubeletClient CertificatePrivateKeyPairStatus `json:"apiServerKubeletClient,omitempty"`
-	FrontProxyCA           CertificatePrivateKeyPairStatus `json:"frontProxyCA,omitempty"`
-	FrontProxyClient       CertificatePrivateKeyPairStatus `json:"frontProxyClient,omitempty"`
-	SA                     PublicKeyPrivateKeyPairStatus   `json:"sa,omitempty"`
-	ETCD                   *ETCDCertificatesStatus         `json:"etcd,omitempty"`
-}
-
-// ETCDStatus defines the observed state of ETCDStatus.
-type ETCDStatus struct {
-	Role etcd.Role `json:"role,omitempty"`
-	User etcd.User `json:"user,omitempty"`
-}
-
-// StorageStatus defines the observed state of StorageStatus.
-type StorageStatus struct {
-	ETCD *ETCDStatus `json:"etcd,omitempty"`
-}
-
-// TenantControlPlaneKubeconfigsStatus contains information about a the generated kubeconfig.
-type KubeconfigStatus struct {
-	SecretName string      `json:"secretName,omitempty"`
-	LastUpdate metav1.Time `json:"lastUpdate,omitempty"`
-}
-
-// TenantControlPlaneKubeconfigsStatus stores information about all the generated kubeconfigs.
-type KubeconfigsStatus struct {
-	Admin             KubeconfigStatus `json:"admin,omitempty"`
-	ControllerManager KubeconfigStatus `json:"controlerManager,omitempty"`
-	Scheduler         KubeconfigStatus `json:"scheduler,omitempty"`
-}
-
-// KubeadmConfigStatus contains the status of the configuration required by kubeadm.
-type KubeadmConfigStatus struct {
-	ConfigmapName   string      `json:"configmapName,omitempty"`
-	LastUpdate      metav1.Time `json:"lastUpdate,omitempty"`
-	ResourceVersion string      `json:"resourceVersion"`
-}
-
-// KubeadmPhasesStatus contains the status of of a kubeadm phase action.
-type KubeadmPhaseStatus struct {
-	KubeadmConfigResourceVersion string      `json:"kubeadmConfigResourceVersion,omitempty"`
-	LastUpdate                   metav1.Time `json:"lastUpdate,omitempty"`
-}
-
-// KubeadmPhasesStatus contains the status of the different kubeadm phases action.
-type KubeadmPhasesStatus struct {
-	UploadConfigKubeadm KubeadmPhaseStatus `json:"uploadConfigKubeadm"`
-	UploadConfigKubelet KubeadmPhaseStatus `json:"uploadConfigKubelet"`
-	AddonCoreDNS        KubeadmPhaseStatus `json:"addonCoreDNS"`
-	AddonKubeProxy      KubeadmPhaseStatus `json:"addonKubeProxy"`
-	BootstrapToken      KubeadmPhaseStatus `json:"bootstrapToken"`
-}
-
-// TenantControlPlaneStatus defines the observed state of TenantControlPlane.
-type TenantControlPlaneStatus struct {
-	// Storage Status contains information about Kubernetes storage system
-	Storage StorageStatus `json:"storage,omitempty"`
-	// Certificates contains information about the different certificates
-	// that are necessary to run a kubernetes control plane
-	Certificates CertificatesStatus `json:"certificates,omitempty"`
-	// KubeConfig contains information about the kubenconfigs that control plane pieces need
-	KubeConfig KubeconfigsStatus `json:"kubeconfig,omitempty"`
-	// Kubernetes contains information about the reconciliation of the required Kubernetes resources deployed in the admin cluster
-	Kubernetes KubernetesStatus `json:"kubernetesResources,omitempty"`
-	// KubeadmConfig contains the status of the configuration required by kubeadm
-	KubeadmConfig KubeadmConfigStatus `json:"kubeadmconfig,omitempty"`
-	// KubeadmPhase contains the status of the kubeadm phases action
-	KubeadmPhase KubeadmPhasesStatus `json:"kubeadmPhase,omitempty"`
-	// ControlPlaneEndpoint contains the status of the kubernetes control plane
-	ControlPlaneEndpoint string `json:"controlPlaneEndpoint,omitempty"`
-}
-
-// KubernetesStatus defines the status of the resources deployed in the management cluster,
-// such as Deployment and Service.
-type KubernetesStatus struct {
-	// KubernetesVersion contains the information regarding the running Kubernetes version, and its upgrade status.
-	Version    KubernetesVersion          `json:"version,omitempty"`
-	Deployment KubernetesDeploymentStatus `json:"deployment,omitempty"`
-	Service    KubernetesServiceStatus    `json:"service,omitempty"`
-	Ingress    KubernetesIngressStatus    `json:"ingress,omitempty"`
-}
-
-// +kubebuilder:validation:Enum=Provisioning;Upgrading;Ready;NotReady
-type KubernetesVersionStatus string
-
-var (
-	VersionProvisioning KubernetesVersionStatus = "Provisioning"
-	VersionUpgrading    KubernetesVersionStatus = "Upgrading"
-	VersionReady        KubernetesVersionStatus = "Ready"
-	VersionNotReady     KubernetesVersionStatus = "NotReady"
-)
-
-type KubernetesVersion struct {
-	// Version is the running Kubernetes version of the Tenant Control Plane.
-	Version string `json:"version,omitempty"`
-	// +kubebuilder:default=Provisioning
-	// Status returns the current status of the Kubernetes version, such as its provisioning state, or completed upgrade.
-	Status *KubernetesVersionStatus `json:"status"`
-}
-
-// KubernetesDeploymentStatus defines the status for the Tenant Control Plane Deployment in the management cluster.
-type KubernetesDeploymentStatus struct {
-	appv1.DeploymentStatus `json:",inline"`
-	// The name of the Deployment for the given cluster.
-	Name string `json:"name"`
-	// The namespace which the Deployment for the given cluster is deployed.
-	Namespace string `json:"namespace"`
-}
-
-// KubernetesServiceStatus defines the status for the Tenant Control Plane Service in the management cluster.
-type KubernetesServiceStatus struct {
-	corev1.ServiceStatus `json:",inline"`
-	// The name of the Service for the given cluster.
-	Name string `json:"name"`
-	// The namespace which the Service for the given cluster is deployed.
-	Namespace string `json:"namespace"`
-	// The port where the service is running
-	Port int32 `json:"port"`
-}
-
-// KubernetesIngressStatus defines the status for the Tenant Control Plane Ingress in the management cluster.
-type KubernetesIngressStatus struct {
-	networkingv1.IngressStatus `json:",inline"`
-	// The name of the Ingress for the given cluster.
-	Name string `json:"name"`
-	// The namespace which the Ingress for the given cluster is deployed.
-	Namespace string `json:"namespace"`
+	// Addons contain which addons are enabled
+	Addons AddonsSpec `json:"addons,omitempty"`
 }
 
 // +kubebuilder:object:root=true

api/v1alpha1/zz_generated.deepcopy.go

@@ -9,6 +9,7 @@
 package v1alpha1
 
 import (
+	"k8s.io/api/core/v1"
 	runtime "k8s.io/apimachinery/pkg/runtime"
 )
 
@@ -57,6 +58,85 @@ func (in *AdditionalMetadata) DeepCopy() *AdditionalMetadata {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *AddonSpec) DeepCopyInto(out *AddonSpec) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AddonSpec.
+func (in *AddonSpec) DeepCopy() *AddonSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(AddonSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *AddonStatus) DeepCopyInto(out *AddonStatus) {
+	*out = *in
+	in.LastUpdate.DeepCopyInto(&out.LastUpdate)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AddonStatus.
+func (in *AddonStatus) DeepCopy() *AddonStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(AddonStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *AddonsSpec) DeepCopyInto(out *AddonsSpec) {
+	*out = *in
+	if in.CoreDNS != nil {
+		in, out := &in.CoreDNS, &out.CoreDNS
+		*out = new(AddonSpec)
+		**out = **in
+	}
+	if in.Konnectivity != nil {
+		in, out := &in.Konnectivity, &out.Konnectivity
+		*out = new(KonnectivitySpec)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.KubeProxy != nil {
+		in, out := &in.KubeProxy, &out.KubeProxy
+		*out = new(AddonSpec)
+		**out = **in
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AddonsSpec.
+func (in *AddonsSpec) DeepCopy() *AddonsSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(AddonsSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *AddonsStatus) DeepCopyInto(out *AddonsStatus) {
+	*out = *in
+	in.CoreDNS.DeepCopyInto(&out.CoreDNS)
+	in.KubeProxy.DeepCopyInto(&out.KubeProxy)
+	in.Konnectivity.DeepCopyInto(&out.Konnectivity)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AddonsStatus.
+func (in *AddonsStatus) DeepCopy() *AddonsStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(AddonsStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in AdmissionControllers) DeepCopyInto(out *AdmissionControllers) {
 	{
@@ -123,7 +203,11 @@ func (in *ControlPlane) DeepCopyInto(out *ControlPlane) {
 	*out = *in
 	in.Deployment.DeepCopyInto(&out.Deployment)
 	in.Service.DeepCopyInto(&out.Service)
-	in.Ingress.DeepCopyInto(&out.Ingress)
+	if in.Ingress != nil {
+		in, out := &in.Ingress, &out.Ingress
+		*out = new(IngressSpec)
+		(*in).DeepCopyInto(*out)
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ControlPlane.
@@ -136,9 +220,84 @@ func (in *ControlPlane) DeepCopy() *ControlPlane {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ControlPlaneComponentsResources) DeepCopyInto(out *ControlPlaneComponentsResources) {
+	*out = *in
+	if in.APIServer != nil {
+		in, out := &in.APIServer, &out.APIServer
+		*out = new(v1.ResourceRequirements)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.ControllerManager != nil {
+		in, out := &in.ControllerManager, &out.ControllerManager
+		*out = new(v1.ResourceRequirements)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.Scheduler != nil {
+		in, out := &in.Scheduler, &out.Scheduler
+		*out = new(v1.ResourceRequirements)
+		(*in).DeepCopyInto(*out)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ControlPlaneComponentsResources.
+func (in *ControlPlaneComponentsResources) DeepCopy() *ControlPlaneComponentsResources {
+	if in == nil {
+		return nil
+	}
+	out := new(ControlPlaneComponentsResources)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ControlPlaneExtraArgs) DeepCopyInto(out *ControlPlaneExtraArgs) {
+	*out = *in
+	if in.APIServer != nil {
+		in, out := &in.APIServer, &out.APIServer
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
+	if in.ControllerManager != nil {
+		in, out := &in.ControllerManager, &out.ControllerManager
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
+	if in.Scheduler != nil {
+		in, out := &in.Scheduler, &out.Scheduler
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
+	if in.Kine != nil {
+		in, out := &in.Kine, &out.Kine
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ControlPlaneExtraArgs.
+func (in *ControlPlaneExtraArgs) DeepCopy() *ControlPlaneExtraArgs {
+	if in == nil {
+		return nil
+	}
+	out := new(ControlPlaneExtraArgs)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *DeploymentSpec) DeepCopyInto(out *DeploymentSpec) {
 	*out = *in
+	if in.Resources != nil {
+		in, out := &in.Resources, &out.Resources
+		*out = new(ControlPlaneComponentsResources)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.ExtraArgs != nil {
+		in, out := &in.ExtraArgs, &out.ExtraArgs
+		*out = new(ControlPlaneExtraArgs)
+		(*in).DeepCopyInto(*out)
+	}
 	in.AdditionalMetadata.DeepCopyInto(&out.AdditionalMetadata)
 }
 
@@ -202,6 +361,22 @@ func (in *ETCDStatus) DeepCopy() *ETCDStatus {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ExternalKubernetesObjectStatus) DeepCopyInto(out *ExternalKubernetesObjectStatus) {
+	*out = *in
+	in.LastUpdate.DeepCopyInto(&out.LastUpdate)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExternalKubernetesObjectStatus.
+func (in *ExternalKubernetesObjectStatus) DeepCopy() *ExternalKubernetesObjectStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(ExternalKubernetesObjectStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *IngressSpec) DeepCopyInto(out *IngressSpec) {
 	*out = *in
@@ -218,6 +393,81 @@ func (in *IngressSpec) DeepCopy() *IngressSpec {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *KineStatus) DeepCopyInto(out *KineStatus) {
+	*out = *in
+	out.Config = in.Config
+	in.Setup.DeepCopyInto(&out.Setup)
+	in.Certificate.DeepCopyInto(&out.Certificate)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KineStatus.
+func (in *KineStatus) DeepCopy() *KineStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(KineStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *KonnectivityConfigMap) DeepCopyInto(out *KonnectivityConfigMap) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KonnectivityConfigMap.
+func (in *KonnectivityConfigMap) DeepCopy() *KonnectivityConfigMap {
+	if in == nil {
+		return nil
+	}
+	out := new(KonnectivityConfigMap)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *KonnectivitySpec) DeepCopyInto(out *KonnectivitySpec) {
+	*out = *in
+	if in.Resources != nil {
+		in, out := &in.Resources, &out.Resources
+		*out = new(v1.ResourceRequirements)
+		(*in).DeepCopyInto(*out)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KonnectivitySpec.
+func (in *KonnectivitySpec) DeepCopy() *KonnectivitySpec {
+	if in == nil {
+		return nil
+	}
+	out := new(KonnectivitySpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *KonnectivityStatus) DeepCopyInto(out *KonnectivityStatus) {
+	*out = *in
+	out.ConfigMap = in.ConfigMap
+	in.Certificate.DeepCopyInto(&out.Certificate)
+	in.Kubeconfig.DeepCopyInto(&out.Kubeconfig)
+	in.ServiceAccount.DeepCopyInto(&out.ServiceAccount)
+	in.ClusterRoleBinding.DeepCopyInto(&out.ClusterRoleBinding)
+	in.Agent.DeepCopyInto(&out.Agent)
+	in.Service.DeepCopyInto(&out.Service)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KonnectivityStatus.
+func (in *KonnectivityStatus) DeepCopy() *KonnectivityStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(KonnectivityStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *KubeadmConfigStatus) DeepCopyInto(out *KubeadmConfigStatus) {
 	*out = *in
@@ -255,8 +505,6 @@ func (in *KubeadmPhasesStatus) DeepCopyInto(out *KubeadmPhasesStatus) {
 	*out = *in
 	in.UploadConfigKubeadm.DeepCopyInto(&out.UploadConfigKubeadm)
 	in.UploadConfigKubelet.DeepCopyInto(&out.UploadConfigKubelet)
-	in.AddonCoreDNS.DeepCopyInto(&out.AddonCoreDNS)
-	in.AddonKubeProxy.DeepCopyInto(&out.AddonKubeProxy)
 	in.BootstrapToken.DeepCopyInto(&out.BootstrapToken)
 }
 
@@ -323,6 +571,7 @@ func (in *KubeletSpec) DeepCopy() *KubeletSpec {
 func (in *KubernetesDeploymentStatus) DeepCopyInto(out *KubernetesDeploymentStatus) {
 	*out = *in
 	in.DeploymentStatus.DeepCopyInto(&out.DeploymentStatus)
+	in.LastUpdate.DeepCopyInto(&out.LastUpdate)
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubernetesDeploymentStatus.
@@ -394,7 +643,11 @@ func (in *KubernetesStatus) DeepCopyInto(out *KubernetesStatus) {
 	in.Version.DeepCopyInto(&out.Version)
 	in.Deployment.DeepCopyInto(&out.Deployment)
 	in.Service.DeepCopyInto(&out.Service)
-	in.Ingress.DeepCopyInto(&out.Ingress)
+	if in.Ingress != nil {
+		in, out := &in.Ingress, &out.Ingress
+		*out = new(KubernetesIngressStatus)
+		(*in).DeepCopyInto(*out)
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubernetesStatus.
@@ -430,6 +683,11 @@ func (in *KubernetesVersion) DeepCopy() *KubernetesVersion {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *NetworkProfileSpec) DeepCopyInto(out *NetworkProfileSpec) {
 	*out = *in
+	if in.CertSANs != nil {
+		in, out := &in.CertSANs, &out.CertSANs
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
 	if in.DNSServiceIPs != nil {
 		in, out := &in.DNSServiceIPs, &out.DNSServiceIPs
 		*out = make([]string, len(*in))
@@ -463,6 +721,53 @@ func (in *PublicKeyPrivateKeyPairStatus) DeepCopy() *PublicKeyPrivateKeyPairStat
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *SQLCertificateStatus) DeepCopyInto(out *SQLCertificateStatus) {
+	*out = *in
+	in.LastUpdate.DeepCopyInto(&out.LastUpdate)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SQLCertificateStatus.
+func (in *SQLCertificateStatus) DeepCopy() *SQLCertificateStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(SQLCertificateStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *SQLConfigStatus) DeepCopyInto(out *SQLConfigStatus) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SQLConfigStatus.
+func (in *SQLConfigStatus) DeepCopy() *SQLConfigStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(SQLConfigStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *SQLSetupStatus) DeepCopyInto(out *SQLSetupStatus) {
+	*out = *in
+	in.LastUpdate.DeepCopyInto(&out.LastUpdate)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SQLSetupStatus.
+func (in *SQLSetupStatus) DeepCopy() *SQLSetupStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(SQLSetupStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ServiceSpec) DeepCopyInto(out *ServiceSpec) {
 	*out = *in
@@ -487,6 +792,11 @@ func (in *StorageStatus) DeepCopyInto(out *StorageStatus) {
 		*out = new(ETCDStatus)
 		(*in).DeepCopyInto(*out)
 	}
+	if in.Kine != nil {
+		in, out := &in.Kine, &out.Kine
+		*out = new(KineStatus)
+		(*in).DeepCopyInto(*out)
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StorageStatus.
@@ -564,6 +874,7 @@ func (in *TenantControlPlaneSpec) DeepCopyInto(out *TenantControlPlaneSpec) {
 	in.ControlPlane.DeepCopyInto(&out.ControlPlane)
 	in.Kubernetes.DeepCopyInto(&out.Kubernetes)
 	in.NetworkProfile.DeepCopyInto(&out.NetworkProfile)
+	in.Addons.DeepCopyInto(&out.Addons)
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TenantControlPlaneSpec.
@@ -585,6 +896,7 @@ func (in *TenantControlPlaneStatus) DeepCopyInto(out *TenantControlPlaneStatus)
 	in.Kubernetes.DeepCopyInto(&out.Kubernetes)
 	in.KubeadmConfig.DeepCopyInto(&out.KubeadmConfig)
 	in.KubeadmPhase.DeepCopyInto(&out.KubeadmPhase)
+	in.Addons.DeepCopyInto(&out.Addons)
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TenantControlPlaneStatus.

charts/kamaji/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v2
 name: kamaji
-description: A Kubernetes distribution aimed to build and operate a Managed Kubernetes service with a fraction of operational burde.
+description: Kamaji is a tool aimed to build and operate a Managed Kubernetes Service with a fraction of the operational burden. With Kamaji, you can deploy and operate hundreds of Kubernetes clusters as a hyper-scaler.
 
 # A chart can be either an 'application' or a 'library' chart.
 #
@@ -23,8 +23,8 @@ version: 0.1.0
 # It is recommended to use it with quotes.
 appVersion: 0.1.0
 
-home: https://github.com/clastix/kamaji-internal/tree/master/helm/kamaji
-sources: ["https://github.com/clastix/kamaji-internal"]
+home: https://github.com/clastix/kamaji
+sources: ["https://github.com/clastix/kamaji"]
 kubeVersion: ">=1.18"
 maintainers:
 - email: iam@mendrugory.com
@@ -33,3 +33,5 @@ maintainers:
   name: Dario Tranchitella
 - email: me@maxgio.it
   name: Massimiliano Giovagnoli
+- email: me@bsctl.io
+  name: Adriano Pezzuto

charts/kamaji/README.md

@@ -0,0 +1,114 @@
+# kamaji
+
+![Version: 0.1.1](https://img.shields.io/badge/Version-0.1.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.1.0](https://img.shields.io/badge/AppVersion-0.1.0-informational?style=flat-square)
+
+Kamaji is a tool aimed to build and operate a Managed Kubernetes Service with a fraction of the operational burden. With Kamaji, you can deploy and operate hundreds of Kubernetes clusters as a hyper-scaler.
+
+## Maintainers
+
+| Name | Email | Url |
+| ---- | ------ | --- |
+| Gonzalo Gabriel Jiménez Fuentes | <iam@mendrugory.com> |  |
+| Dario Tranchitella | <dario@tranchitella.eu> |  |
+| Massimiliano Giovagnoli | <me@maxgio.it> |  |
+| Adriano Pezzuto | <me@bsctl.io> |  |
+
+## Source Code
+
+* <https://github.com/clastix/kamaji>
+
+## Requirements
+
+Kubernetes: `>=1.18`
+
+[Kamaji](https://github.com/clastix/kamaji) requires a [multi-tenant `etcd`](https://github.com/clastix/kamaji-internal/blob/master/deploy/getting-started-with-kamaji.md#setup-internal-multi-tenant-etcd) cluster.
+This Helm Chart starting from v0.1.1 provides the installation of an internal `etcd` in order to streamline the local test. If you'd like to use an externally managed etcd instance, you can specify the overrides and by setting the value `etcd.deploy=false`.
+
+> For production use an externally managed `etcd` is highly recommended, the `etcd` addon offered by this Chart is not considered production-grade.
+
+## Install Kamaji
+
+To install the Chart with the release name `kamaji`:
+
+        helm upgrade --install --namespace kamaji-system --create-namespace clastix/kamaji
+
+Show the status:
+
+        helm status kamaji -n kamaji-system
+
+Upgrade the Chart
+
+        helm upgrade kamaji -n kamaji-system clastix/kamaji
+
+Uninstall the Chart
+
+        helm uninstall kamaji -n kamaji-system
+
+## Customize the installation
+
+There are two methods for specifying overrides of values during Chart installation: `--values` and `--set`.
+
+The `--values` option is the preferred method because it allows you to keep your overrides in a YAML file, rather than specifying them all on the command line. Create a copy of the YAML file `values.yaml` and add your overrides to it.
+
+Specify your overrides file when you install the Chart:
+
+        helm upgrade kamaji --install --namespace kamaji-system --create-namespace clastix/kamaji --values myvalues.yaml
+
+The values in your overrides file `myvalues.yaml` will override their counterparts in the Chart's values.yaml file. Any values in `values.yaml` that weren’t overridden will keep their defaults.
+
+If you only need to make minor customizations, you can specify them on the command line by using the `--set` option. For example:
+
+        helm upgrade kamaji --install --namespace kamaji-system --create-namespace clastix/kamaji --set etcd.deploy=false
+
+Here the values you can override:
+
+## Values
+
+| Key | Type | Default | Description |
+|-----|------|---------|-------------|
+| affinity | object | `{}` | Kubernetes affinity rules to apply to Kamaji controller pods |
+| configPath | string | `"./kamaji.yaml"` | Configuration file path alternative. (default "./kamaji.yaml") |
+| etcd.compactionInterval | int | `0` | ETCD Compaction interval (e.g. "5m0s"). (default: "0" (disabled)) |
+| etcd.deploy | bool | `true` | Install an etcd with enabled multi-tenancy along with Kamaji |
+| etcd.image | object | `{"pullPolicy":"IfNotPresent","repository":"quay.io/coreos/etcd","tag":"v3.5.4"}` | Install specific etcd image |
+| etcd.livenessProbe | object | `{"failureThreshold":8,"httpGet":{"path":"/health?serializable=true","port":2381,"scheme":"HTTP"},"initialDelaySeconds":10,"periodSeconds":10,"timeoutSeconds":15}` | The livenessProbe for the etcd container |
+| etcd.overrides.caSecret.name | string | `"etcd-certs"` | Name of the secret which contains CA's certificate and private key. (default: "etcd-certs") |
+| etcd.overrides.caSecret.namespace | string | `"kamaji-system"` | Namespace of the secret which contains CA's certificate and private key. (default: "kamaji-system") |
+| etcd.overrides.clientSecret.name | string | `"root-client-certs"` | Name of the secret which contains ETCD client certificates. (default: "root-client-certs") |
+| etcd.overrides.clientSecret.namespace | string | `"kamaji-system"` | Name of the namespace where the secret which contains ETCD client certificates is. (default: "kamaji-system") |
+| etcd.overrides.endpoints | object | `{"etcd-0":"https://etcd-0.etcd.kamaji-system.svc.cluster.local","etcd-1":"https://etcd-1.etcd.kamaji-system.svc.cluster.local","etcd-2":"https://etcd-2.etcd.kamaji-system.svc.cluster.local"}` | (map) Dictionary of the endpoints for the etcd cluster's members, key is the name of the etcd server. Don't define any port, inflected from .etcd.peerApiPort value. |
+| etcd.peerApiPort | int | `2380` | The peer API port which servers are listening to. |
+| etcd.persistence.accessModes[0] | string | `"ReadWriteOnce"` |  |
+| etcd.persistence.size | string | `"10Gi"` |  |
+| etcd.persistence.storageClass | string | `""` |  |
+| etcd.port | int | `2379` | The client request port. |
+| etcd.serviceAccount.create | bool | `true` | Create a ServiceAccount, required to install and provision the etcd backing storage (default: true) |
+| etcd.serviceAccount.name | string | `""` | Define the ServiceAccount name to use during the setup and provision of the etcd backing storage (default: "") |
+| extraArgs | list | `[]` | A list of extra arguments to add to the kamaji controller default ones |
+| fullnameOverride | string | `""` |  |
+| healthProbeBindAddress | string | `":8081"` | The address the probe endpoint binds to. (default ":8081") |
+| image.pullPolicy | string | `"Always"` |  |
+| image.repository | string | `"clastix/kamaji"` | The container image of the Kamaji controller. |
+| image.tag | string | `"latest"` |  |
+| imagePullSecrets | list | `[]` |  |
+| livenessProbe | object | `{"httpGet":{"path":"/healthz","port":"healthcheck"},"initialDelaySeconds":15,"periodSeconds":20}` | The livenessProbe for the controller container |
+| loggingDevel.enable | bool | `false` | (string) Development Mode defaults(encoder=consoleEncoder,logLevel=Debug,stackTraceLevel=Warn). Production Mode defaults(encoder=jsonEncoder,logLevel=Info,stackTraceLevel=Error) (default false) |
+| metricsBindAddress | string | `":8080"` | (string) The address the metric endpoint binds to. (default ":8080") |
+| nameOverride | string | `""` |  |
+| nodeSelector | object | `{}` | Kubernetes node selector rules to schedule Kamaji controller |
+| podAnnotations | object | `{}` | The annotations to apply to the Kamaji controller pods. |
+| podSecurityContext | object | `{"runAsNonRoot":true}` | The securityContext to apply to the Kamaji controller pods. |
+| readinessProbe | object | `{"httpGet":{"path":"/readyz","port":"healthcheck"},"initialDelaySeconds":5,"periodSeconds":10}` | The readinessProbe for the controller container |
+| replicaCount | int | `1` | The number of the pod replicas for the Kamaji controller. |
+| resources.limits.cpu | string | `"200m"` |  |
+| resources.limits.memory | string | `"100Mi"` |  |
+| resources.requests.cpu | string | `"100m"` |  |
+| resources.requests.memory | string | `"20Mi"` |  |
+| securityContext | object | `{"allowPrivilegeEscalation":false}` | The securityContext to apply to the Kamaji controller container only. It does not apply to the Kamaji RBAC proxy container. |
+| service.port | int | `8443` |  |
+| service.type | string | `"ClusterIP"` |  |
+| serviceAccount.annotations | object | `{}` |  |
+| serviceAccount.create | bool | `true` |  |
+| serviceAccount.name | string | `"kamaji-controller-manager"` |  |
+| temporaryDirectoryPath | string | `"/tmp/kamaji"` | Directory which will be used to work with temporary files. (default "/tmp/kamaji") |
+| tolerations | list | `[]` | Kubernetes node taints that the Kamaji controller pods would tolerate |

charts/kamaji/README.md.gotmpl

@@ -0,0 +1,56 @@
+{{ template "chart.header" . }}
+{{ template "chart.deprecationWarning" . }}
+
+{{ template "chart.badgesSection" . }}
+
+{{ template "chart.description" . }}
+
+{{ template "chart.maintainersSection" . }}
+
+{{ template "chart.sourcesSection" . }}
+
+{{ template "chart.requirementsSection" . }}
+
+[Kamaji](https://github.com/clastix/kamaji) requires a [multi-tenant `etcd`](https://github.com/clastix/kamaji-internal/blob/master/deploy/getting-started-with-kamaji.md#setup-internal-multi-tenant-etcd) cluster.
+This Helm Chart starting from v0.1.1 provides the installation of an internal `etcd` in order to streamline the local test. If you'd like to use an externally managed etcd instance, you can specify the overrides and by setting the value `etcd.deploy=false`.
+
+> For production use an externally managed `etcd` is highly recommended, the `etcd` addon offered by this Chart is not considered production-grade.
+
+## Install Kamaji
+
+To install the Chart with the release name `kamaji`:
+
+
+        helm upgrade --install --namespace kamaji-system --create-namespace clastix/kamaji
+
+Show the status:
+
+        helm status kamaji -n kamaji-system
+
+Upgrade the Chart
+
+        helm upgrade kamaji -n kamaji-system clastix/kamaji
+
+Uninstall the Chart
+
+        helm uninstall kamaji -n kamaji-system
+
+## Customize the installation
+
+There are two methods for specifying overrides of values during Chart installation: `--values` and `--set`.
+
+The `--values` option is the preferred method because it allows you to keep your overrides in a YAML file, rather than specifying them all on the command line. Create a copy of the YAML file `values.yaml` and add your overrides to it.
+
+Specify your overrides file when you install the Chart:
+
+        helm upgrade kamaji --install --namespace kamaji-system --create-namespace clastix/kamaji --values myvalues.yaml
+
+The values in your overrides file `myvalues.yaml` will override their counterparts in the Chart's values.yaml file. Any values in `values.yaml` that weren’t overridden will keep their defaults.
+
+If you only need to make minor customizations, you can specify them on the command line by using the `--set` option. For example:
+
+        helm upgrade kamaji --install --namespace kamaji-system --create-namespace clastix/kamaji --set etcd.deploy=false
+
+Here the values you can override:
+
+{{ template "chart.valuesSection" . }}

charts/kamaji/templates/NOTES.txt

@@ -0,0 +1,5 @@
+List the available CRDs installed by Kamaji:
+  kubectl get customresourcedefinitions.apiextensions.k8s.io
+
+List all the Tenant Control Plane resources deployed in your cluster:
+  kubectl get tenantcontrolplanes.kamaji.clastix.io --all-namespaces

charts/kamaji/templates/_helpers_etcd.tpl

@@ -0,0 +1,142 @@
+{{/*
+Create a default fully qualified etcd name.
+*/}}
+{{- define "etcd.fullname" -}}
+{{- printf "etcd" }}
+{{- end }}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "etcd.serviceAccountName" -}}
+{{- if .Values.etcd.serviceAccount.create }}
+{{- default (include "etcd.fullname" .) .Values.etcd.serviceAccount.name }}
+{{- else }}
+{{- default "default" .Values.etcd.serviceAccount.name }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create the name of the Service to use
+*/}}
+{{- define "etcd.serviceName" -}}
+{{- printf "%s" (include "etcd.fullname" .) | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "etcd.labels" -}}
+app.kubernetes.io/name: {{ include "kamaji.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+app.kubernetes.io/components: etcd
+{{- end }}
+
+{{/*
+Selector labels.
+*/}}
+{{- define "etcd.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "kamaji.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+app.kubernetes.io/component: etcd
+{{- end }}
+
+{{/*
+Name of the etcd CA secret.
+*/}}
+{{- define "etcd.caSecretName" }}
+{{- if .Values.etcd.deploy }}
+{{- printf "%s-%s" (include "etcd.fullname" .) "certs" | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- required "A valid .Values.etcd.overrides.caSecret.name required!" .Values.etcd.overrides.caSecret.name }}
+{{- end }}
+{{- end }}
+
+{{/*
+Namespace of the etcd CA secret.
+*/}}
+{{- define "etcd.caSecretNamespace" }}
+{{- if .Values.etcd.deploy }}
+{{- .Release.Namespace }}
+{{- else }}
+{{- required "A valid .Values.etcd.overrides.caSecret.namespace required!" .Values.etcd.overrides.caSecret.namespace }}
+{{- end }}
+{{- end }}
+
+{{/*
+Name of the certificate signing requests for the certificates required by etcd.
+*/}}
+{{- define "etcd.csrConfigMapName" }}
+{{- printf "%s-csr" (include "etcd.fullname" .) }}
+{{- end }}
+
+{{/*
+Name of the etcd root-client secret.
+*/}}
+{{- define "etcd.clientSecretName" }}
+{{- if .Values.etcd.deploy }}
+{{- printf "root-client-certs" }}
+{{- else }}
+{{- required "A valid .Values.etcd.overrides.clientSecret.name required!" .Values.etcd.overrides.clientSecret.name }}
+{{- end }}
+{{- end }}
+
+{{/*
+Namespace of the etcd root-client secret.
+*/}}
+{{- define "etcd.clientSecretNamespace" }}
+{{- if .Values.etcd.deploy }}
+{{- .Release.Namespace }}
+{{- else }}
+{{- required "A valid .Values.etcd.overrides.clientSecret.namespace required!" .Values.etcd.overrides.clientSecret.namespace }}
+{{- end }}
+{{- end }}
+
+{{/*
+Comma separated list of etcd endpoints, using the overrides in case of unmanaged etcd.
+*/}}
+{{- define "etcd.endpoints" }}
+{{- $list := list -}}
+{{- if .Values.etcd.deploy }}
+    {{- range $count := until 3 -}}
+        {{- $list = append $list (printf "https://%s-%d.%s.%s.svc.cluster.local:%d" "etcd" $count ( include "etcd.serviceName" . ) $.Release.Namespace (int $.Values.etcd.port) ) -}}
+    {{- end }}
+{{- else if .Values.etcd.overrides.endpoints }}
+    {{- range $v := .Values.etcd.overrides.endpoints -}}
+        {{- $list = append $list (printf "%s:%d" $v (int $.Values.etcd.port) ) -}}
+    {{- end -}}
+{{- else if not .Values.etcd.overrides.endpoints }}
+    {{- fail "A valid .Values.etcd.overrides.endpoints required!" }}
+{{- end }}
+{{- join "," $list -}}
+{{- end }}
+
+{{/*
+Key-value of the etcd peers, using the overrides in case of unmanaged etcd.
+*/}}
+{{- define "etcd.initialCluster" }}
+{{- $list := list -}}
+{{- if .Values.etcd.deploy }}
+    {{- range $i, $count := until 3 -}}
+        {{- $list = append $list ( printf "etcd-%d=https://%s-%d.%s.%s.svc.cluster.local:%d" $i "etcd" $count ( include "etcd.serviceName" . ) $.Release.Namespace (int $.Values.etcd.peerApiPort) ) -}}
+    {{- end }}
+{{- else if .Values.etcd.overrides.endpoints }}
+    {{- range $k, $v := .Values.etcd.overrides.endpoints -}}
+        {{- $list = append $list ( printf "%s=%s:%d" $k $v (int $.Values.etcd.peerApiPort) ) -}}
+    {{- end -}}
+{{- else if not .Values.etcd.overrides.endpoints }}
+    {{- fail "A valid .Values.etcd.overrides.endpoints required!" }}
+{{- end }}
+{{- join "," $list -}}
+{{- end }}
+
+{{/*
+Retrieve the current Kubernetes version to launch a kubectl container with the minimum version skew possible.
+*/}}
+{{- define "etcd.jobsTagKubeVersion" -}}
+{{- if contains "-eks-" .Capabilities.KubeVersion.GitVersion }}
+{{- print "v" .Capabilities.KubeVersion.Major "." (.Capabilities.KubeVersion.Minor | replace "+" "") -}}
+{{- else }}
+{{- print "v" .Capabilities.KubeVersion.Major "." .Capabilities.KubeVersion.Minor -}}
+{{- end }}
+{{- end }}

charts/kamaji/templates/controller.yaml

@@ -40,12 +40,12 @@ spec:
           protocol: TCP
       - args:
         - --config-file={{ .Values.configPath }}
-        - --etcd-ca-secret-name={{ .Values.etcd.caSecret.name }}
-        - --etcd-ca-secret-namespace={{ .Values.etcd.caSecret.namespace }}
-        - --etcd-client-secret-name={{ .Values.etcd.clientSecret.name }}
-        - --etcd-client-secret-namespace={{ .Values.etcd.clientSecret.namespace }}
+        - --etcd-ca-secret-name={{ include "etcd.caSecretName" . }}
+        - --etcd-ca-secret-namespace={{ include "etcd.caSecretNamespace" . }}
+        - --etcd-client-secret-name={{ include "etcd.clientSecretName" . }}
+        - --etcd-client-secret-namespace={{ include "etcd.clientSecretNamespace" . }}
         - --etcd-compaction-interval={{ .Values.etcd.compactionInterval }}
-        - --etcd-endpoints={{ .Values.etcd.endpoints }}
+        - --etcd-endpoints={{ include "etcd.endpoints" . }}
         - --health-probe-bind-address={{ .Values.healthProbeBindAddress }}
         - --leader-elect
         - --metrics-bind-address={{ .Values.metricsBindAddress }}

charts/kamaji/templates/etcd_cm.yaml

@@ -0,0 +1,94 @@
+{{- if .Values.etcd.deploy }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  labels:
+    {{- include "etcd.labels" . | nindent 4 }}
+  name: {{ include "etcd.csrConfigMapName" . }}
+  namespace: {{ .Release.Namespace }}
+data:
+  ca-csr.json: |-
+    {
+      "CN": "Clastix CA",
+      "key": {
+        "algo": "rsa",
+        "size": 2048
+      },
+      "names": [
+        {
+          "C": "IT",
+          "ST": "Italy",
+          "L": "Milan"
+        }
+      ]
+    }
+  config.json: |-
+    {
+      "signing": {
+        "default": {
+          "expiry": "8760h"
+        },
+        "profiles": {
+          "server-authentication": {
+            "usages": ["signing", "key encipherment", "server auth"],
+            "expiry": "8760h"
+          },
+          "client-authentication": {
+            "usages": ["signing", "key encipherment", "client auth"],
+            "expiry": "8760h"
+          },
+          "peer-authentication": {
+            "usages": ["signing", "key encipherment", "server auth", "client auth"],
+            "expiry": "8760h"
+          }
+        }
+      }
+    }
+  server-csr.json: |-
+    {
+      "CN": "etcd",
+      "key": {
+        "algo": "rsa",
+        "size": 2048
+      },
+      "hosts": [
+{{- range $count := until 3 -}}
+        {{ printf "\"etcd-%d.%s.%s.svc.cluster.local\"," $count (include "etcd.serviceName" .) $.Release.Namespace }}
+{{- end }}
+        "etcd-server.{{ .Release.Namespace }}.svc.cluster.local",
+        "etcd-server.{{ .Release.Namespace }}.svc",
+        "etcd-server",
+        "127.0.0.1"
+      ]
+    }
+  peer-csr.json: |-
+    {
+      "CN": "etcd",
+      "key": {
+        "algo": "rsa",
+        "size": 2048
+      },
+      "hosts": [
+{{- range $count := until 3 -}}
+        {{ printf "\"etcd-%d\"," $count }}
+        {{ printf "\"etcd-%d.%s\"," $count (include "etcd.serviceName" .) }}
+        {{ printf "\"etcd-%d.%s.%s.svc\"," $count (include "etcd.serviceName" .) $.Release.Namespace }}
+        {{ printf "\"etcd-%d.%s.%s.svc.cluster.local\"," $count (include "etcd.serviceName" .) $.Release.Namespace }}
+{{- end }}
+        "127.0.0.1"
+      ]
+    }
+  root-client-csr.json: |-
+    {
+      "CN": "root",
+      "key": {
+        "algo": "rsa",
+        "size": 2048
+      },
+      "names": [
+        {
+          "O": "system:masters"
+        }
+      ]
+    }
+{{- end }}

charts/kamaji/templates/etcd_job_postdelete.yaml

@@ -0,0 +1,31 @@
+{{- if .Values.etcd.deploy }}
+apiVersion: batch/v1
+kind: Job
+metadata:
+  labels:
+    {{- include "etcd.labels" . | nindent 4 }}
+  annotations:
+    "helm.sh/hook": pre-delete
+    "helm.sh/hook-weight": "-5"
+    "helm.sh/hook-delete-policy": "hook-succeeded,hook-failed"
+  name: "{{ .Release.Name }}-etcd-teardown"
+  namespace: {{ .Release.Namespace }}
+spec:
+  template:
+    metadata:
+      name: "{{ .Release.Name }}"
+    spec:
+      serviceAccountName: {{ include "etcd.serviceAccountName" . }}
+      restartPolicy: Never
+      containers:
+        - name: kubectl
+          image: {{ printf "clastix/kubectl:%s" (include "etcd.jobsTagKubeVersion" .) }}
+          command:
+            - kubectl
+            - --namespace={{ .Release.Namespace }}
+            - delete
+            - secret
+            - --ignore-not-found=true
+            - {{ include "etcd.caSecretName" . }}
+            - {{ include "etcd.clientSecretName" . }}
+{{- end }}

charts/kamaji/templates/etcd_job_postinstall.yaml

@@ -0,0 +1,91 @@
+{{- if .Values.etcd.deploy }}
+apiVersion: batch/v1
+kind: Job
+metadata:
+  labels:
+    {{- include "etcd.labels" . | nindent 4 }}
+  annotations:
+    "helm.sh/hook": post-install
+    "helm.sh/hook-weight": "-5"
+    "helm.sh/hook-delete-policy": "hook-succeeded,hook-failed"
+  name: "{{ .Release.Name }}-etcd-setup"
+  namespace: {{ .Release.Namespace }}
+spec:
+  template:
+    metadata:
+      name: "{{ .Release.Name }}"
+    spec:
+      serviceAccountName: {{ include "etcd.serviceAccountName" . }}
+      restartPolicy: Never
+      initContainers:
+        - name: cfssl
+          image: cfssl/cfssl:latest
+          command:
+            - bash
+            - -c
+            - |-
+              cfssl gencert -initca /csr/ca-csr.json | cfssljson -bare /certs/ca &&
+              mv /certs/ca.pem /certs/ca.crt && mv /certs/ca-key.pem /certs/ca.key &&
+              cfssl gencert -ca=/certs/ca.crt -ca-key=/certs/ca.key -config=/csr/config.json -profile=peer-authentication /csr/peer-csr.json | cfssljson -bare /certs/peer &&
+              cfssl gencert -ca=/certs/ca.crt -ca-key=/certs/ca.key -config=/csr/config.json -profile=peer-authentication /csr/server-csr.json | cfssljson -bare /certs/server &&
+              cfssl gencert -ca=/certs/ca.crt -ca-key=/certs/ca.key -config=/csr/config.json -profile=client-authentication /csr/root-client-csr.json | cfssljson -bare /certs/root-client
+          volumeMounts:
+            - mountPath: /certs
+              name: certs
+            - mountPath: /csr
+              name: csr
+        - name: kubectl
+          image: {{ printf "clastix/kubectl:%s" (include "etcd.jobsTagKubeVersion" .) }}
+          command:
+            - sh
+            - -c
+            - |-
+              kubectl --namespace={{ .Release.Namespace }} delete secret --ignore-not-found=true {{ include "etcd.caSecretName" . }} {{ include "etcd.clientSecretName" . }} &&
+              kubectl --namespace={{ .Release.Namespace }} create secret generic {{ include "etcd.caSecretName" . }} --from-file=/certs/ca.crt --from-file=/certs/ca.key --from-file=/certs/peer-key.pem --from-file=/certs/peer.pem --from-file=/certs/server-key.pem --from-file=/certs/server.pem &&
+              kubectl --namespace={{ .Release.Namespace }} create secret tls {{ include "etcd.clientSecretName" . }} --key=/certs/root-client-key.pem --cert=/certs/root-client.pem &&
+              kubectl --namespace={{ .Release.Namespace }} rollout status sts/etcd --timeout=300s
+          volumeMounts:
+            - mountPath: /certs
+              name: certs
+      containers:
+        - command:
+            - bash
+            - -c
+            - |-
+              etcdctl member list -w table &&
+              etcdctl user add --no-password=true root &&
+              etcdctl role add root &&
+              etcdctl user grant-role root root &&
+              etcdctl auth enable
+          env:
+            - name: ETCDCTL_ENDPOINTS
+              value: https://etcd-0.{{ include "etcd.serviceName" . }}.{{ .Release.Namespace }}.svc.cluster.local:2379
+            - name: ETCDCTL_CACERT
+              value: /opt/certs/ca/ca.crt
+            - name: ETCDCTL_CERT
+              value: /opt/certs/root-certs/tls.crt
+            - name: ETCDCTL_KEY
+              value: /opt/certs/root-certs/tls.key
+          image: quay.io/coreos/etcd:v3.5.1
+          imagePullPolicy: Always
+          name: etcd-client
+          volumeMounts:
+            - name: root-certs
+              mountPath: /opt/certs/root-certs
+            - name: certs
+              mountPath: /opt/certs/ca
+      securityContext:
+        runAsUser: 1000
+        runAsGroup: 1000
+        fsGroup: 1000
+      volumes:
+        - name: root-certs
+          secret:
+            secretName: {{ include "etcd.clientSecretName" . }}
+            optional: true
+        - name: csr
+          configMap:
+            name: {{ include "etcd.csrConfigMapName" . }}
+        - name: certs
+          emptyDir: {}
+{{- end }}

charts/kamaji/templates/etcd_rbac.yaml

@@ -0,0 +1,49 @@
+{{- if .Values.etcd.deploy }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  labels:
+    {{- include "etcd.labels" . | nindent 4 }}
+  name: etcd-gen-certs-role
+  namespace: {{ .Release.Namespace }}
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - secrets
+    verbs:
+      - delete
+    resourceNames:
+      - {{ include "etcd.caSecretName" . }}
+      - {{ include "etcd.clientSecretName" . }}
+  - apiGroups:
+      - ""
+    resources:
+      - secrets
+    verbs:
+      - create
+  - apiGroups:
+      - apps
+    resources:
+      - statefulsets
+    verbs:
+      - get
+      - list
+      - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  labels:
+    {{- include "etcd.labels" . | nindent 4 }}
+  name: etcd-gen-certs-rolebiding
+  namespace: {{ .Release.Namespace }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: etcd-gen-certs-role
+subjects:
+  - kind: ServiceAccount
+    name: {{ include "etcd.serviceAccountName" . }}
+    namespace: {{ .Release.Namespace }}
+{{- end }}

charts/kamaji/templates/etcd_sa.yaml

@@ -0,0 +1,9 @@
+{{- if .Values.etcd.deploy }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    {{- include "etcd.labels" . | nindent 4 }}
+  name: {{ include "etcd.serviceAccountName" . }}
+  namespace: {{ .Release.Namespace }}
+{{- end }}

charts/kamaji/templates/etcd_service.yaml

@@ -0,0 +1,18 @@
+{{- if .Values.etcd.deploy }}
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    {{- include "etcd.labels" . | nindent 4 }}
+  name: {{ include "etcd.serviceName" . }}
+  namespace: {{ .Release.Namespace }}
+spec:
+  clusterIP: None
+  ports:
+    - port: {{ .Values.etcd.port }}
+      name: client
+    - port: {{ .Values.etcd.peerApiPort }}
+      name: peer
+  selector:
+    {{- include "etcd.selectorLabels" . | nindent 4 }}
+{{- end }}

charts/kamaji/templates/etcd_sts.yaml

@@ -0,0 +1,93 @@
+{{- if .Values.etcd.deploy }}
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  labels:
+    {{- include "etcd.labels" . | nindent 4 }}
+  name: {{ include "etcd.fullname" . }}
+  namespace: {{ .Release.Namespace }}
+spec:
+  serviceName: {{ include "etcd.serviceName" . }}
+  selector:
+    matchLabels:
+      {{- include "etcd.selectorLabels" . | nindent 6 }}
+  replicas: 3
+  template:
+    metadata:
+      name: etcd
+      labels:
+        {{- include "etcd.selectorLabels" . | nindent 8 }}
+    spec:
+      volumes:
+        - name: certs
+          secret:
+            secretName: {{ include "etcd.caSecretName" . }}
+      containers:
+        - name: etcd
+          image: {{ .Values.etcd.image.repository }}:{{ .Values.etcd.image.tag | default "v3.5.4" }}
+          imagePullPolicy: {{ .Values.etcd.image.pullPolicy }}
+          ports:
+            - containerPort: 2379
+              name: client
+            - containerPort: 2380
+              name: peer
+          volumeMounts:
+            - name: data
+              mountPath: /var/run/etcd
+            - name: certs
+              mountPath: /etc/etcd/pki
+          command:
+            - etcd
+            - --data-dir=/var/run/etcd
+            - --name=$(POD_NAME)
+            - --initial-cluster-state=new
+            - --initial-cluster={{ include "etcd.initialCluster" . }}
+            - --initial-advertise-peer-urls=https://$(POD_NAME).etcd.$(POD_NAMESPACE).svc.cluster.local:2380
+            - --advertise-client-urls=https://$(POD_NAME).etcd.$(POD_NAMESPACE).svc.cluster.local:2379
+            - --initial-cluster-token=kamaji
+            - --listen-client-urls=https://0.0.0.0:2379
+            - --listen-metrics-urls=http://0.0.0.0:2381
+            - --listen-peer-urls=https://0.0.0.0:2380
+            - --client-cert-auth=true
+            - --peer-client-cert-auth=true
+            - --trusted-ca-file=/etc/etcd/pki/ca.crt
+            - --cert-file=/etc/etcd/pki/server.pem
+            - --key-file=/etc/etcd/pki/server-key.pem
+            - --peer-trusted-ca-file=/etc/etcd/pki/ca.crt
+            - --peer-cert-file=/etc/etcd/pki/peer.pem
+            - --peer-key-file=/etc/etcd/pki/peer-key.pem
+            - --auto-compaction-mode=periodic
+            - --auto-compaction-retention=5m
+            - --snapshot-count=10000
+            - --quota-backend-bytes=8589934592
+            - --v=8
+          env:
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+          {{- with .Values.etcd.livenessProbe }}
+          livenessProbe:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.etcd.startupProbe }}
+          startupProbe:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+  volumeClaimTemplates:
+  - metadata:
+      name: data
+    spec:
+      storageClassName: {{ .Values.etcd.persistence.storageClassName }}
+      accessModes:
+      {{- range .Values.etcd.persistence.accessModes }}
+      - {{ . | quote }}
+      {{- end }}
+      resources:
+        requests:
+          storage: {{ .Values.etcd.persistence.size }}
+{{- end }}

charts/kamaji/values.sample.yaml

@@ -0,0 +1,2 @@
+etcd:
+  endpoints: "https://etcd-0.etcd.kamaji-system.svc.cluster.local:2379,https://etcd-1.etcd.kamaji-system.svc.cluster.local:2379,https://etcd-2.etcd.kamaji-system.svc.cluster.local:2379"

charts/kamaji/values.yaml

@@ -7,8 +7,8 @@ replicaCount: 1
 
 image:
   # -- The container image of the Kamaji controller.
-  repository: quay.io/clastix/kamaji
-  pullPolicy: IfNotPresent
+  repository: clastix/kamaji
+  pullPolicy: Always
   # Overrides the image tag whose default is the chart appVersion.
   tag: latest
 
@@ -19,24 +19,62 @@ extraArgs: []
 configPath: "./kamaji.yaml"
 
 etcd:
-  caSecret:
-    # -- Name of the secret which contains CA's certificate and private key. (default: "etcd-certs")
-    name: etcd-certs
-    # -- Namespace of the secret which contains CA's certificate and private key. (default: "kamaji")
-    namespace: kamaji-system
+  # -- Install an etcd with enabled multi-tenancy along with Kamaji
+  deploy: true
 
-  clientSecret:
-    # -- Name of the secret which contains ETCD client certificates. (default: "root-client-certs")
-    name: root-client-certs
-    # -- Name of the namespace where the secret which contains ETCD client certificates is. (default: "kamaji")
-    namespace: kamaji-system
+  # -- The peer API port which servers are listening to.
+  peerApiPort: 2380
 
+  # -- The client request port.
+  port: 2379
+
+  # -- Install specific etcd image
+  image:
+    repository: quay.io/coreos/etcd
+    tag: "v3.5.4"
+    pullPolicy: IfNotPresent
+
+  # -- The livenessProbe for the etcd container
+  livenessProbe:
+    failureThreshold: 8
+    httpGet:
+      path: /health?serializable=true
+      port: 2381
+      scheme: HTTP
+    initialDelaySeconds: 10
+    periodSeconds: 10
+    timeoutSeconds: 15
+
+  serviceAccount:
+    # -- Create a ServiceAccount, required to install and provision the etcd backing storage (default: true)
+    create: true
+    # -- Define the ServiceAccount name to use during the setup and provision of the etcd backing storage (default: "")
+    name: ""
+  persistence:
+    size: 10Gi
+    storageClass: ""
+    accessModes:
+    - ReadWriteOnce
+
+  overrides:
+    caSecret:
+      # -- Name of the secret which contains CA's certificate and private key. (default: "etcd-certs")
+      name: etcd-certs
+      # -- Namespace of the secret which contains CA's certificate and private key. (default: "kamaji-system")
+      namespace: kamaji-system
+    clientSecret:
+      # -- Name of the secret which contains ETCD client certificates. (default: "root-client-certs")
+      name: root-client-certs
+      # -- Name of the namespace where the secret which contains ETCD client certificates is. (default: "kamaji-system")
+      namespace: kamaji-system
+    # -- (map) Dictionary of the endpoints for the etcd cluster's members, key is the name of the etcd server. Don't define any port, inflected from .etcd.peerApiPort value.
+    endpoints:
+      etcd-0: https://etcd-0.etcd.kamaji-system.svc.cluster.local
+      etcd-1: https://etcd-1.etcd.kamaji-system.svc.cluster.local
+      etcd-2: https://etcd-2.etcd.kamaji-system.svc.cluster.local
   # -- ETCD Compaction interval (e.g. "5m0s"). (default: "0" (disabled))
   compactionInterval: 0
 
-  # -- (string) Comma-separated list of the endpoints of the etcd cluster's members.
-  endpoints: "etcd-0.etcd.kamaji-system.svc.cluster.local:2379,etcd-1.etcd.kamaji-system.svc.cluster.local:2379,etcd-2.etcd.kamaji-system.svc.cluster.local:2379"
-
 # -- The address the probe endpoint binds to. (default ":8081")
 healthProbeBindAddress: ":8081"
 
@@ -93,24 +131,6 @@ service:
   type: ClusterIP
   port: 8443
 
-ingress:
-  # -- Whether to expose the Kamaji controller through an Ingress.
-  enabled: false
-  # -- Name of the ingress class to route through this controller.
-  className: ""
-  annotations: {}
-    # kubernetes.io/ingress.class: nginx
-    # kubernetes.io/tls-acme: "true"
-  hosts:
-    - host: chart-example.local
-      paths:
-        - path: /
-          pathType: ImplementationSpecific
-  tls: []
-  #  - secretName: chart-example-tls
-  #    hosts:
-  #      - chart-example.local
-
 resources:
   limits:
     cpu: 200m

config/crd/bases/kamaji.clastix.io_tenantcontrolplanes.yaml

@@ -60,6 +60,68 @@ spec:
           spec:
             description: TenantControlPlaneSpec defines the desired state of TenantControlPlane.
             properties:
+              addons:
+                description: Addons contain which addons are enabled
+                properties:
+                  coreDNS:
+                    description: AddonSpec defines the spec for every addon.
+                    type: object
+                  konnectivity:
+                    description: KonnectivitySpec defines the spec for Konnectivity.
+                    properties:
+                      agentImage:
+                        default: us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-agent
+                        description: AgentImage defines the container image for Konnectivity's
+                          agent.
+                        type: string
+                      proxyPort:
+                        description: Port of Konnectivity proxy server.
+                        format: int32
+                        type: integer
+                      resources:
+                        description: Resources define the amount of CPU and memory
+                          to allocate to the Konnectivity server.
+                        properties:
+                          limits:
+                            additionalProperties:
+                              anyOf:
+                              - type: integer
+                              - type: string
+                              pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                              x-kubernetes-int-or-string: true
+                            description: 'Limits describes the maximum amount of compute
+                              resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                            type: object
+                          requests:
+                            additionalProperties:
+                              anyOf:
+                              - type: integer
+                              - type: string
+                              pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                              x-kubernetes-int-or-string: true
+                            description: 'Requests describes the minimum amount of
+                              compute resources required. If Requests is omitted for
+                              a container, it defaults to Limits if that is explicitly
+                              specified, otherwise to an implementation-defined value.
+                              More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                            type: object
+                        type: object
+                      serverImage:
+                        default: us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-server
+                        description: ServerImage defines the container image for Konnectivity's
+                          server.
+                        type: string
+                      version:
+                        default: v0.0.31
+                        description: Version for Konnectivity server and agent.
+                        type: string
+                    required:
+                    - proxyPort
+                    type: object
+                  kubeProxy:
+                    description: AddonSpec defines the spec for every addon.
+                    type: object
+                type: object
               controlPlane:
                 description: ControlPlane defines how the Tenant Control Plane Kubernetes
                   resources must be created in the Admin Cluster, such as the number
@@ -83,10 +145,124 @@ spec:
                               type: string
                             type: object
                         type: object
+                      extraArgs:
+                        description: ExtraArgs allows adding additional arguments
+                          to the Control Plane components, such as kube-apiserver,
+                          controller-manager, and scheduler.
+                        properties:
+                          apiServer:
+                            items:
+                              type: string
+                            type: array
+                          controllerManager:
+                            items:
+                              type: string
+                            type: array
+                          kine:
+                            description: Available only if Kamaji is running using
+                              Kine as backing storage.
+                            items:
+                              type: string
+                            type: array
+                          scheduler:
+                            items:
+                              type: string
+                            type: array
+                        type: object
                       replicas:
                         default: 2
                         format: int32
                         type: integer
+                      resources:
+                        description: Resources defines the amount of memory and CPU
+                          to allocate to each component of the Control Plane (kube-apiserver,
+                          controller-manager, and scheduler).
+                        properties:
+                          apiServer:
+                            description: ResourceRequirements describes the compute
+                              resource requirements.
+                            properties:
+                              limits:
+                                additionalProperties:
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
+                                description: 'Limits describes the maximum amount
+                                  of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                                type: object
+                              requests:
+                                additionalProperties:
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
+                                description: 'Requests describes the minimum amount
+                                  of compute resources required. If Requests is omitted
+                                  for a container, it defaults to Limits if that is
+                                  explicitly specified, otherwise to an implementation-defined
+                                  value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                                type: object
+                            type: object
+                          controllerManager:
+                            description: ResourceRequirements describes the compute
+                              resource requirements.
+                            properties:
+                              limits:
+                                additionalProperties:
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
+                                description: 'Limits describes the maximum amount
+                                  of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                                type: object
+                              requests:
+                                additionalProperties:
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
+                                description: 'Requests describes the minimum amount
+                                  of compute resources required. If Requests is omitted
+                                  for a container, it defaults to Limits if that is
+                                  explicitly specified, otherwise to an implementation-defined
+                                  value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                                type: object
+                            type: object
+                          scheduler:
+                            description: ResourceRequirements describes the compute
+                              resource requirements.
+                            properties:
+                              limits:
+                                additionalProperties:
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
+                                description: 'Limits describes the maximum amount
+                                  of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                                type: object
+                              requests:
+                                additionalProperties:
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
+                                description: 'Requests describes the minimum amount
+                                  of compute resources required. If Requests is omitted
+                                  for a container, it defaults to Limits if that is
+                                  explicitly specified, otherwise to an implementation-defined
+                                  value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                                type: object
+                            type: object
+                        type: object
                     type: object
                   ingress:
                     description: Defining the options for an Optional Ingress which
@@ -106,8 +282,6 @@ spec:
                               type: string
                             type: object
                         type: object
-                      enabled:
-                        type: boolean
                       hostname:
                         description: Hostname is an optional field which will be used
                           as Ingress's Host. If it is not defined, Ingress's host
@@ -116,8 +290,6 @@ spec:
                         type: string
                       ingressClassName:
                         type: string
-                    required:
-                    - enabled
                     type: object
                   service:
                     description: Defining the options for the Tenant Control Plane
@@ -246,14 +418,22 @@ spec:
                       in the section of ExternalIPs of the Kubernetes Service (only
                       ClusterIP or NodePort)
                     type: boolean
-                  dnsServiceIPs:
+                  certSANs:
+                    description: CertSANs sets extra Subject Alternative Names (SANs)
+                      for the API Server signing certificate. Use this field to add
+                      additional hostnames when exposing the Tenant Control Plane
+                      with third solutions.
+                    items:
+                      type: string
+                    type: array
+                  dnsServiceIPs:
+                    default:
+                    - 10.96.0.10
                     items:
                       type: string
                     type: array
-                  domain:
-                    description: Domain of the tenant control plane
-                    type: string
                   podCidr:
+                    default: 10.244.0.0/16
                     description: CIDR for Kubernetes Pods
                     type: string
                   port:
@@ -262,14 +442,9 @@ spec:
                     format: int32
                     type: integer
                   serviceCidr:
+                    default: 10.96.0.0/16
                     description: Kubernetes Service
                     type: string
-                required:
-                - dnsServiceIPs
-                - domain
-                - podCidr
-                - port
-                - serviceCidr
                 type: object
             required:
             - controlPlane
@@ -278,13 +453,291 @@ spec:
           status:
             description: TenantControlPlaneStatus defines the observed state of TenantControlPlane.
             properties:
+              addons:
+                description: Addons contains the status of the different Addons
+                properties:
+                  coreDNS:
+                    description: AddonStatus defines the observed state of an Addon.
+                    properties:
+                      checksum:
+                        type: string
+                      enabled:
+                        type: boolean
+                      lastUpdate:
+                        format: date-time
+                        type: string
+                    required:
+                    - enabled
+                    type: object
+                  konnectivity:
+                    description: KonnectivityStatus defines the status of Konnectivity
+                      as Addon.
+                    properties:
+                      agent:
+                        properties:
+                          checksum:
+                            type: string
+                          lastUpdate:
+                            description: Last time when k8s object was updated
+                            format: date-time
+                            type: string
+                          name:
+                            type: string
+                          namespace:
+                            type: string
+                        type: object
+                      certificate:
+                        description: CertificatePrivateKeyPairStatus defines the status.
+                        properties:
+                          checksum:
+                            type: string
+                          lastUpdate:
+                            format: date-time
+                            type: string
+                          secretName:
+                            type: string
+                        type: object
+                      clusterrolebinding:
+                        properties:
+                          checksum:
+                            type: string
+                          lastUpdate:
+                            description: Last time when k8s object was updated
+                            format: date-time
+                            type: string
+                          name:
+                            type: string
+                          namespace:
+                            type: string
+                        type: object
+                      configMap:
+                        properties:
+                          checksum:
+                            type: string
+                          name:
+                            type: string
+                        type: object
+                      enabled:
+                        type: boolean
+                      kubeconfig:
+                        description: KubeconfigStatus contains information about the
+                          generated kubeconfig.
+                        properties:
+                          checksum:
+                            type: string
+                          lastUpdate:
+                            format: date-time
+                            type: string
+                          secretName:
+                            type: string
+                        type: object
+                      sa:
+                        properties:
+                          checksum:
+                            type: string
+                          lastUpdate:
+                            description: Last time when k8s object was updated
+                            format: date-time
+                            type: string
+                          name:
+                            type: string
+                          namespace:
+                            type: string
+                        type: object
+                      service:
+                        description: KubernetesServiceStatus defines the status for
+                          the Tenant Control Plane Service in the management cluster.
+                        properties:
+                          conditions:
+                            description: Current service state
+                            items:
+                              description: "Condition contains details for one aspect
+                                of the current state of this API Resource. --- This
+                                struct is intended for direct use as an array at the
+                                field path .status.conditions.  For example, type
+                                FooStatus struct{     // Represents the observations
+                                of a foo's current state.     // Known .status.conditions.type
+                                are: \"Available\", \"Progressing\", and \"Degraded\"
+                                \    // +patchMergeKey=type     // +patchStrategy=merge
+                                \    // +listType=map     // +listMapKey=type     Conditions
+                                []metav1.Condition `json:\"conditions,omitempty\"
+                                patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"`
+                                \n     // other fields }"
+                              properties:
+                                lastTransitionTime:
+                                  description: lastTransitionTime is the last time
+                                    the condition transitioned from one status to
+                                    another. This should be when the underlying condition
+                                    changed.  If that is not known, then using the
+                                    time when the API field changed is acceptable.
+                                  format: date-time
+                                  type: string
+                                message:
+                                  description: message is a human readable message
+                                    indicating details about the transition. This
+                                    may be an empty string.
+                                  maxLength: 32768
+                                  type: string
+                                observedGeneration:
+                                  description: observedGeneration represents the .metadata.generation
+                                    that the condition was set based upon. For instance,
+                                    if .metadata.generation is currently 12, but the
+                                    .status.conditions[x].observedGeneration is 9,
+                                    the condition is out of date with respect to the
+                                    current state of the instance.
+                                  format: int64
+                                  minimum: 0
+                                  type: integer
+                                reason:
+                                  description: reason contains a programmatic identifier
+                                    indicating the reason for the condition's last
+                                    transition. Producers of specific condition types
+                                    may define expected values and meanings for this
+                                    field, and whether the values are considered a
+                                    guaranteed API. The value should be a CamelCase
+                                    string. This field may not be empty.
+                                  maxLength: 1024
+                                  minLength: 1
+                                  pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
+                                  type: string
+                                status:
+                                  description: status of the condition, one of True,
+                                    False, Unknown.
+                                  enum:
+                                  - "True"
+                                  - "False"
+                                  - Unknown
+                                  type: string
+                                type:
+                                  description: type of condition in CamelCase or in
+                                    foo.example.com/CamelCase. --- Many .condition.type
+                                    values are consistent across resources like Available,
+                                    but because arbitrary conditions can be useful
+                                    (see .node.status.conditions), the ability to
+                                    deconflict is important. The regex it matches
+                                    is (dns1123SubdomainFmt/)?(qualifiedNameFmt)
+                                  maxLength: 316
+                                  pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
+                                  type: string
+                              required:
+                              - lastTransitionTime
+                              - message
+                              - reason
+                              - status
+                              - type
+                              type: object
+                            type: array
+                            x-kubernetes-list-map-keys:
+                            - type
+                            x-kubernetes-list-type: map
+                          loadBalancer:
+                            description: LoadBalancer contains the current status
+                              of the load-balancer, if one is present.
+                            properties:
+                              ingress:
+                                description: Ingress is a list containing ingress
+                                  points for the load-balancer. Traffic intended for
+                                  the service should be sent to these ingress points.
+                                items:
+                                  description: 'LoadBalancerIngress represents the
+                                    status of a load-balancer ingress point: traffic
+                                    intended for the service should be sent to an
+                                    ingress point.'
+                                  properties:
+                                    hostname:
+                                      description: Hostname is set for load-balancer
+                                        ingress points that are DNS based (typically
+                                        AWS load-balancers)
+                                      type: string
+                                    ip:
+                                      description: IP is set for load-balancer ingress
+                                        points that are IP based (typically GCE or
+                                        OpenStack load-balancers)
+                                      type: string
+                                    ports:
+                                      description: Ports is a list of records of service
+                                        ports If used, every port defined in the service
+                                        should have an entry in it
+                                      items:
+                                        properties:
+                                          error:
+                                            description: 'Error is to record the problem
+                                              with the service port The format of
+                                              the error shall comply with the following
+                                              rules: - built-in error values shall
+                                              be specified in this file and those
+                                              shall use   CamelCase names - cloud
+                                              provider specific error values must
+                                              have names that comply with the   format
+                                              foo.example.com/CamelCase. --- The regex
+                                              it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)'
+                                            maxLength: 316
+                                            pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
+                                            type: string
+                                          port:
+                                            description: Port is the port number of
+                                              the service port of which status is
+                                              recorded here
+                                            format: int32
+                                            type: integer
+                                          protocol:
+                                            default: TCP
+                                            description: 'Protocol is the protocol
+                                              of the service port of which status
+                                              is recorded here The supported values
+                                              are: "TCP", "UDP", "SCTP"'
+                                            type: string
+                                        required:
+                                        - port
+                                        - protocol
+                                        type: object
+                                      type: array
+                                      x-kubernetes-list-type: atomic
+                                  type: object
+                                type: array
+                            type: object
+                          name:
+                            description: The name of the Service for the given cluster.
+                            type: string
+                          namespace:
+                            description: The namespace which the Service for the given
+                              cluster is deployed.
+                            type: string
+                          port:
+                            description: The port where the service is running
+                            format: int32
+                            type: integer
+                        required:
+                        - name
+                        - namespace
+                        - port
+                        type: object
+                    required:
+                    - enabled
+                    type: object
+                  kubeProxy:
+                    description: AddonStatus defines the observed state of an Addon.
+                    properties:
+                      checksum:
+                        type: string
+                      enabled:
+                        type: boolean
+                      lastUpdate:
+                        format: date-time
+                        type: string
+                    required:
+                    - enabled
+                    type: object
+                type: object
               certificates:
                 description: Certificates contains information about the different
                   certificates that are necessary to run a kubernetes control plane
                 properties:
                   apiServer:
-                    description: CertificatePrivateKeyPair defines the status.
+                    description: CertificatePrivateKeyPairStatus defines the status.
                     properties:
+                      checksum:
+                        type: string
                       lastUpdate:
                         format: date-time
                         type: string
@@ -292,8 +745,10 @@ spec:
                         type: string
                     type: object
                   apiServerKubeletClient:
-                    description: CertificatePrivateKeyPair defines the status.
+                    description: CertificatePrivateKeyPairStatus defines the status.
                     properties:
+                      checksum:
+                        type: string
                       lastUpdate:
                         format: date-time
                         type: string
@@ -301,8 +756,10 @@ spec:
                         type: string
                     type: object
                   ca:
-                    description: CertificatePrivateKeyPair defines the status.
+                    description: CertificatePrivateKeyPairStatus defines the status.
                     properties:
+                      checksum:
+                        type: string
                       lastUpdate:
                         format: date-time
                         type: string
@@ -310,13 +767,15 @@ spec:
                         type: string
                     type: object
                   etcd:
-                    description: ETCDAPIServerCertificate defines the observed state
+                    description: ETCDCertificatesStatus defines the observed state
                       of ETCD Certificate for API server.
                     properties:
                       apiServer:
-                        description: ETCDAPIServerCertificate defines the observed
+                        description: APIServerCertificatesStatus defines the observed
                           state of ETCD Certificate for API server.
                         properties:
+                          checksum:
+                            type: string
                           lastUpdate:
                             format: date-time
                             type: string
@@ -324,9 +783,11 @@ spec:
                             type: string
                         type: object
                       ca:
-                        description: ETCDAPIServerCertificate defines the observed
-                          state of ETCD Certificate for API server.
+                        description: ETCDCertificateStatus defines the observed state
+                          of ETCD Certificate for API server.
                         properties:
+                          checksum:
+                            type: string
                           lastUpdate:
                             format: date-time
                             type: string
@@ -335,8 +796,10 @@ spec:
                         type: object
                     type: object
                   frontProxyCA:
-                    description: CertificatePrivateKeyPair defines the status.
+                    description: CertificatePrivateKeyPairStatus defines the status.
                     properties:
+                      checksum:
+                        type: string
                       lastUpdate:
                         format: date-time
                         type: string
@@ -344,8 +807,10 @@ spec:
                         type: string
                     type: object
                   frontProxyClient:
-                    description: CertificatePrivateKeyPair defines the status.
+                    description: CertificatePrivateKeyPairStatus defines the status.
                     properties:
+                      checksum:
+                        type: string
                       lastUpdate:
                         format: date-time
                         type: string
@@ -353,8 +818,10 @@ spec:
                         type: string
                     type: object
                   sa:
-                    description: CertificatePrivateKeyPair defines the status.
+                    description: PublicKeyPrivateKeyPairStatus defines the status.
                     properties:
+                      checksum:
+                        type: string
                       lastUpdate:
                         format: date-time
                         type: string
@@ -370,59 +837,37 @@ spec:
                 description: KubeadmPhase contains the status of the kubeadm phases
                   action
                 properties:
-                  addonCoreDNS:
-                    description: KubeadmPhasesStatus contains the status of of a kubeadm
-                      phase action.
-                    properties:
-                      kubeadmConfigResourceVersion:
-                        type: string
-                      lastUpdate:
-                        format: date-time
-                        type: string
-                    type: object
-                  addonKubeProxy:
-                    description: KubeadmPhasesStatus contains the status of of a kubeadm
-                      phase action.
-                    properties:
-                      kubeadmConfigResourceVersion:
-                        type: string
-                      lastUpdate:
-                        format: date-time
-                        type: string
-                    type: object
                   bootstrapToken:
-                    description: KubeadmPhasesStatus contains the status of of a kubeadm
+                    description: KubeadmPhaseStatus contains the status of a kubeadm
                       phase action.
                     properties:
-                      kubeadmConfigResourceVersion:
+                      checksum:
                         type: string
                       lastUpdate:
                         format: date-time
                         type: string
                     type: object
                   uploadConfigKubeadm:
-                    description: KubeadmPhasesStatus contains the status of of a kubeadm
+                    description: KubeadmPhaseStatus contains the status of a kubeadm
                       phase action.
                     properties:
-                      kubeadmConfigResourceVersion:
+                      checksum:
                         type: string
                       lastUpdate:
                         format: date-time
                         type: string
                     type: object
                   uploadConfigKubelet:
-                    description: KubeadmPhasesStatus contains the status of of a kubeadm
+                    description: KubeadmPhaseStatus contains the status of a kubeadm
                       phase action.
                     properties:
-                      kubeadmConfigResourceVersion:
+                      checksum:
                         type: string
                       lastUpdate:
                         format: date-time
                         type: string
                     type: object
                 required:
-                - addonCoreDNS
-                - addonKubeProxy
                 - bootstrapToken
                 - uploadConfigKubeadm
                 - uploadConfigKubelet
@@ -431,34 +876,37 @@ spec:
                 description: KubeadmConfig contains the status of the configuration
                   required by kubeadm
                 properties:
+                  checksum:
+                    description: Checksum of the kubeadm configuration to detect changes
+                    type: string
                   configmapName:
                     type: string
                   lastUpdate:
                     format: date-time
                     type: string
-                  resourceVersion:
-                    type: string
-                required:
-                - resourceVersion
                 type: object
               kubeconfig:
                 description: KubeConfig contains information about the kubenconfigs
                   that control plane pieces need
                 properties:
                   admin:
-                    description: TenantControlPlaneKubeconfigsStatus contains information
-                      about a the generated kubeconfig.
+                    description: KubeconfigStatus contains information about the generated
+                      kubeconfig.
                     properties:
+                      checksum:
+                        type: string
                       lastUpdate:
                         format: date-time
                         type: string
                       secretName:
                         type: string
                     type: object
-                  controlerManager:
-                    description: TenantControlPlaneKubeconfigsStatus contains information
-                      about a the generated kubeconfig.
+                  controllerManager:
+                    description: KubeconfigStatus contains information about the generated
+                      kubeconfig.
                     properties:
+                      checksum:
+                        type: string
                       lastUpdate:
                         format: date-time
                         type: string
@@ -466,9 +914,11 @@ spec:
                         type: string
                     type: object
                   scheduler:
-                    description: TenantControlPlaneKubeconfigsStatus contains information
-                      about a the generated kubeconfig.
+                    description: KubeconfigStatus contains information about the generated
+                      kubeconfig.
                     properties:
+                      checksum:
+                        type: string
                       lastUpdate:
                         format: date-time
                         type: string
@@ -531,6 +981,10 @@ spec:
                           - type
                           type: object
                         type: array
+                      lastUpdate:
+                        description: Last time when deployment was updated
+                        format: date-time
+                        type: string
                       name:
                         description: The name of the Deployment for the given cluster.
                         type: string
@@ -827,8 +1281,6 @@ spec:
                         description: Version is the running Kubernetes version of
                           the Tenant Control Plane.
                         type: string
-                    required:
-                    - status
                     type: object
                 type: object
               storage:
@@ -874,6 +1326,40 @@ spec:
                         - name
                         type: object
                     type: object
+                  kine:
+                    properties:
+                      certificate:
+                        properties:
+                          checksum:
+                            type: string
+                          lastUpdate:
+                            format: date-time
+                            type: string
+                          secretName:
+                            type: string
+                        type: object
+                      config:
+                        properties:
+                          checksum:
+                            type: string
+                          secretName:
+                            type: string
+                        type: object
+                      driver:
+                        type: string
+                      setup:
+                        properties:
+                          checksum:
+                            type: string
+                          lastUpdate:
+                            format: date-time
+                            type: string
+                          schema:
+                            type: string
+                          user:
+                            type: string
+                        type: object
+                    type: object
                 type: object
             type: object
         type: object

config/install.yaml

@@ -60,6 +60,60 @@ spec:
           spec:
             description: TenantControlPlaneSpec defines the desired state of TenantControlPlane.
             properties:
+              addons:
+                description: Addons contain which addons are enabled
+                properties:
+                  coreDNS:
+                    description: AddonSpec defines the spec for every addon.
+                    type: object
+                  konnectivity:
+                    description: KonnectivitySpec defines the spec for Konnectivity.
+                    properties:
+                      agentImage:
+                        default: us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-agent
+                        description: AgentImage defines the container image for Konnectivity's agent.
+                        type: string
+                      proxyPort:
+                        description: Port of Konnectivity proxy server.
+                        format: int32
+                        type: integer
+                      resources:
+                        description: Resources define the amount of CPU and memory to allocate to the Konnectivity server.
+                        properties:
+                          limits:
+                            additionalProperties:
+                              anyOf:
+                              - type: integer
+                              - type: string
+                              pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                              x-kubernetes-int-or-string: true
+                            description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                            type: object
+                          requests:
+                            additionalProperties:
+                              anyOf:
+                              - type: integer
+                              - type: string
+                              pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                              x-kubernetes-int-or-string: true
+                            description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                            type: object
+                        type: object
+                      serverImage:
+                        default: us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-server
+                        description: ServerImage defines the container image for Konnectivity's server.
+                        type: string
+                      version:
+                        default: v0.0.31
+                        description: Version for Konnectivity server and agent.
+                        type: string
+                    required:
+                    - proxyPort
+                    type: object
+                  kubeProxy:
+                    description: AddonSpec defines the spec for every addon.
+                    type: object
+                type: object
               controlPlane:
                 description: ControlPlane defines how the Tenant Control Plane Kubernetes resources must be created in the Admin Cluster, such as the number of Pod replicas, the Service resource, or the Ingress.
                 properties:
@@ -78,10 +132,101 @@ spec:
                               type: string
                             type: object
                         type: object
+                      extraArgs:
+                        description: ExtraArgs allows adding additional arguments to the Control Plane components, such as kube-apiserver, controller-manager, and scheduler.
+                        properties:
+                          apiServer:
+                            items:
+                              type: string
+                            type: array
+                          controllerManager:
+                            items:
+                              type: string
+                            type: array
+                          kine:
+                            description: Available only if Kamaji is running using Kine as backing storage.
+                            items:
+                              type: string
+                            type: array
+                          scheduler:
+                            items:
+                              type: string
+                            type: array
+                        type: object
                       replicas:
                         default: 2
                         format: int32
                         type: integer
+                      resources:
+                        description: Resources defines the amount of memory and CPU to allocate to each component of the Control Plane (kube-apiserver, controller-manager, and scheduler).
+                        properties:
+                          apiServer:
+                            description: ResourceRequirements describes the compute resource requirements.
+                            properties:
+                              limits:
+                                additionalProperties:
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
+                                description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                                type: object
+                              requests:
+                                additionalProperties:
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
+                                description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                                type: object
+                            type: object
+                          controllerManager:
+                            description: ResourceRequirements describes the compute resource requirements.
+                            properties:
+                              limits:
+                                additionalProperties:
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
+                                description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                                type: object
+                              requests:
+                                additionalProperties:
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
+                                description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                                type: object
+                            type: object
+                          scheduler:
+                            description: ResourceRequirements describes the compute resource requirements.
+                            properties:
+                              limits:
+                                additionalProperties:
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
+                                description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                                type: object
+                              requests:
+                                additionalProperties:
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
+                                description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                                type: object
+                            type: object
+                        type: object
                     type: object
                   ingress:
                     description: Defining the options for an Optional Ingress which will expose API Server of the Tenant Control Plane
@@ -98,15 +243,11 @@ spec:
                               type: string
                             type: object
                         type: object
-                      enabled:
-                        type: boolean
                       hostname:
                         description: Hostname is an optional field which will be used as Ingress's Host. If it is not defined, Ingress's host will be "<tenant>.<namespace>.<domain>", where domain is specified under NetworkProfileSpec
                         type: string
                       ingressClassName:
                         type: string
-                    required:
-                    - enabled
                     type: object
                   service:
                     description: Defining the options for the Tenant Control Plane Service resource.
@@ -225,14 +366,19 @@ spec:
                   allowAddressAsExternalIP:
                     description: AllowAddressAsExternalIP will include tenantControlPlane.Spec.NetworkProfile.Address in the section of ExternalIPs of the Kubernetes Service (only ClusterIP or NodePort)
                     type: boolean
-                  dnsServiceIPs:
+                  certSANs:
+                    description: CertSANs sets extra Subject Alternative Names (SANs) for the API Server signing certificate. Use this field to add additional hostnames when exposing the Tenant Control Plane with third solutions.
+                    items:
+                      type: string
+                    type: array
+                  dnsServiceIPs:
+                    default:
+                    - 10.96.0.10
                     items:
                       type: string
                     type: array
-                  domain:
-                    description: Domain of the tenant control plane
-                    type: string
                   podCidr:
+                    default: 10.244.0.0/16
                     description: CIDR for Kubernetes Pods
                     type: string
                   port:
@@ -241,14 +387,9 @@ spec:
                     format: int32
                     type: integer
                   serviceCidr:
+                    default: 10.96.0.0/16
                     description: Kubernetes Service
                     type: string
-                required:
-                - dnsServiceIPs
-                - domain
-                - podCidr
-                - port
-                - serviceCidr
                 type: object
             required:
             - controlPlane
@@ -257,12 +398,225 @@ spec:
           status:
             description: TenantControlPlaneStatus defines the observed state of TenantControlPlane.
             properties:
+              addons:
+                description: Addons contains the status of the different Addons
+                properties:
+                  coreDNS:
+                    description: AddonStatus defines the observed state of an Addon.
+                    properties:
+                      checksum:
+                        type: string
+                      enabled:
+                        type: boolean
+                      lastUpdate:
+                        format: date-time
+                        type: string
+                    required:
+                    - enabled
+                    type: object
+                  konnectivity:
+                    description: KonnectivityStatus defines the status of Konnectivity as Addon.
+                    properties:
+                      agent:
+                        properties:
+                          checksum:
+                            type: string
+                          lastUpdate:
+                            description: Last time when k8s object was updated
+                            format: date-time
+                            type: string
+                          name:
+                            type: string
+                          namespace:
+                            type: string
+                        type: object
+                      certificate:
+                        description: CertificatePrivateKeyPairStatus defines the status.
+                        properties:
+                          checksum:
+                            type: string
+                          lastUpdate:
+                            format: date-time
+                            type: string
+                          secretName:
+                            type: string
+                        type: object
+                      clusterrolebinding:
+                        properties:
+                          checksum:
+                            type: string
+                          lastUpdate:
+                            description: Last time when k8s object was updated
+                            format: date-time
+                            type: string
+                          name:
+                            type: string
+                          namespace:
+                            type: string
+                        type: object
+                      configMap:
+                        properties:
+                          checksum:
+                            type: string
+                          name:
+                            type: string
+                        type: object
+                      enabled:
+                        type: boolean
+                      kubeconfig:
+                        description: KubeconfigStatus contains information about the generated kubeconfig.
+                        properties:
+                          checksum:
+                            type: string
+                          lastUpdate:
+                            format: date-time
+                            type: string
+                          secretName:
+                            type: string
+                        type: object
+                      sa:
+                        properties:
+                          checksum:
+                            type: string
+                          lastUpdate:
+                            description: Last time when k8s object was updated
+                            format: date-time
+                            type: string
+                          name:
+                            type: string
+                          namespace:
+                            type: string
+                        type: object
+                      service:
+                        description: KubernetesServiceStatus defines the status for the Tenant Control Plane Service in the management cluster.
+                        properties:
+                          conditions:
+                            description: Current service state
+                            items:
+                              description: "Condition contains details for one aspect of the current state of this API Resource. --- This struct is intended for direct use as an array at the field path .status.conditions.  For example, type FooStatus struct{     // Represents the observations of a foo's current state.     // Known .status.conditions.type are: \"Available\", \"Progressing\", and \"Degraded\"     // +patchMergeKey=type     // +patchStrategy=merge     // +listType=map     // +listMapKey=type     Conditions []metav1.Condition `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"` \n     // other fields }"
+                              properties:
+                                lastTransitionTime:
+                                  description: lastTransitionTime is the last time the condition transitioned from one status to another. This should be when the underlying condition changed.  If that is not known, then using the time when the API field changed is acceptable.
+                                  format: date-time
+                                  type: string
+                                message:
+                                  description: message is a human readable message indicating details about the transition. This may be an empty string.
+                                  maxLength: 32768
+                                  type: string
+                                observedGeneration:
+                                  description: observedGeneration represents the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date with respect to the current state of the instance.
+                                  format: int64
+                                  minimum: 0
+                                  type: integer
+                                reason:
+                                  description: reason contains a programmatic identifier indicating the reason for the condition's last transition. Producers of specific condition types may define expected values and meanings for this field, and whether the values are considered a guaranteed API. The value should be a CamelCase string. This field may not be empty.
+                                  maxLength: 1024
+                                  minLength: 1
+                                  pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
+                                  type: string
+                                status:
+                                  description: status of the condition, one of True, False, Unknown.
+                                  enum:
+                                  - "True"
+                                  - "False"
+                                  - Unknown
+                                  type: string
+                                type:
+                                  description: type of condition in CamelCase or in foo.example.com/CamelCase. --- Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be useful (see .node.status.conditions), the ability to deconflict is important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)
+                                  maxLength: 316
+                                  pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
+                                  type: string
+                              required:
+                              - lastTransitionTime
+                              - message
+                              - reason
+                              - status
+                              - type
+                              type: object
+                            type: array
+                            x-kubernetes-list-map-keys:
+                            - type
+                            x-kubernetes-list-type: map
+                          loadBalancer:
+                            description: LoadBalancer contains the current status of the load-balancer, if one is present.
+                            properties:
+                              ingress:
+                                description: Ingress is a list containing ingress points for the load-balancer. Traffic intended for the service should be sent to these ingress points.
+                                items:
+                                  description: 'LoadBalancerIngress represents the status of a load-balancer ingress point: traffic intended for the service should be sent to an ingress point.'
+                                  properties:
+                                    hostname:
+                                      description: Hostname is set for load-balancer ingress points that are DNS based (typically AWS load-balancers)
+                                      type: string
+                                    ip:
+                                      description: IP is set for load-balancer ingress points that are IP based (typically GCE or OpenStack load-balancers)
+                                      type: string
+                                    ports:
+                                      description: Ports is a list of records of service ports If used, every port defined in the service should have an entry in it
+                                      items:
+                                        properties:
+                                          error:
+                                            description: 'Error is to record the problem with the service port The format of the error shall comply with the following rules: - built-in error values shall be specified in this file and those shall use   CamelCase names - cloud provider specific error values must have names that comply with the   format foo.example.com/CamelCase. --- The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)'
+                                            maxLength: 316
+                                            pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
+                                            type: string
+                                          port:
+                                            description: Port is the port number of the service port of which status is recorded here
+                                            format: int32
+                                            type: integer
+                                          protocol:
+                                            default: TCP
+                                            description: 'Protocol is the protocol of the service port of which status is recorded here The supported values are: "TCP", "UDP", "SCTP"'
+                                            type: string
+                                        required:
+                                        - port
+                                        - protocol
+                                        type: object
+                                      type: array
+                                      x-kubernetes-list-type: atomic
+                                  type: object
+                                type: array
+                            type: object
+                          name:
+                            description: The name of the Service for the given cluster.
+                            type: string
+                          namespace:
+                            description: The namespace which the Service for the given cluster is deployed.
+                            type: string
+                          port:
+                            description: The port where the service is running
+                            format: int32
+                            type: integer
+                        required:
+                        - name
+                        - namespace
+                        - port
+                        type: object
+                    required:
+                    - enabled
+                    type: object
+                  kubeProxy:
+                    description: AddonStatus defines the observed state of an Addon.
+                    properties:
+                      checksum:
+                        type: string
+                      enabled:
+                        type: boolean
+                      lastUpdate:
+                        format: date-time
+                        type: string
+                    required:
+                    - enabled
+                    type: object
+                type: object
               certificates:
                 description: Certificates contains information about the different certificates that are necessary to run a kubernetes control plane
                 properties:
                   apiServer:
-                    description: CertificatePrivateKeyPair defines the status.
+                    description: CertificatePrivateKeyPairStatus defines the status.
                     properties:
+                      checksum:
+                        type: string
                       lastUpdate:
                         format: date-time
                         type: string
@@ -270,8 +624,10 @@ spec:
                         type: string
                     type: object
                   apiServerKubeletClient:
-                    description: CertificatePrivateKeyPair defines the status.
+                    description: CertificatePrivateKeyPairStatus defines the status.
                     properties:
+                      checksum:
+                        type: string
                       lastUpdate:
                         format: date-time
                         type: string
@@ -279,8 +635,10 @@ spec:
                         type: string
                     type: object
                   ca:
-                    description: CertificatePrivateKeyPair defines the status.
+                    description: CertificatePrivateKeyPairStatus defines the status.
                     properties:
+                      checksum:
+                        type: string
                       lastUpdate:
                         format: date-time
                         type: string
@@ -288,11 +646,13 @@ spec:
                         type: string
                     type: object
                   etcd:
-                    description: ETCDAPIServerCertificate defines the observed state of ETCD Certificate for API server.
+                    description: ETCDCertificatesStatus defines the observed state of ETCD Certificate for API server.
                     properties:
                       apiServer:
-                        description: ETCDAPIServerCertificate defines the observed state of ETCD Certificate for API server.
+                        description: APIServerCertificatesStatus defines the observed state of ETCD Certificate for API server.
                         properties:
+                          checksum:
+                            type: string
                           lastUpdate:
                             format: date-time
                             type: string
@@ -300,8 +660,10 @@ spec:
                             type: string
                         type: object
                       ca:
-                        description: ETCDAPIServerCertificate defines the observed state of ETCD Certificate for API server.
+                        description: ETCDCertificateStatus defines the observed state of ETCD Certificate for API server.
                         properties:
+                          checksum:
+                            type: string
                           lastUpdate:
                             format: date-time
                             type: string
@@ -310,8 +672,10 @@ spec:
                         type: object
                     type: object
                   frontProxyCA:
-                    description: CertificatePrivateKeyPair defines the status.
+                    description: CertificatePrivateKeyPairStatus defines the status.
                     properties:
+                      checksum:
+                        type: string
                       lastUpdate:
                         format: date-time
                         type: string
@@ -319,8 +683,10 @@ spec:
                         type: string
                     type: object
                   frontProxyClient:
-                    description: CertificatePrivateKeyPair defines the status.
+                    description: CertificatePrivateKeyPairStatus defines the status.
                     properties:
+                      checksum:
+                        type: string
                       lastUpdate:
                         format: date-time
                         type: string
@@ -328,8 +694,10 @@ spec:
                         type: string
                     type: object
                   sa:
-                    description: CertificatePrivateKeyPair defines the status.
+                    description: PublicKeyPrivateKeyPairStatus defines the status.
                     properties:
+                      checksum:
+                        type: string
                       lastUpdate:
                         format: date-time
                         type: string
@@ -343,54 +711,34 @@ spec:
               kubeadmPhase:
                 description: KubeadmPhase contains the status of the kubeadm phases action
                 properties:
-                  addonCoreDNS:
-                    description: KubeadmPhasesStatus contains the status of of a kubeadm phase action.
-                    properties:
-                      kubeadmConfigResourceVersion:
-                        type: string
-                      lastUpdate:
-                        format: date-time
-                        type: string
-                    type: object
-                  addonKubeProxy:
-                    description: KubeadmPhasesStatus contains the status of of a kubeadm phase action.
-                    properties:
-                      kubeadmConfigResourceVersion:
-                        type: string
-                      lastUpdate:
-                        format: date-time
-                        type: string
-                    type: object
                   bootstrapToken:
-                    description: KubeadmPhasesStatus contains the status of of a kubeadm phase action.
+                    description: KubeadmPhaseStatus contains the status of a kubeadm phase action.
                     properties:
-                      kubeadmConfigResourceVersion:
+                      checksum:
                         type: string
                       lastUpdate:
                         format: date-time
                         type: string
                     type: object
                   uploadConfigKubeadm:
-                    description: KubeadmPhasesStatus contains the status of of a kubeadm phase action.
+                    description: KubeadmPhaseStatus contains the status of a kubeadm phase action.
                     properties:
-                      kubeadmConfigResourceVersion:
+                      checksum:
                         type: string
                       lastUpdate:
                         format: date-time
                         type: string
                     type: object
                   uploadConfigKubelet:
-                    description: KubeadmPhasesStatus contains the status of of a kubeadm phase action.
+                    description: KubeadmPhaseStatus contains the status of a kubeadm phase action.
                     properties:
-                      kubeadmConfigResourceVersion:
+                      checksum:
                         type: string
                       lastUpdate:
                         format: date-time
                         type: string
                     type: object
                 required:
-                - addonCoreDNS
-                - addonKubeProxy
                 - bootstrapToken
                 - uploadConfigKubeadm
                 - uploadConfigKubelet
@@ -398,31 +746,34 @@ spec:
               kubeadmconfig:
                 description: KubeadmConfig contains the status of the configuration required by kubeadm
                 properties:
+                  checksum:
+                    description: Checksum of the kubeadm configuration to detect changes
+                    type: string
                   configmapName:
                     type: string
                   lastUpdate:
                     format: date-time
                     type: string
-                  resourceVersion:
-                    type: string
-                required:
-                - resourceVersion
                 type: object
               kubeconfig:
                 description: KubeConfig contains information about the kubenconfigs that control plane pieces need
                 properties:
                   admin:
-                    description: TenantControlPlaneKubeconfigsStatus contains information about a the generated kubeconfig.
+                    description: KubeconfigStatus contains information about the generated kubeconfig.
                     properties:
+                      checksum:
+                        type: string
                       lastUpdate:
                         format: date-time
                         type: string
                       secretName:
                         type: string
                     type: object
-                  controlerManager:
-                    description: TenantControlPlaneKubeconfigsStatus contains information about a the generated kubeconfig.
+                  controllerManager:
+                    description: KubeconfigStatus contains information about the generated kubeconfig.
                     properties:
+                      checksum:
+                        type: string
                       lastUpdate:
                         format: date-time
                         type: string
@@ -430,8 +781,10 @@ spec:
                         type: string
                     type: object
                   scheduler:
-                    description: TenantControlPlaneKubeconfigsStatus contains information about a the generated kubeconfig.
+                    description: KubeconfigStatus contains information about the generated kubeconfig.
                     properties:
+                      checksum:
+                        type: string
                       lastUpdate:
                         format: date-time
                         type: string
@@ -483,6 +836,10 @@ spec:
                           - type
                           type: object
                         type: array
+                      lastUpdate:
+                        description: Last time when deployment was updated
+                        format: date-time
+                        type: string
                       name:
                         description: The name of the Deployment for the given cluster.
                         type: string
@@ -686,8 +1043,6 @@ spec:
                       version:
                         description: Version is the running Kubernetes version of the Tenant Control Plane.
                         type: string
-                    required:
-                    - status
                     type: object
                 type: object
               storage:
@@ -732,6 +1087,40 @@ spec:
                         - name
                         type: object
                     type: object
+                  kine:
+                    properties:
+                      certificate:
+                        properties:
+                          checksum:
+                            type: string
+                          lastUpdate:
+                            format: date-time
+                            type: string
+                          secretName:
+                            type: string
+                        type: object
+                      config:
+                        properties:
+                          checksum:
+                            type: string
+                          secretName:
+                            type: string
+                        type: object
+                      driver:
+                        type: string
+                      setup:
+                        properties:
+                          checksum:
+                            type: string
+                          lastUpdate:
+                            format: date-time
+                            type: string
+                          schema:
+                            type: string
+                          user:
+                            type: string
+                        type: object
+                    type: object
                 type: object
             type: object
         type: object
@@ -1021,8 +1410,8 @@ spec:
         - --leader-elect
         command:
         - /manager
-        image: quay.io/clastix/kamaji:latest
-        imagePullPolicy: IfNotPresent
+        image: clastix/kamaji:latest
+        imagePullPolicy: Always
         livenessProbe:
           httpGet:
             path: /healthz

config/manager/kustomization.yaml

@@ -12,5 +12,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 images:
 - name: controller
-  newName: quay.io/clastix/kamaji
+  newName: clastix/kamaji
   newTag: latest

config/manager/manager.yaml

@@ -30,7 +30,7 @@ spec:
         args:
         - --leader-elect
         image: controller:latest
-        imagePullPolicy: IfNotPresent
+        imagePullPolicy: Always
         name: manager
         securityContext:
           allowPrivilegeEscalation: false

config/samples/kamaji_v1alpha1_tenantcontrolplane.yaml

@@ -5,44 +5,18 @@ metadata:
 spec:
   controlPlane:
     deployment:
-      replicas: 2
-      additionalMetadata:
-        annotations:
-          environment.clastix.io: test
-          tier.clastix.io: "0"
-        labels:
-          tenant.clastix.io: test
-          kind.clastix.io: deployment
+      replicas: 1
     service:
-      additionalMetadata:
-        annotations:
-          environment.clastix.io: test
-          tier.clastix.io: "0"
-        labels:
-          tenant.clastix.io: test
-          kind.clastix.io: service
-      serviceType: ClusterIP
-    ingress:
-      enabled: true
-      hostname: kamaji.local
-      ingressClassName: nginx
-      additionalMetadata:
-        annotations:
-          kubernetes.io/ingress.allow-http: "false"
-          nginx.ingress.kubernetes.io/secure-backends: "true"
-          nginx.ingress.kubernetes.io/ssl-passthrough: "true"      
+      serviceType: LoadBalancer
   kubernetes:
     version: "v1.23.1"
     kubelet:
-      cgroupfs: systemd
+      cgroupfs: cgroupfs
     admissionControllers:
       - ResourceQuota
       - LimitRanger
   networkProfile:
-    address: "127.0.0.1"
     port: 6443
-    domain: "clastix.labs"
-    serviceCidr: "10.96.0.0/16"
-    podCidr: "10.244.0.0/16"
-    dnsServiceIPs:
-      - "10.96.0.10"
+  addons:
+    coreDNS: {}
+    kubeProxy: {}

controllers/resources.go

@@ -0,0 +1,362 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+package controllers
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/go-logr/logr"
+	"github.com/google/uuid"
+	k8stypes "k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+	"github.com/clastix/kamaji/internal/resources"
+	"github.com/clastix/kamaji/internal/resources/konnectivity"
+	"github.com/clastix/kamaji/internal/sql"
+	"github.com/clastix/kamaji/internal/types"
+)
+
+const (
+	separator = ","
+)
+
+type GroupResourceBuilderConfiguration struct {
+	client              client.Client
+	log                 logr.Logger
+	tcpReconcilerConfig TenantControlPlaneReconcilerConfig
+	tenantControlPlane  kamajiv1alpha1.TenantControlPlane
+	DBConnection        sql.DBConnection
+}
+
+type GroupDeleteableResourceBuilderConfiguration struct {
+	client              client.Client
+	log                 logr.Logger
+	tcpReconcilerConfig TenantControlPlaneReconcilerConfig
+	tenantControlPlane  kamajiv1alpha1.TenantControlPlane
+	DBConnection        sql.DBConnection
+}
+
+// GetResources returns a list of resources that will be used to provide tenant control planes
+// Currently there is only a default approach
+// TODO: the idea of this function is to become a factory to return the group of resources according to the given configuration.
+func GetResources(config GroupResourceBuilderConfiguration) []resources.Resource {
+	return getDefaultResources(config)
+}
+
+// GetDeletableResources returns a list of resources that have to be deleted when tenant control planes are deleted
+// Currently there is only a default approach
+// TODO: the idea of this function is to become a factory to return the group of deleteable resources according to the given configuration.
+func GetDeletableResources(config GroupDeleteableResourceBuilderConfiguration) []resources.DeleteableResource {
+	return getDefaultDeleteableResources(config)
+}
+
+func getDefaultResources(config GroupResourceBuilderConfiguration) []resources.Resource {
+	resources := append(getUpgradeResources(config.client, config.tenantControlPlane), getKubernetesServiceResources(config.client, config.tenantControlPlane)...)
+	resources = append(resources, getKubeadmConfigResources(config.client, config.tcpReconcilerConfig, config.tenantControlPlane)...)
+	resources = append(resources, getKubernetesCertificatesResources(config.client, config.log, config.tcpReconcilerConfig, config.tenantControlPlane)...)
+	resources = append(resources, getKubeconfigResources(config.client, config.log, config.tcpReconcilerConfig, config.tenantControlPlane)...)
+	resources = append(resources, getKubernetesStorageResources(config.client, config.log, config.tcpReconcilerConfig, config.DBConnection, config.tenantControlPlane)...)
+	resources = append(resources, getInternalKonnectivityResources(config.client, config.log, config.tcpReconcilerConfig, config.tenantControlPlane)...)
+	resources = append(resources, getKubernetesDeploymentResources(config.client, config.tcpReconcilerConfig, config.tenantControlPlane)...)
+	resources = append(resources, getKubernetesIngressResources(config.client, config.tenantControlPlane)...)
+	resources = append(resources, getKubeadmPhaseResources(config.client, config.log, config.tenantControlPlane)...)
+	resources = append(resources, getKubeadmAddonResources(config.client, config.log, config.tenantControlPlane)...)
+	resources = append(resources, getExternalKonnectivityResources(config.client, config.log, config.tcpReconcilerConfig, config.tenantControlPlane)...)
+
+	return resources
+}
+
+func getDefaultDeleteableResources(config GroupDeleteableResourceBuilderConfiguration) []resources.DeleteableResource {
+	switch config.tcpReconcilerConfig.ETCDStorageType {
+	case types.ETCD:
+		return []resources.DeleteableResource{
+			&resources.ETCDSetupResource{
+				Name:                  "etcd-setup",
+				Client:                config.client,
+				Log:                   config.log,
+				ETCDClientCertsSecret: getNamespacedName(config.tcpReconcilerConfig.ETCDClientSecretNamespace, config.tcpReconcilerConfig.ETCDClientSecretName),
+				ETCDCACertsSecret:     getNamespacedName(config.tcpReconcilerConfig.ETCDCASecretNamespace, config.tcpReconcilerConfig.ETCDCASecretName),
+				Endpoints:             getArrayFromString(config.tcpReconcilerConfig.ETCDEndpoints),
+			},
+		}
+	case types.KineMySQL, types.KinePostgreSQL:
+		return []resources.DeleteableResource{
+			&resources.SQLSetup{
+				Client:       config.client,
+				Name:         "sql-setup",
+				DBConnection: config.DBConnection,
+			},
+		}
+	default:
+		return []resources.DeleteableResource{}
+	}
+}
+
+func getUpgradeResources(c client.Client, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource {
+	return []resources.Resource{
+		&resources.KubernetesUpgrade{
+			Name:   "upgrade",
+			Client: c,
+		},
+	}
+}
+
+func getKubernetesServiceResources(c client.Client, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource {
+	return []resources.Resource{
+		&resources.KubernetesServiceResource{
+			Client: c,
+		},
+	}
+}
+
+func getKubeadmConfigResources(c client.Client, tcpReconcilerConfig TenantControlPlaneReconcilerConfig, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource {
+	return []resources.Resource{
+		&resources.KubeadmConfigResource{
+			Name:                   "kubeadmconfig",
+			ETCDs:                  getArrayFromString(tcpReconcilerConfig.ETCDEndpoints),
+			ETCDCompactionInterval: tcpReconcilerConfig.ETCDCompactionInterval,
+			Client:                 c,
+			TmpDirectory:           getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
+		},
+	}
+}
+
+func getKubernetesCertificatesResources(c client.Client, log logr.Logger, tcpReconcilerConfig TenantControlPlaneReconcilerConfig, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource {
+	return []resources.Resource{
+		&resources.CACertificate{
+			Name:         "ca",
+			Client:       c,
+			Log:          log,
+			TmpDirectory: getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
+		},
+		&resources.FrontProxyCACertificate{
+			Name:         "front-proxy-ca-certificate",
+			Client:       c,
+			Log:          log,
+			TmpDirectory: getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
+		},
+		&resources.SACertificate{
+			Name:         "sa-certificate",
+			Client:       c,
+			Log:          log,
+			TmpDirectory: getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
+		},
+		&resources.APIServerCertificate{
+			Name:         "api-server-certificate",
+			Client:       c,
+			Log:          log,
+			TmpDirectory: getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
+		},
+		&resources.APIServerKubeletClientCertificate{
+			Name:         "api-server-kubelet-client-certificate",
+			Client:       c,
+			Log:          log,
+			TmpDirectory: getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
+		},
+		&resources.FrontProxyClientCertificate{
+			Name:         "front-proxy-client-certificate",
+			Client:       c,
+			Log:          log,
+			TmpDirectory: getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
+		},
+	}
+}
+
+func getKubeconfigResources(c client.Client, log logr.Logger, tcpReconcilerConfig TenantControlPlaneReconcilerConfig, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource {
+	return []resources.Resource{
+		&resources.KubeconfigResource{
+			Name:               "admin-kubeconfig",
+			Client:             c,
+			Log:                log,
+			KubeConfigFileName: resources.AdminKubeConfigFileName,
+			TmpDirectory:       getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
+		},
+		&resources.KubeconfigResource{
+			Name:               "controller-manager-kubeconfig",
+			Client:             c,
+			Log:                log,
+			KubeConfigFileName: resources.ControllerManagerKubeConfigFileName,
+			TmpDirectory:       getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
+		},
+		&resources.KubeconfigResource{
+			Name:               "scheduler-kubeconfig",
+			Client:             c,
+			Log:                log,
+			KubeConfigFileName: resources.SchedulerKubeConfigFileName,
+			TmpDirectory:       getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
+		},
+	}
+}
+
+func getKubernetesStorageResources(c client.Client, log logr.Logger, tcpReconcilerConfig TenantControlPlaneReconcilerConfig, dbConnection sql.DBConnection, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource {
+	switch tcpReconcilerConfig.ETCDStorageType {
+	case types.ETCD:
+		return []resources.Resource{
+			&resources.ETCDCACertificatesResource{
+				Name:                  "etcd-ca-certificates",
+				Client:                c,
+				Log:                   log,
+				ETCDCASecretName:      tcpReconcilerConfig.ETCDCASecretName,
+				ETCDCASecretNamespace: tcpReconcilerConfig.ETCDCASecretNamespace,
+			},
+			&resources.ETCDCertificatesResource{
+				Name:   "etcd-certificates",
+				Client: c,
+				Log:    log,
+			},
+			&resources.ETCDSetupResource{
+				Name:                  "etcd-setup",
+				Client:                c,
+				Log:                   log,
+				ETCDClientCertsSecret: getNamespacedName(tcpReconcilerConfig.ETCDClientSecretNamespace, tcpReconcilerConfig.ETCDClientSecretName),
+				ETCDCACertsSecret:     getNamespacedName(tcpReconcilerConfig.ETCDCASecretNamespace, tcpReconcilerConfig.ETCDCASecretName),
+				Endpoints:             getArrayFromString(tcpReconcilerConfig.ETCDEndpoints),
+			},
+		}
+	case types.KineMySQL, types.KinePostgreSQL:
+		return []resources.Resource{
+			&resources.SQLStorageConfig{
+				Client: c,
+				Name:   "sql-config",
+				Host:   dbConnection.GetHost(),
+				Port:   dbConnection.GetPort(),
+				Driver: dbConnection.Driver(),
+			},
+			&resources.SQLSetup{
+				Client:       c,
+				Name:         "sql-setup",
+				DBConnection: dbConnection,
+				Driver:       dbConnection.Driver(),
+			},
+			&resources.SQLCertificate{
+				Client:                   c,
+				Name:                     "sql-certificate",
+				StorageType:              tcpReconcilerConfig.ETCDStorageType,
+				SQLConfigSecretName:      tcpReconcilerConfig.KineSecretName,
+				SQLConfigSecretNamespace: tcpReconcilerConfig.KineSecretNamespace,
+			},
+		}
+	default:
+		return []resources.Resource{}
+	}
+}
+
+func getKubernetesDeploymentResources(c client.Client, tcpReconcilerConfig TenantControlPlaneReconcilerConfig, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource {
+	return []resources.Resource{
+		&resources.KubernetesDeploymentResource{
+			Client:                 c,
+			ETCDEndpoints:          getArrayFromString(tcpReconcilerConfig.ETCDEndpoints),
+			ETCDCompactionInterval: tcpReconcilerConfig.ETCDCompactionInterval,
+			ETCDStorageType:        tcpReconcilerConfig.ETCDStorageType,
+			KineContainerImage:     tcpReconcilerConfig.KineContainerImage,
+		},
+	}
+}
+
+func getKubernetesIngressResources(c client.Client, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource {
+	return []resources.Resource{
+		&resources.KubernetesIngressResource{
+			Client: c,
+		},
+	}
+}
+
+func getKubeadmPhaseResources(c client.Client, log logr.Logger, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource {
+	return []resources.Resource{
+		&resources.KubeadmPhase{
+			Name:   "upload-config-kubeadm",
+			Client: c,
+			Log:    log,
+			Phase:  resources.PhaseUploadConfigKubeadm,
+		},
+		&resources.KubeadmPhase{
+			Name:   "upload-config-kubelet",
+			Client: c,
+			Log:    log,
+			Phase:  resources.PhaseUploadConfigKubelet,
+		},
+		&resources.KubeadmPhase{
+			Name:   "bootstrap-token",
+			Client: c,
+			Log:    log,
+			Phase:  resources.PhaseBootstrapToken,
+		},
+	}
+}
+
+func getKubeadmAddonResources(c client.Client, log logr.Logger, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource {
+	return []resources.Resource{
+		&resources.KubeadmAddonResource{
+			Name:         "coredns",
+			Client:       c,
+			Log:          log,
+			KubeadmAddon: resources.AddonCoreDNS,
+		},
+		&resources.KubeadmAddonResource{
+			Name:         "kubeproxy",
+			Client:       c,
+			Log:          log,
+			KubeadmAddon: resources.AddonKubeProxy,
+		},
+	}
+}
+
+func getExternalKonnectivityResources(c client.Client, log logr.Logger, tcpReconcilerConfig TenantControlPlaneReconcilerConfig, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource {
+	return []resources.Resource{
+		&konnectivity.ServiceAccountResource{
+			Client: c,
+			Name:   "konnectivity-sa",
+		},
+		&konnectivity.ClusterRoleBindingResource{
+			Client: c,
+			Name:   "konnectivity-clusterrolebinding",
+		},
+		&konnectivity.KubernetesDeploymentResource{
+			Client: c,
+			Name:   "konnectivity-deployment",
+		},
+		&konnectivity.ServiceResource{
+			Client: c,
+			Name:   "konnectivity-service",
+		},
+		&konnectivity.Agent{
+			Client: c,
+			Name:   "konnectivity-agent",
+		},
+	}
+}
+
+func getInternalKonnectivityResources(c client.Client, log logr.Logger, tcpReconcilerConfig TenantControlPlaneReconcilerConfig, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource {
+	return []resources.Resource{
+		&konnectivity.EgressSelectorConfigurationResource{
+			Client: c,
+			Name:   "konnectivity-egress-selector-configuration",
+		},
+		&konnectivity.CertificateResource{
+			Client: c,
+			Log:    log,
+			Name:   "konnectivity-certificate",
+		},
+		&konnectivity.KubeconfigResource{
+			Client: c,
+			Name:   "konnectivity-kubeconfig",
+		},
+	}
+}
+
+func getArrayFromString(s string) []string {
+	var a []string
+	a = append(a, strings.Split(s, separator)...)
+
+	return a
+}
+
+func getNamespacedName(namespace string, name string) k8stypes.NamespacedName {
+	return k8stypes.NamespacedName{Namespace: namespace, Name: name}
+}
+
+func getTmpDirectory(base string, tenantControlPlane kamajiv1alpha1.TenantControlPlane) string {
+	return fmt.Sprintf("%s/%s/%s", base, tenantControlPlane.GetName(), uuid.New())
+}

controllers/storage.go

@@ -0,0 +1,85 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package controllers
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"fmt"
+	"strings"
+
+	"github.com/pkg/errors"
+	corev1 "k8s.io/api/core/v1"
+	k8stypes "k8s.io/apimachinery/pkg/types"
+
+	"github.com/clastix/kamaji/internal/sql"
+	"github.com/clastix/kamaji/internal/types"
+)
+
+func (r *TenantControlPlaneReconciler) getStorageConnection(ctx context.Context) (sql.DBConnection, error) {
+	var driver sql.Driver
+	var dbName string
+
+	// TODO: https://github.com/clastix/kamaji/issues/67
+	switch r.Config.ETCDStorageType {
+	case types.ETCD:
+		return nil, nil
+	case types.KineMySQL:
+		driver = sql.MySQL
+		dbName = "mysql"
+	case types.KinePostgreSQL:
+		driver = sql.PostgreSQL
+	default:
+		return nil, nil
+	}
+
+	secret := &corev1.Secret{}
+	namespacedName := k8stypes.NamespacedName{Namespace: r.Config.KineSecretNamespace, Name: r.Config.KineSecretName}
+	if err := r.Client.Get(ctx, namespacedName, secret); err != nil {
+		return nil, err
+	}
+
+	if t := "kamaji.clastix.io/kine"; string(secret.Type) != t {
+		return nil, fmt.Errorf("expecting a secret of type %s", t)
+	}
+
+	keys := []string{"ca.crt", "server.crt", "server.key", "username", "password"}
+
+	if secret.Data == nil {
+		return nil, fmt.Errorf("the Kine secret %s/%s is missing all the required keys (%s)", secret.GetNamespace(), secret.GetName(), strings.Join(keys, ","))
+	}
+
+	for _, key := range keys {
+		if _, ok := secret.Data[key]; !ok {
+			return nil, fmt.Errorf("missing required key %s for the Kine secret %s/%s", key, secret.GetNamespace(), secret.GetName())
+		}
+	}
+
+	rootCAs := x509.NewCertPool()
+	if ok := rootCAs.AppendCertsFromPEM(secret.Data["ca.crt"]); !ok {
+		return nil, fmt.Errorf("error create root CA for the DB connector")
+	}
+
+	certificate, err := tls.X509KeyPair(secret.Data["server.crt"], secret.Data["server.key"])
+	if err != nil {
+		return nil, errors.Wrap(err, "cannot retrieve x.509 key pair from the Kine Secret")
+	}
+
+	return sql.GetDBConnection(
+		sql.ConnectionConfig{
+			SQLDriver: driver,
+			User:      string(secret.Data["username"]),
+			Password:  string(secret.Data["password"]),
+			Host:      r.Config.KineHost,
+			Port:      r.Config.KinePort,
+			DBName:    dbName,
+			TLSConfig: &tls.Config{
+				ServerName:   r.Config.KineHost,
+				RootCAs:      rootCAs,
+				Certificates: []tls.Certificate{certificate},
+			},
+		},
+	)
+}

controllers/tenantcontrolplane_controller.go

@@ -6,9 +6,7 @@ package controllers
 import (
 	"context"
 	"fmt"
-	"strings"
 
-	"github.com/google/uuid"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	networkingv1 "k8s.io/api/networking/v1"
@@ -21,11 +19,13 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/log"
 
 	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+	kamajierrors "github.com/clastix/kamaji/internal/errors"
 	"github.com/clastix/kamaji/internal/resources"
+	"github.com/clastix/kamaji/internal/sql"
+	"github.com/clastix/kamaji/internal/types"
 )
 
 const (
-	separator = ","
 	finalizer = "finalizer.kamaji.clastix.io"
 )
 
@@ -38,6 +38,7 @@ type TenantControlPlaneReconciler struct {
 
 // TenantControlPlaneReconcilerConfig gives the necessary configuration for TenantControlPlaneReconciler.
 type TenantControlPlaneReconcilerConfig struct {
+	ETCDStorageType           types.ETCDStorageType
 	ETCDCASecretName          string
 	ETCDCASecretNamespace     string
 	ETCDClientSecretName      string
@@ -45,6 +46,12 @@ type TenantControlPlaneReconcilerConfig struct {
 	ETCDEndpoints             string
 	ETCDCompactionInterval    string
 	TmpBaseDirectory          string
+	DBConnection              sql.DBConnection
+	KineSecretName            string
+	KineSecretNamespace       string
+	KineHost                  string
+	KinePort                  int
+	KineContainerImage        string
 }
 
 //+kubebuilder:rbac:groups=kamaji.clastix.io,resources=tenantcontrolplanes,verbs=get;list;watch;create;update;patch;delete
@@ -75,190 +82,71 @@ func (r *TenantControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R
 		return ctrl.Result{}, nil
 	}
 
-	if markedToBeDeleted {
-		registeredDeleteableResources := []resources.DeleteableResource{
-			&resources.ETCDSetupResource{
-				Name:                  "etcd-setup",
-				Client:                r.Client,
-				Scheme:                r.Scheme,
-				Log:                   log,
-				ETCDClientCertsSecret: getNamespacedName(r.Config.ETCDClientSecretNamespace, r.Config.ETCDClientSecretName),
-				ETCDCACertsSecret:     getNamespacedName(r.Config.ETCDCASecretNamespace, r.Config.ETCDCASecretName),
-				Endpoints:             getArrayFromString(r.Config.ETCDEndpoints),
-			},
+	dbConnection, err := r.getStorageConnection(ctx)
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+	defer func() {
+		// TODO: Currently, etcd is not accessed using this dbConnection. For that reason we need this check
+		// Check: https://github.com/clastix/kamaji/issues/67
+		if dbConnection != nil {
+			dbConnection.Close()
 		}
+	}()
 
-		for _, resource := range registeredDeleteableResources {
-			if err := resource.Delete(ctx, tenantControlPlane); err != nil {
+	if markedToBeDeleted {
+		log.Info("marked for deletion, performing clean-up")
+
+		groupDeleteableResourceBuilderConfiguration := GroupDeleteableResourceBuilderConfiguration{
+			client:              r.Client,
+			log:                 log,
+			tcpReconcilerConfig: r.Config,
+			tenantControlPlane:  *tenantControlPlane,
+			DBConnection:        dbConnection,
+		}
+		registeredDeletableResources := GetDeletableResources(groupDeleteableResourceBuilderConfiguration)
+
+		for _, resource := range registeredDeletableResources {
+			if err := resources.HandleDeletion(ctx, resource, tenantControlPlane); err != nil {
 				return ctrl.Result{}, err
 			}
 		}
 
 		if hasFinalizer {
-			controllerutil.RemoveFinalizer(tenantControlPlane, finalizer)
-			if err := r.Update(ctx, tenantControlPlane); err != nil {
+			log.Info("removing finalizer")
+
+			if err := r.RemoveFinalizer(ctx, tenantControlPlane); err != nil {
 				return ctrl.Result{}, err
 			}
 		}
 
+		log.Info("resource deletion has been completed")
+
 		return ctrl.Result{}, nil
 	}
 
 	if !hasFinalizer {
-		controllerutil.AddFinalizer(tenantControlPlane, finalizer)
-		if err := r.Update(ctx, tenantControlPlane); err != nil {
-			return ctrl.Result{}, err
-		}
-
-		return ctrl.Result{}, nil
+		return ctrl.Result{}, r.AddFinalizer(ctx, tenantControlPlane)
 	}
 
-	registeredResources := []resources.Resource{
-		&resources.KubernetesServiceResource{
-			Client: r.Client,
-		},
-		&resources.KubeadmConfigResource{
-			Name:                   "kubeadmconfig",
-			Port:                   tenantControlPlane.Spec.NetworkProfile.Port,
-			KubernetesVersion:      tenantControlPlane.Spec.Kubernetes.Version,
-			PodCIDR:                tenantControlPlane.Spec.NetworkProfile.PodCIDR,
-			ServiceCIDR:            tenantControlPlane.Spec.NetworkProfile.ServiceCIDR,
-			Domain:                 tenantControlPlane.Spec.NetworkProfile.Domain,
-			ETCDs:                  getArrayFromString(r.Config.ETCDEndpoints),
-			ETCDCompactionInterval: r.Config.ETCDCompactionInterval,
-			Client:                 r.Client,
-			Scheme:                 r.Scheme,
-			Log:                    log,
-			TmpDirectory:           getTmpDirectory(r.Config.TmpBaseDirectory, *tenantControlPlane),
-		},
-		&resources.CACertificate{
-			Name:         "ca",
-			Client:       r.Client,
-			Log:          log,
-			TmpDirectory: getTmpDirectory(r.Config.TmpBaseDirectory, *tenantControlPlane),
-		},
-		&resources.FrontProxyCACertificate{
-			Name:         "front-proxy-ca-certificate",
-			Client:       r.Client,
-			Log:          log,
-			TmpDirectory: getTmpDirectory(r.Config.TmpBaseDirectory, *tenantControlPlane),
-		},
-		&resources.SACertificate{
-			Name:         "sa-certificate",
-			Client:       r.Client,
-			Log:          log,
-			TmpDirectory: getTmpDirectory(r.Config.TmpBaseDirectory, *tenantControlPlane),
-		},
-		&resources.APIServerCertificate{
-			Name:         "api-server-certificate",
-			Client:       r.Client,
-			Log:          log,
-			TmpDirectory: getTmpDirectory(r.Config.TmpBaseDirectory, *tenantControlPlane),
-		},
-		&resources.APIServerKubeletClientCertificate{
-			Name:         "api-server-kubelet-client-certificate",
-			Client:       r.Client,
-			Log:          log,
-			TmpDirectory: getTmpDirectory(r.Config.TmpBaseDirectory, *tenantControlPlane),
-		},
-		&resources.FrontProxyClientCertificate{
-			Name:         "front-proxy-client-certificate",
-			Client:       r.Client,
-			Log:          log,
-			TmpDirectory: getTmpDirectory(r.Config.TmpBaseDirectory, *tenantControlPlane),
-		},
-		&resources.KubeconfigResource{
-			Name:               "admin-kubeconfig",
-			Client:             r.Client,
-			Scheme:             r.Scheme,
-			Log:                log,
-			KubeConfigFileName: resources.AdminKubeConfigFileName,
-			TmpDirectory:       getTmpDirectory(r.Config.TmpBaseDirectory, *tenantControlPlane),
-		},
-		&resources.KubeconfigResource{
-			Name:               "controller-manager-kubeconfig",
-			Client:             r.Client,
-			Scheme:             r.Scheme,
-			Log:                log,
-			KubeConfigFileName: resources.ControllerManagerKubeConfigFileName,
-			TmpDirectory:       getTmpDirectory(r.Config.TmpBaseDirectory, *tenantControlPlane),
-		},
-		&resources.KubeconfigResource{
-			Name:               "scheduler-kubeconfig",
-			Client:             r.Client,
-			Scheme:             r.Scheme,
-			Log:                log,
-			KubeConfigFileName: resources.SchedulerKubeConfigFileName,
-			TmpDirectory:       getTmpDirectory(r.Config.TmpBaseDirectory, *tenantControlPlane),
-		},
-		&resources.ETCDCACertificatesResource{
-			Name:                  "etcd-ca-certificates",
-			Client:                r.Client,
-			Log:                   log,
-			ETCDCASecretName:      r.Config.ETCDCASecretName,
-			ETCDCASecretNamespace: r.Config.ETCDCASecretNamespace,
-		},
-		&resources.ETCDCertificatesResource{
-			Name:   "etcd-certificates",
-			Client: r.Client,
-			Log:    log,
-		},
-		&resources.ETCDSetupResource{
-			Name:                  "etcd-setup",
-			Client:                r.Client,
-			Scheme:                r.Scheme,
-			Log:                   log,
-			ETCDClientCertsSecret: getNamespacedName(r.Config.ETCDClientSecretNamespace, r.Config.ETCDClientSecretName),
-			ETCDCACertsSecret:     getNamespacedName(r.Config.ETCDCASecretNamespace, r.Config.ETCDCASecretName),
-			Endpoints:             getArrayFromString(r.Config.ETCDEndpoints),
-		},
-		&resources.KubernetesUpgrade{
-			Name:   "upgrade",
-			Client: r.Client,
-		},
-		&resources.KubernetesDeploymentResource{
-			Client:                 r.Client,
-			ETCDEndpoints:          getArrayFromString(r.Config.ETCDEndpoints),
-			ETCDCompactionInterval: r.Config.ETCDCompactionInterval,
-		},
-		&resources.KubernetesIngressResource{
-			Client: r.Client,
-		},
-		&resources.KubeadmPhaseResource{
-			Name:         "upload-config-kubeadm",
-			Client:       r.Client,
-			Log:          log,
-			KubeadmPhase: resources.PhaseUploadConfigKubeadm,
-		},
-		&resources.KubeadmPhaseResource{
-			Name:         "upload-config-kubelet",
-			Client:       r.Client,
-			Log:          log,
-			KubeadmPhase: resources.PhaseUploadConfigKubelet,
-		},
-		&resources.KubeadmPhaseResource{
-			Name:         "addon-coredns",
-			Client:       r.Client,
-			Log:          log,
-			KubeadmPhase: resources.PhaseAddonCoreDNS,
-		},
-		&resources.KubeadmPhaseResource{
-			Name:         "addon-kubeproxy",
-			Client:       r.Client,
-			Log:          log,
-			KubeadmPhase: resources.PhaseAddonKubeProxy,
-		},
-		&resources.KubeadmPhaseResource{
-			Name:         "bootstrap-token",
-			Client:       r.Client,
-			Log:          log,
-			KubeadmPhase: resources.PhaseBootstrapToken,
-		},
+	groupResourceBuilderConfiguration := GroupResourceBuilderConfiguration{
+		client:              r.Client,
+		log:                 log,
+		tcpReconcilerConfig: r.Config,
+		tenantControlPlane:  *tenantControlPlane,
+		DBConnection:        dbConnection,
 	}
+	registeredResources := GetResources(groupResourceBuilderConfiguration)
 
 	for _, resource := range registeredResources {
 		result, err := resources.Handle(ctx, resource, tenantControlPlane)
 		if err != nil {
+			if kamajierrors.ShouldReconcileErrorBeIgnored(err) {
+				log.V(1).Info("sentinel error, enqueuing back request", "error", err.Error())
+
+				return ctrl.Result{Requeue: true}, nil
+			}
+
 			return ctrl.Result{}, err
 		}
 
@@ -275,6 +163,8 @@ func (r *TenantControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R
 		return ctrl.Result{}, nil
 	}
 
+	log.Info(fmt.Sprintf("%s has been reconciled", tenantControlPlane.GetName()))
+
 	return ctrl.Result{}, nil
 }
 
@@ -324,21 +214,6 @@ func (r *TenantControlPlaneReconciler) updateStatus(ctx context.Context, namespa
 	return nil
 }
 
-func getArrayFromString(s string) []string {
-	var a []string
-	a = append(a, strings.Split(s, separator)...)
-
-	return a
-}
-
-func getNamespacedName(namespace string, name string) k8stypes.NamespacedName {
-	return k8stypes.NamespacedName{Namespace: namespace, Name: name}
-}
-
-func getTmpDirectory(base string, tenantControlPlane kamajiv1alpha1.TenantControlPlane) string {
-	return fmt.Sprintf("%s/%s/%s", base, tenantControlPlane.GetName(), uuid.New())
-}
-
 func hasFinalizer(tenantControlPlane kamajiv1alpha1.TenantControlPlane) bool {
 	for _, f := range tenantControlPlane.GetFinalizers() {
 		if f == finalizer {
@@ -348,3 +223,15 @@ func hasFinalizer(tenantControlPlane kamajiv1alpha1.TenantControlPlane) bool {
 
 	return false
 }
+
+func (r *TenantControlPlaneReconciler) AddFinalizer(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+	controllerutil.AddFinalizer(tenantControlPlane, finalizer)
+
+	return r.Update(ctx, tenantControlPlane)
+}
+
+func (r *TenantControlPlaneReconciler) RemoveFinalizer(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+	controllerutil.RemoveFinalizer(tenantControlPlane, finalizer)
+
+	return r.Update(ctx, tenantControlPlane)
+}

deploy/Makefile

@@ -0,0 +1,16 @@
+include etcd/Makefile
+
+deploy_path := $(patsubst %/,%,$(dir $(abspath $(lastword $(MAKEFILE_LIST)))))
+
+.DEFAULT_GOAL := kamaji
+
+.PHONY: etcd-cluster
+reqs: etcd-cluster
+
+.PHONY: kamaji
+kamaji: reqs
+	@kubectl apply -f $(deploy_path)/../../config/install.yaml
+
+.PHONY: destroy
+destroy: etcd-certificates/cleanup
+	@kubectl delete -f $(deploy_path)/../../config/install.yaml

deploy/README.md

@@ -0,0 +1,26 @@
+# Deploy Kamaji
+
+## Quickstart with KinD
+
+```sh
+make -C kind
+```
+
+## Multi-tenant etcd cluster
+
+> This assumes you already have a running Kubernetes cluster and kubeconfig.
+
+```sh
+make -C etcd
+```
+
+## Multi-tenant cluster using Kine
+
+`kine` is an `etcd` shim that allows using different datastore.
+
+Kamaji actually support the following backends:
+
+- [MySQL](kine/mysql/README.md)
+- [PostgreSQL](kine/postgresql/README.md)
+
+> This assumes you already have a running Kubernetes cluster and kubeconfig.

deploy/etcd/etcd-cluster.yaml

@@ -54,6 +54,13 @@ spec:
         app: etcd
     spec:
       serviceAccountName: etcd
+      topologySpreadConstraints:
+      - maxSkew: 1
+        topologyKey: topology.kubernetes.io/zone
+        whenUnsatisfiable: DoNotSchedule
+        labelSelector:
+          matchLabels:
+            app: etcd
       volumes:
       - name: certs
         secret:
@@ -90,9 +97,10 @@ spec:
           - --peer-trusted-ca-file=/etc/etcd/pki/ca.crt
           - --peer-cert-file=/etc/etcd/pki/peer.pem 
           - --peer-key-file=/etc/etcd/pki/peer-key.pem
+          - --snapshot-count=8000
           - --auto-compaction-mode=periodic
           - --auto-compaction-retention=5m
-          - --v=8
+          - --quota-backend-bytes=8589934592
           env:
           - name: POD_NAME
             valueFrom:
@@ -109,5 +117,4 @@ spec:
       accessModes: ["ReadWriteOnce"]
       resources:
         requests:
-          storage: 2Gi
-
+          storage: 8Gi

deploy/kamaji-azure.env

@@ -1,5 +1,30 @@
+# azure parameters
 export KAMAJI_REGION=westeurope
 export KAMAJI_RG=Kamaji
 # https://docs.microsoft.com/en-us/azure/aks/faq#why-are-two-resource-groups-created-with-aks
 export KAMAJI_CLUSTER=kamaji
 export KAMAJI_NODE_RG=MC_${KAMAJI_RG}_${KAMAJI_CLUSTER}_${KAMAJI_REGION}
+
+# kamaji parameters
+export KAMAJI_NAMESPACE=kamaji-system
+
+# tenant cluster parameters
+export TENANT_NAMESPACE=tenants
+export TENANT_NAME=tenant-00
+export TENANT_DOMAIN=$KAMAJI_REGION.cloudapp.azure.com
+export TENANT_VERSION=v1.23.5
+export TENANT_PORT=6443 # port used to expose the tenant api server
+export TENANT_PROXY_PORT=8132 # port used to expose the konnectivity server
+export TENANT_POD_CIDR=10.36.0.0/16
+export TENANT_SVC_CIDR=10.96.0.0/16
+export TENANT_DNS_SERVICE=10.96.0.10
+
+export TENANT_VM_SIZE=Standard_D2ds_v4
+export TENANT_VM_IMAGE=UbuntuLTS
+export TENANT_RG=$TENANT_NAME
+export TENANT_NSG=$TENANT_NAME-nsg
+export TENANT_VNET_NAME=$TENANT_NAME
+export TENANT_VNET_ADDRESS=172.12.0.0/16
+export TENANT_SUBNET_NAME=$TENANT_NAME-subnet
+export TENANT_SUBNET_ADDRESS=172.12.10.0/24
+export TENANT_VMSS=$TENANT_NAME-vmss

deploy/kamaji-external-etcd.env

@@ -1,5 +0,0 @@
-# etcd machine addresses
-export ETCD0=192.168.32.10
-export ETCD1=192.168.32.11
-export ETCD2=192.168.32.12
-export ETCDHOSTS=($ETCD0 $ETCD1 $ETCD2)

deploy/kamaji-internal-etcd.env

@@ -1,5 +0,0 @@
-# etcd endpoints
-export ETCD_NAMESPACE=etcd-system
-export ETCD0=etcd-0.etcd.${ETCD_NAMESPACE}.svc.cluster.local
-export ETCD1=etcd-1.etcd.${ETCD_NAMESPACE}.svc.cluster.local
-export ETCD2=etcd-2.etcd.${ETCD_NAMESPACE}.svc.cluster.local

deploy/kamaji-tenant-azure.env

@@ -1,22 +0,0 @@
-export KAMAJI_REGION=westeurope
-
-# tenant cluster parameters
-export TENANT_NAMESPACE=tenants
-export TENANT_NAME=tenant-00
-export TENANT_DOMAIN=$KAMAJI_REGION.cloudapp.azure.com
-export TENANT_VERSION=v1.23.4
-export TENANT_ADDR=10.240.0.100 # IP used to expose the tenant control plane
-export TENANT_PORT=6443 # PORT used to expose the tenant control plane
-export TENANT_POD_CIDR=10.36.0.0/16
-export TENANT_SVC_CIDR=10.96.0.0/16
-export TENANT_DNS_SERVICE=10.96.0.10
-
-export TENANT_VM_SIZE=Standard_D2ds_v4
-export TENANT_VM_IMAGE=UbuntuLTS
-export TENANT_RG=$TENANT_NAME
-export TENANT_NSG=$TENANT_NAME-nsg
-export TENANT_VNET_NAME=$TENANT_NAME
-export TENANT_VNET_ADDRESS=192.168.0.0/16
-export TENANT_SUBNET_NAME=$TENANT_NAME-subnet
-export TENANT_SUBNET_ADDRESS=192.168.10.0/24
-export TENANT_VMSS=$TENANT_NAME-vmss

deploy/kamaji-tenant.env

@@ -1,17 +0,0 @@
-
-# tenant cluster parameters
-export TENANT_NAMESPACE=tenants
-export TENANT_NAME=tenant-00
-export TENANT_DOMAIN=clastix.labs
-export TENANT_VERSION=v1.23.1
-export TENANT_ADDR=192.168.32.150 # IP used to expose the tenant control plane
-export TENANT_PORT=6443 # PORT used to expose the tenant control plane
-export TENANT_POD_CIDR=10.36.0.0/16
-export TENANT_SVC_CIDR=10.96.0.0/16
-export TENANT_DNS_SERVICE=10.96.0.10
-
-# tenant node addresses
-export WORKER0=192.168.32.90
-export WORKER1=192.168.32.91
-export WORKER2=192.168.32.92
-export WORKER3=192.168.32.93

deploy/kamaji.env

@@ -0,0 +1,18 @@
+# kamaji parameters
+export KAMAJI_NAMESPACE=kamaji-system
+
+# tenant cluster parameters
+export TENANT_NAMESPACE=tenants
+export TENANT_NAME=tenant-00
+export TENANT_DOMAIN=clastix.labs
+export TENANT_VERSION=v1.23.5
+export TENANT_PORT=6443 # port used to expose the tenant api server
+export TENANT_PROXY_PORT=8132 # port used to expose the konnectivity server
+export TENANT_POD_CIDR=10.36.0.0/16
+export TENANT_SVC_CIDR=10.96.0.0/16
+export TENANT_DNS_SERVICE=10.96.0.10
+
+# tenant node addresses
+export WORKER0=172.12.0.10
+export WORKER1=172.12.0.11
+export WORKER2=172.12.0.12

deploy/kind/Makefile

@@ -2,7 +2,7 @@ kind_path := $(patsubst %/,%,$(dir $(abspath $(lastword $(MAKEFILE_LIST)))))
 
 include ../etcd/Makefile
 
-.PHONY: kind ingress-nginx kamaji-kind-worker-build
+.PHONY: kind ingress-nginx
 
 .DEFAULT_GOAL := kamaji
 
@@ -11,7 +11,10 @@ prometheus-stack:
 	helm repo update
 	helm install prometheus-stack --create-namespace -n monitoring prometheus-community/kube-prometheus-stack
 
-kamaji: kind ingress-nginx etcd-cluster
+reqs: kind ingress-nginx etcd-cluster
+
+kamaji: reqs
+	@kubectl apply -f $(kind_path)/../../config/install.yaml
 
 destroy: kind/destroy etcd-certificates/cleanup
 
@@ -26,11 +29,5 @@ ingress-nginx: ingress-nginx-install
 ingress-nginx-install:
 	kubectl apply -f $(kind_path)/nginx-deploy.yaml
 
-kamaji-kind-worker-build:
-	docker build -f $(kind_path)/kamaji-kind-worker.dockerfile -t quay.io/clastix/kamaji-kind-worker:$${WORKER_VERSION:-latest} .
-
-kamaji-kind-worker-push: kamaji-kind-worker-build
-	docker push quay.io/clastix/kamaji-kind-worker:$${WORKER_VERSION:-latest}
-
 kamaji-kind-worker-join:
 	$(kind_path)/join-node.bash

deploy/kind/README.md

@@ -1,138 +0,0 @@
-# Setup a minimal Kamaji for development
-
-This document explains how to deploy a minimal Kamaji setup on [KinD](https://kind.sigs.k8s.io/) for development scopes. Please refer to the [Kamaji documentation](../../README.md) for understanding all the terms used in this guide, as for example: `admin cluster` and `tenant control plane`.
-
-## Tools
-
-We assume you have installed on your workstation:
-
-- [Docker](https://docs.docker.com/engine/install/)
-- [KinD](https://kind.sigs.k8s.io/)
-- [kubectl](https://kubernetes.io/docs/tasks/tools/)
-- [kubeadm](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/install-kubeadm/)
-- [jq](https://stedolan.github.io/jq/)
-- [openssl](https://www.openssl.org/)
-- [cfssl](https://github.com/cloudflare/cfssl)
-- [cfssljson](https://github.com/cloudflare/cfssl)
-
-## Setup Kamaji on KinD
-
-The instance of Kamaji is made of a single node hosting:
-
-- admin control-plane
-- admin worker
-- multi-tenant etcd cluster
-
-The multi-tenant etcd cluster is deployed as statefulset into the Kamaji node.
-
-Run `make kamaji` to setup Kamaji on KinD.
-
-```bash
-cd ./deploy/kind
-make kamaji
-```
-
-At this moment you will have your KinD up and running and ETCD cluster in multitenant mode. 
-
-### Install Kamaji
-
-```bash
-$ kubectl apply -f ../../config/install.yaml
-```
-
-### Deploy Tenant Control Plane
-
-Now it is the moment of deploying your first tenant control plane.
-
-```bash
-$ kubectl apply -f - <<EOF
-apiVersion: kamaji.clastix.io/v1alpha1
-kind: TenantControlPlane
-metadata:
-  name: tenant1
-spec:
-  controlPlane:
-    deployment:
-      replicas: 2
-      additionalMetadata:
-        annotations:
-          environment.clastix.io: tenant1
-          tier.clastix.io: "0"
-        labels:
-          tenant.clastix.io: tenant1
-          kind.clastix.io: deployment
-    service:
-      additionalMetadata:
-        annotations:
-          environment.clastix.io: tenant1
-          tier.clastix.io: "0"
-        labels:
-          tenant.clastix.io: tenant1
-          kind.clastix.io: service
-      serviceType: NodePort
-    ingress:
-      enabled: false
-  kubernetes:
-    version: "v1.23.4"
-    kubelet:
-      cgroupfs: cgroupfs
-    admissionControllers:
-    - LimitRanger
-    - ResourceQuota
-  networkProfile:
-    address: "172.18.0.2"
-    port: 31443
-    domain: "clastix.labs"
-    serviceCidr: "10.96.0.0/16"
-    podCidr: "10.244.0.0/16"
-    dnsServiceIPs: 
-    - "10.96.0.10"
-EOF
-```
-
-> Check networkProfile fields according to your installation
-> To let Kamaji works in kind, you have indicate that the service must be [NodePort](https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport)
-
-### Get Kubeconfig
-
-Let's retrieve kubeconfig and store in `/tmp/kubeconfig`
-
-```bash
-$ kubectl get secrets tenant1-admin-kubeconfig -o json \
- | jq -r '.data["admin.conf"]' \
- | base64 -d > /tmp/kubeconfig
- ```
-
-It can be export it, to facilitate the next tasks:
-
-```bash
-$ export KUBECONFIG=/tmp/kubeconfig
-```
-
-### Install CNI
-
-We highly recommend to install [kindnet](https://github.com/aojea/kindnet) as CNI for your kamaji TCP.
-
-```bash
-$ kubectl create -f https://raw.githubusercontent.com/aojea/kindnet/master/install-kindnet.yaml
-```
-
-### Join worker nodes
-
-```bash
-$ make kamaji-kind-worker-join
-```
-
-> To add more worker nodes, run again the command above.
-
-Check out the node:
-
-```bash
-$ kubectl get nodes
-NAME           STATUS   ROLES    AGE   VERSION
-d2d4b468c9de   Ready    <none>   44s   v1.23.4
-```
-
-> For more complex scenarios (exposing port, different version and so on), run `join-node.bash`
-
-Tenant control plane provision has been finished in a minimal Kamaji setup based on KinD. Therefore, you could develop, test and make your own experiments with Kamaji.

deploy/kind/cni-kindnet-config.json

@@ -1,33 +0,0 @@
-{
-    "cniVersion": "0.3.1",
-    "name": "kindnet",
-    "plugins": [
-      {
-        "type": "ptp",
-        "ipMasq": false,
-        "ipam": {
-          "type": "host-local",
-          "dataDir": "/run/cni-ipam-state",
-          "routes": [
-            {
-              "dst": "0.0.0.0/0"
-            }
-          ],
-          "ranges": [
-            [
-              {
-                "subnet": "10.244.0.0/24"
-              }
-            ]
-          ]
-        },
-        "mtu": 1500
-      },
-      {
-        "type": "portmap",
-        "capabilities": {
-          "portMappings": true
-        }
-      }
-    ]
-  }

deploy/kind/etcd-cluster.yaml

@@ -1,177 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  creationTimestamp: "2021-12-08T17:49:38Z"
-  name: etcd
-  namespace: kamaji-system
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: etcd-server
-  namespace: kamaji-system
-spec:
-  type: ClusterIP
-  ports:
-    - name: client
-      port: 2379
-      protocol: TCP
-      targetPort: 2379
-  selector:
-    app: etcd
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: etcd
-  namespace: kamaji-system
-spec:
-  clusterIP: None
-  ports:
-    - port: 2379
-      name: client
-    - port: 2380
-      name: peer
-  selector:
-    app: etcd
----
-apiVersion: apps/v1
-kind: StatefulSet
-metadata:
-  name: etcd
-  labels:
-    app: etcd
-  namespace: kamaji-system
-spec:
-  serviceName: etcd
-  selector:
-    matchLabels:
-      app: etcd
-  replicas: 3
-  template:
-    metadata:
-      name: etcd
-      labels:
-        app: etcd
-    spec:
-      serviceAccountName: etcd
-      volumes:
-      - name: certs
-        secret:
-          secretName: etcd-certs
-      containers:
-        - name: etcd
-          image: quay.io/coreos/etcd:v3.5.1
-          ports:
-            - containerPort: 2379
-              name: client
-            - containerPort: 2380
-              name: peer
-          volumeMounts:
-            - name: data
-              mountPath: /var/run/etcd
-            - name: certs
-              mountPath: /etc/etcd/pki
-          command:
-          - etcd 
-          - --data-dir=/var/run/etcd
-          - --name=$(POD_NAME)
-          - --initial-cluster-state=new
-          - --initial-cluster=etcd-0=https://etcd-0.etcd.$(POD_NAMESPACE).svc.cluster.local:2380,etcd-1=https://etcd-1.etcd.$(POD_NAMESPACE).svc.cluster.local:2380,etcd-2=https://etcd-2.etcd.$(POD_NAMESPACE).svc.cluster.local:2380
-          - --initial-advertise-peer-urls=https://$(POD_NAME).etcd.$(POD_NAMESPACE).svc.cluster.local:2380
-          - --initial-cluster-token=kamaji
-          - --listen-client-urls=https://0.0.0.0:2379
-          - --advertise-client-urls=https://etcd-0.etcd.$(POD_NAMESPACE).svc.cluster.local:2379,https://etcd-1.etcd.$(POD_NAMESPACE).svc.cluster.local:2379,https://etcd-2.etcd.$(POD_NAMESPACE).svc.cluster.local:2379,https://etcd-server.$(POD_NAMESPACE).svc.cluster.local:2379
-          - --client-cert-auth=true
-          - --trusted-ca-file=/etc/etcd/pki/ca.crt
-          - --cert-file=/etc/etcd/pki/server.pem
-          - --key-file=/etc/etcd/pki/server-key.pem
-          - --listen-peer-urls=https://0.0.0.0:2380
-          - --peer-client-cert-auth=true
-          - --peer-trusted-ca-file=/etc/etcd/pki/ca.crt
-          - --peer-cert-file=/etc/etcd/pki/peer.pem 
-          - --peer-key-file=/etc/etcd/pki/peer-key.pem
-          - --auto-compaction-mode=periodic
-          - --auto-compaction-retention=5m
-          - --v=8
-          env:
-          - name: POD_NAME
-            valueFrom:
-              fieldRef:
-                fieldPath: metadata.name
-          - name: POD_NAMESPACE
-            valueFrom:
-              fieldRef:
-                fieldPath: metadata.namespace
-          livenessProbe:
-            failureThreshold: 8
-            httpGet:
-              host: 127.0.0.1
-              path: /health
-              port: 2381
-              scheme: HTTP
-            initialDelaySeconds: 10
-            periodSeconds: 10
-            timeoutSeconds: 15
-          startupProbe:
-            failureThreshold: 24
-            httpGet:
-              host: 127.0.0.1
-              path: /health
-              port: 2381
-              scheme: HTTP
-            initialDelaySeconds: 10
-            periodSeconds: 10
-            timeoutSeconds: 15
-  volumeClaimTemplates:
-    - metadata:
-        name: data
-      spec:
-        accessModes: ["ReadWriteOnce"]
-        resources:
-          requests:
-            storage: 8Gi
----
-apiVersion: v1
-kind: Pod
-metadata:
-  labels:
-    app: etcd
-  name: etcd-root-client
-  namespace: kamaji-system
-spec:
-  serviceAccountName: etcd
-  containers:
-  - command:
-    - sleep
-    - infinity
-    env:
-    - name: POD_NAMESPACE
-      valueFrom:
-        fieldRef:
-          fieldPath: metadata.namespace    
-    - name: ETCDCTL_ENDPOINTS
-      value: https://etcd-server.$(POD_NAMESPACE).svc.cluster.local:2379
-    - name: ETCDCTL_CACERT
-      value: /opt/certs/ca/ca.crt
-    - name: ETCDCTL_CERT
-      value: /opt/certs/root-client-certs/tls.crt
-    - name: ETCDCTL_KEY
-      value: /opt/certs/root-client-certs/tls.key
-    image: quay.io/coreos/etcd:v3.5.1
-    imagePullPolicy: IfNotPresent
-    name: etcd-client
-    resources: {}
-    volumeMounts:
-    - name: root-client-certs
-      mountPath: /opt/certs/root-client-certs
-    - name: ca
-      mountPath: /opt/certs/ca     
-  volumes:
-  - name: root-client-certs
-    secret:
-      secretName: root-client-certs
-  - name: ca
-    secret:
-      secretName: etcd-certs     
-

deploy/kind/join-node.bash

@@ -3,11 +3,12 @@
 set -e
 
 # Constants
-export DOCKER_IMAGE_NAME="quay.io/clastix/kamaji-kind-worker"
+export DOCKER_IMAGE_NAME="kindest/node"
 export DOCKER_NETWORK="kind"
 
 # Variables
-export KUBERNETES_VERSION=${1:-latest}
+export KUBERNETES_VERSION=${1:-v1.23.5}
+export KUBECONFIG="${KUBECONFIG:-/tmp/kubeconfig}"
 
 if [ -z $2 ]
 then
@@ -24,9 +25,12 @@ kubectl cluster-info
 echo "Are you pointing to the right tenant control plane? (Type return to continue)"
 read
 
-JOIN_CMD="$(kubeadm --kubeconfig=/tmp/kubeconfig token create --print-join-command) --ignore-preflight-errors=SystemVerification"
+JOIN_CMD="$(kubeadm --kubeconfig=${KUBECONFIG} token create --print-join-command) --ignore-preflight-errors=SystemVerification"
 echo "Deploying new node..."
 NODE=$(docker run -d --privileged -v /lib/modules:/lib/modules:ro -v /var --net $DOCKER_NETWORK $MAPPING_PORT $DOCKER_IMAGE_NAME:$KUBERNETES_VERSION)
 sleep 10
 echo "Joining new node..."
 docker exec -e JOIN_CMD="$JOIN_CMD" $NODE /bin/bash -c "$JOIN_CMD"
+
+echo "Node has joined! Remember to install the kind-net CNI by issuing the following command:"
+echo "  $: kubectl apply -f https://raw.githubusercontent.com/aojea/kindnet/master/install-kindnet.yaml"

deploy/kind/kamaji-kind-worker.dockerfile

@@ -1,4 +0,0 @@
-ARG KUBERNETES_VERSION=v1.23.4
-FROM kindest/node:$KUBERNETES_VERSION
-
-COPY ./cni-kindnet-config.json /etc/cni/net.d/10-kindnet.conflist

deploy/kind/kind-kamaji.yaml

@@ -10,6 +10,10 @@ nodes:
     nodeRegistration:
       kubeletExtraArgs:
         node-labels: "ingress-ready=true"
+  ## required for Cluster API local development
+  extraMounts:
+    - hostPath: /var/run/docker.sock
+      containerPath: /var/run/docker.sock
   extraPortMappings:
     ## expose port 80 of the node to port 80 on the host
   - containerPort: 80
@@ -19,12 +23,15 @@ nodes:
   - containerPort: 443
     hostPort: 443
     protocol: TCP
+    ## expose port 31132 of the node to port 31132 on the host for konnectivity
+  - containerPort: 31132
+    hostPort: 31132
+    protocol: TCP   
     ## expose port 31443 of the node to port 31443 on the host
   - containerPort: 31443
     hostPort: 31443
-    protocol: TCP    
+    protocol: TCP
     ## expose port 6443 of the node to port 8443 on the host
   - containerPort: 6443
     hostPort: 8443
     protocol: TCP
-

deploy/kine/README.md

@@ -0,0 +1,47 @@
+# Kine integration
+
+[kine](https://github.com/k3s-io/kine) is an `etcd` shim that allows to use a different datastore for your Kubernetes cluster.
+
+Kamaji actually allows to run a shared datastore using different MySQL and PostgreSQL schemas per Tenant.
+This can help in overcoming the `etcd` limitation regarding scalability and cluster size, as well with HA and replication.
+
+## Kamaji additional CLI flags
+
+Once a compatible database is running, we need to provide information about it to Kamaji by using the following flags:
+
+```
+--etcd-storage-type={kine-mysql,kine-postgresql}
+--kine-host=<database host>
+--kine-port=<database port>
+--kine-secret-name=<secret name>
+--kine-secret-namespace=<secret namespace>
+```
+
+## Kine Secret
+
+The Kine Secret must be configured as follows:
+
+```yaml
+apiVersion: v1
+data:
+  ca.crt: "content of the Certificate Authority for SSL connection"
+  password: "password of the super user"
+  server.crt: "content of the certificate for SSL connection"
+  server.key: "content of the private key for SSL connection"
+  username: "username of the super user"
+kind: Secret
+metadata:
+  name: kine-secret
+  namespace: kamaji-system
+type: kamaji.clastix.io/kine
+```
+
+> Please, pay attention to the type `kamaji.clastix.io/kine`: this check is enforced at the code level to ensure the required data is provided.
+
+> Actually, the `kine` integration expects a secured connection to the database since the sensitivity data of the Tenant.
+
+## Drivers
+
+Further details on the setup for each driver are available here:
+- [MySQL/MariaDB](../deploy/kine/mysql/README.md)
+- [PostgreSQL](../deploy/kine/postgresql/README.md)

deploy/kine/mysql/Makefile

@@ -0,0 +1,40 @@
+ROOT_DIR:=$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))
+
+mariadb: mariadb-certificates mariadb-secret mariadb-kine-secret mariadb-deployment
+
+mariadb-certificates:
+	rm -rf $(ROOT_DIR)/certs && mkdir $(ROOT_DIR)/certs
+	cfssl gencert -initca $(ROOT_DIR)/ca-csr.json | cfssljson -bare $(ROOT_DIR)/certs/ca
+	@mv $(ROOT_DIR)/certs/ca.pem $(ROOT_DIR)/certs/ca.crt
+	@mv $(ROOT_DIR)/certs/ca-key.pem $(ROOT_DIR)/certs/ca.key
+	cfssl gencert -ca=$(ROOT_DIR)/certs/ca.crt -ca-key=$(ROOT_DIR)/certs/ca.key \
+		-config=$(ROOT_DIR)/config.json -profile=server \
+		$(ROOT_DIR)/server-csr.json | cfssljson -bare $(ROOT_DIR)/certs/server
+	@mv $(ROOT_DIR)/certs/server.pem $(ROOT_DIR)/certs/server.crt
+	@mv $(ROOT_DIR)/certs/server-key.pem $(ROOT_DIR)/certs/server.key
+	chmod 644 $(ROOT_DIR)/certs/*
+
+mariadb-secret:
+	@kubectl -n kamaji-system create secret generic mysql-config \
+		--from-file=$(ROOT_DIR)/certs/ca.crt --from-file=$(ROOT_DIR)/certs/ca.key \
+		--from-file=$(ROOT_DIR)/certs/server.key --from-file=$(ROOT_DIR)/certs/server.crt \
+		--from-file=$(ROOT_DIR)/mysql-ssl.cnf \
+		--from-literal=MYSQL_ROOT_PASSWORD=root \
+		--dry-run=client -o yaml | kubectl apply -f -
+
+mariadb-kine-secret:
+	@\
+		CA=$$(cat $(ROOT_DIR)/certs/ca.crt | base64 | tr -d '\n') \
+		CRT=$$(cat $(ROOT_DIR)/certs/server.crt | base64 | tr -d '\n') \
+		KEY=$$(cat $(ROOT_DIR)/certs/server.key | base64 | tr -d '\n') \
+		ROOT_USERNAME=$$(echo -n root | base64) \
+		ROOT_PASSWORD=$$(kubectl -n kamaji-system get secret mysql-config -o jsonpath='{.data.MYSQL_ROOT_PASSWORD}') \
+		envsubst < $(ROOT_DIR)/../secret.yaml | kubectl -n kamaji-system apply -f -
+
+mariadb-deployment:
+	@kubectl -n kamaji-system apply -f $(ROOT_DIR)/mariadb.yaml
+
+mariadb-destroy:
+	@kubectl delete -n kamaji-system -f $(ROOT_DIR)/mariadb.yaml --ignore-not-found
+	@kubectl delete -n kamaji-system secret mysql-config --ignore-not-found
+	@kubectl delete -n kamaji-system secret kine-secret --ignore-not-found

deploy/kine/mysql/README.md

@@ -0,0 +1,82 @@
+# MySQL as Kubernetes Storage
+
+Kamaji offers the possibility of having a different storage system than `ETCD` thanks to [kine](https://github.com/k3s-io/kine). One of the implementations is [MySQL](https://www.mysql.com/).
+
+Kamaji project is developed using [kind](https://kind.sigs.k8s.io), therefore, MySQL (or [MariaDB](https://mariadb.org/) in this case) will be deployed into the local kubernetes cluster in order to be used as storage for the tenants.
+
+There is a Makefile to help with the process:
+
+# Setup
+
+Setup of the MySQL/MariaDB backend can be easily issued with a single command.
+
+```bash
+$ make mariadb
+```
+
+This action will perform all the necessary stuffs to have MariaDB as Kubernetes storage backend using kine.
+
+```shell
+rm -rf /home/prometherion/Documents/clastix/kamaji/deploy/mysql/certs && mkdir /home/prometherion/Documents/clastix/kamaji/deploy/mysql/certs
+cfssl gencert -initca /home/prometherion/Documents/clastix/kamaji/deploy/mysql/ca-csr.json | cfssljson -bare /home/prometherion/Documents/clastix/kamaji/deploy/mysql/certs/ca
+2022/08/18 23:52:56 [INFO] generating a new CA key and certificate from CSR
+2022/08/18 23:52:56 [INFO] generate received request
+2022/08/18 23:52:56 [INFO] received CSR
+2022/08/18 23:52:56 [INFO] generating key: rsa-2048
+2022/08/18 23:52:56 [INFO] encoded CSR
+2022/08/18 23:52:56 [INFO] signed certificate with serial number 310428005543054656774215122317606431230766314770
+cfssl gencert -ca=/home/prometherion/Documents/clastix/kamaji/deploy/mysql/certs/ca.crt -ca-key=/home/prometherion/Documents/clastix/kamaji/deploy/mysql/certs/ca.key \
+        -config=/home/prometherion/Documents/clastix/kamaji/deploy/mysql/config.json -profile=server \
+        /home/prometherion/Documents/clastix/kamaji/deploy/mysql/server-csr.json | cfssljson -bare /home/prometherion/Documents/clastix/kamaji/deploy/mysql/certs/server
+2022/08/18 23:52:56 [INFO] generate received request
+2022/08/18 23:52:56 [INFO] received CSR
+2022/08/18 23:52:56 [INFO] generating key: rsa-2048
+2022/08/18 23:52:56 [INFO] encoded CSR
+2022/08/18 23:52:56 [INFO] signed certificate with serial number 582698914718104852311252458344736030793138969927
+chmod 644 /home/prometherion/Documents/clastix/kamaji/deploy/mysql/certs/*
+secret/mysql-config created
+secret/kine-secret created
+serviceaccount/mariadb created
+service/mariadb created
+deployment.apps/mariadb created
+persistentvolumeclaim/pvc-mariadb created
+```
+
+## Certificate creation
+
+```bash
+$ make mariadb-certificates
+```
+
+Communication between kine and the backend is encrypted, therefore, a CA and a certificate from it must be created.
+
+## Secret Deployment
+
+```bash
+$ make mariadb-secrets
+```
+
+Previous certificates and MySQL configuration have to be available in order to be used.
+They will be under the secret `kamaji-system:mysql-config`, used by the MySQL/MariaDB instance.
+
+## Kine Secret
+
+```bash
+$ make mariadb-kine-secret
+```
+
+Organize the required Kine data such as username, password, CA, certificate, and private key to be stored in the Kamaji desired format.
+
+## Deployment
+
+```bash
+$ make mariadb-deployment
+```
+
+Finally, starts the MySQL/MariaDB installation with all the required settings, such as SSL connection, and configuration.
+
+# Cleanup
+
+```bash
+$ make mariadb-destroy
+```

deploy/kine/mysql/ca-csr.json

@@ -0,0 +1,18 @@
+{
+  "CN": "Clastix CA",
+  "key": {
+    "algo": "rsa",
+    "size": 2048
+  },
+  "names": [
+    {
+      "C": "IT",
+      "ST": "Italy",
+      "L": "Milan"
+    }
+  ],
+  "hosts": [
+    "127.0.0.1",
+    "localhost"
+  ]
+}

deploy/kine/mysql/config.json

@@ -0,0 +1,18 @@
+{
+  "signing": {
+      "default": {
+          "expiry": "8760h"
+      },
+      "profiles": {
+          "server": {
+              "expiry": "8760h",
+              "usages": [
+                  "signing",
+                  "key encipherment",
+                  "server auth",
+                  "client auth"
+              ]
+          }
+      }
+  }
+}

deploy/kine/mysql/kine.yaml

@@ -0,0 +1,61 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kine-tenant
+  namespace:
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: kine-tenant
+  namespace:
+spec:
+  type: ClusterIP
+  ports:
+    - name: server
+      port: 2379
+      protocol: TCP
+      targetPort: 2379
+  selector:
+    app: kine-tenant
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: kine-tenant
+  labels:
+    app: kine-tenant
+  namespace:
+spec:
+  selector:
+    matchLabels:
+      app: kine-tenant
+  replicas: 1
+  template:
+    metadata:
+      name: kine-tenant
+      labels:
+        app: kine-tenant
+    spec:
+      serviceAccountName: kine-tenant
+      volumes:
+      - name: certs
+        secret:
+          secretName: mysql-certs
+      containers:
+        - name: kine-tenant
+          image: rancher/kine:v0.9.2-amd64
+          ports:
+            - containerPort: 2379
+              name: server
+          volumeMounts:
+          - name: certs
+            mountPath: /kine
+          env:    
+          - name: GODEBUG
+            value: "x509ignoreCN=0"
+          args:
+          - --endpoint=mysql://tenant1:tenant1@tcp(mysql:3306)/tenant1
+          - --ca-file=/kine/ca.crt
+          - --cert-file=/kine/server.crt
+          - --key-file=/kine/server.key

deploy/kine/mysql/mariadb.yaml

@@ -0,0 +1,77 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: mariadb
+  namespace:
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: mariadb
+  namespace:
+spec:
+  type: ClusterIP
+  ports:
+    - name: server
+      port: 3306
+      protocol: TCP
+      targetPort: 3306
+  selector:
+    app: mariadb
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: mariadb
+  labels:
+    app: mariadb
+  namespace:
+spec:
+  selector:
+    matchLabels:
+      app: mariadb
+  replicas: 1
+  template:
+    metadata:
+      name: mariadb
+      labels:
+        app: mariadb
+    spec:
+      serviceAccountName: mariadb
+      volumes:
+      - name: certs
+        secret:
+          secretName: mysql-config
+      - name: data
+        persistentVolumeClaim:
+          claimName: pvc-mariadb
+      containers:
+        - name: mariadb
+          image: mariadb:10.7.4
+          ports:
+            - containerPort: 3306
+              name: server
+          volumeMounts:
+          - name: data
+            mountPath: /var/lib/mariadb
+          - name: certs
+            mountPath: /etc/mysql/conf.d/
+          env:
+          - name: MYSQL_ROOT_PASSWORD
+            valueFrom:
+              secretKeyRef:
+                name: mysql-config
+                key: MYSQL_ROOT_PASSWORD                
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: pvc-mariadb
+  namespace:
+spec:
+  accessModes:
+  - ReadWriteOnce
+  resources:
+    requests:
+      storage: 2Gi
+  storageClassName: standard

deploy/kine/mysql/mysql-ssl.cnf

@@ -0,0 +1,5 @@
+[mysqld]
+ssl-ca=/etc/mysql/conf.d/ca.crt
+ssl-cert=/etc/mysql/conf.d/server.crt
+ssl-key=/etc/mysql/conf.d/server.key
+require_secure_transport=ON

deploy/kine/mysql/server-csr.json

@@ -0,0 +1,19 @@
+{
+  "CN": "mariadb.kamaji-system.svc.cluster.local",
+  "key": {
+    "algo": "rsa",
+    "size": 2048
+  },
+  "hosts": [
+    "127.0.0.1",
+    "localhost",
+    "mariadb",
+    "mariadb.kamaji-system",
+    "mariadb.kamaji-system.svc",
+    "mariadb.kamaji-system.svc.cluster.local",
+    "mysql",
+    "mysql.kamaji-system",
+    "mysql.kamaji-system.svc",
+    "mysql.kamaji-system.svc.cluster.local"
+  ]
+}

deploy/kine/postgresql/Makefile

@@ -0,0 +1,36 @@
+ROOT_DIR:=$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))
+
+postgresql: cnpg-setup cnpg-deploy postgresql-secret postgresql-kine-secret
+
+cnpg-setup:
+	@kubectl apply -f https://raw.githubusercontent.com/cloudnative-pg/cloudnative-pg/main/releases/cnpg-1.16.0.yaml
+
+cnpg-deploy:
+	@kubectl -n cnpg-system rollout status deployment/cnpg-controller-manager
+	@kubectl -n kamaji-system apply -f postgresql.yaml
+	@while ! kubectl -n kamaji-system get secret postgresql-superuser > /dev/null 2>&1; do sleep 1; done
+
+CNPG = $(shell git rev-parse --show-toplevel)/bin/kubectl-cnpg
+cnpg:
+	@test -f $(shell git rev-parse --show-toplevel)/bin/kubectl-cnpg || curl -sSfL \
+      https://github.com/cloudnative-pg/cloudnative-pg/raw/main/hack/install-cnpg-plugin.sh | \
+      sh -s -- -b $(shell git rev-parse --show-toplevel)/bin
+
+postgresql-secret: cnpg
+	@kubectl -n kamaji-system get secret postgres-root-cert > /dev/null 2>&1 || $(CNPG) certificate postgres-root-cert \
+    		--cnpg-cluster postgresql \
+    		--cnpg-user $$(kubectl -n kamaji-system get secret postgresql-superuser -o jsonpath='{.data.username}' | base64 -d)
+
+postgresql-kine-secret:
+	@\
+		CA=$$(kubectl -n kamaji-system get secret postgresql-ca -o jsonpath='{.data.ca\.crt}') \
+		CRT=$$(kubectl -n kamaji-system get secret postgres-root-cert -o jsonpath='{.data.tls\.crt}') \
+		KEY=$$(kubectl -n kamaji-system get secret postgres-root-cert -o jsonpath='{.data.tls\.key}') \
+		ROOT_USERNAME=$$(kubectl -n kamaji-system get secret postgresql-superuser -o jsonpath='{.data.username}' ) \
+		ROOT_PASSWORD=$$(kubectl -n kamaji-system get secret postgresql-superuser -o jsonpath='{.data.password}' ) \
+		envsubst < $(ROOT_DIR)/../secret.yaml | kubectl -n kamaji-system apply -f -
+
+postgresql-destroy:
+	@kubectl delete -f https://raw.githubusercontent.com/cloudnative-pg/cloudnative-pg/main/releases/cnpg-1.16.0.yaml --ignore-not-found && \
+	kubectl delete secret postgres-root-cert --ignore-not-found && \
+	kubectl delete secret kine-secret --ignore-not-found

deploy/kine/postgresql/README.md

@@ -0,0 +1,76 @@
+# PostgreSQL as Kubernetes Storage
+
+Kamaji offers the possibility of having a different storage system than `etcd` thanks to [kine](https://github.com/k3s-io/kine). One of the implementations is [PostgreSQL](https://www.postgresql.org/).
+
+Kamaji project is developed using [kind](https://kind.sigs.k8s.io), therefore, a PostgreSQL instance must be deployed in advance into the local kubernetes cluster in order to be used as storage for the tenants.
+For the sake of simplicity, the [cloudnative-pg](https://cloudnative-pg.io/) Operator will be used to simplify the setup of it.
+
+There is a Makefile to help with the process:
+
+## Setup
+
+```bash
+$ make postgresql
+```
+
+This target will install the `cloudnative-pg`, creating the PostgreSQL instance in the Kamaji Namespace, along with the generation of the required Secret resource for the kine integration.
+
+This action is idempotent and doesn't overwrite values if they already exist.
+
+```shell
+namespace/cnpg-system unchanged
+customresourcedefinition.apiextensions.k8s.io/backups.postgresql.cnpg.io configured
+customresourcedefinition.apiextensions.k8s.io/clusters.postgresql.cnpg.io configured
+customresourcedefinition.apiextensions.k8s.io/poolers.postgresql.cnpg.io configured
+customresourcedefinition.apiextensions.k8s.io/scheduledbackups.postgresql.cnpg.io configured
+serviceaccount/cnpg-manager unchanged
+clusterrole.rbac.authorization.k8s.io/cnpg-manager configured
+clusterrolebinding.rbac.authorization.k8s.io/cnpg-manager-rolebinding unchanged
+configmap/cnpg-default-monitoring unchanged
+service/cnpg-webhook-service unchanged
+deployment.apps/cnpg-controller-manager unchanged
+mutatingwebhookconfiguration.admissionregistration.k8s.io/cnpg-mutating-webhook-configuration configured
+validatingwebhookconfiguration.admissionregistration.k8s.io/cnpg-validating-webhook-configuration configured
+deployment "cnpg-controller-manager" successfully rolled out
+cluster.postgresql.cnpg.io/postgresql unchanged
+secret/postgres-root-cert created
+secret/kine-secret created
+```
+
+## Operator setup
+
+```bash
+$ make cnpg-setup
+```
+
+This target will apply all the required manifests with the `cloudnative-pg` CRD, and required RBAC, and Deployment.
+
+Release [v1.16.0](https://github.com/cloudnative-pg/cloudnative-pg/releases/tag/v1.16.0) has been tested successfully.
+
+## SSL certificate Secret generation
+
+```bash
+$ make postgresql-secret
+```
+
+This target will download locally the `kubectl-cnpg` utility to generate an SSL certificate required to secure the connection to the PostgreSQL instance.
+
+## Kine Secret generation
+
+```bash
+$ make postgresql-kine-secret
+```
+
+Generate the Kine secret required for Kamaji.
+
+> Requires the generation of the `postgresql-secret`
+
+## Teardown
+
+```bash
+$ make postgresql-destroy
+```
+
+This will lead to the deletion of the `cloudnative-pg` Operator, along with any instance, and related secrets.
+
+This action is idempotent.

deploy/kine/postgresql/postgresql.yaml

@@ -0,0 +1,13 @@
+apiVersion: postgresql.cnpg.io/v1
+kind: Cluster
+metadata:
+  name: postgresql
+spec:
+  description: PostgreSQL cluster used by Kamaji along with kine
+  instances: 3
+  postgresql:
+    pg_hba:
+      - hostssl app all all cert  # makes authentication entirely based on certificates
+  primaryUpdateStrategy: unsupervised
+  storage:
+    size: 1Gi

deploy/kine/secret.yaml

@@ -0,0 +1,14 @@
+# secret.yaml is the Secret object that Kamaji is expecting to user to connect to the Kine SQL datastore:
+# certificates keys are required, username and password are optional.
+apiVersion: v1
+kind: Secret
+data:
+  ca.crt: ${CA}
+  server.crt: ${CRT}
+  server.key: ${KEY}
+  username: ${ROOT_USERNAME}
+  password: ${ROOT_PASSWORD}
+metadata:
+  creationTimestamp: null
+  name: kine-secret
+type: kamaji.clastix.io/kine

deploy/nodes-prerequisites.sh

@@ -18,7 +18,9 @@ EOF
 for i in "${!HOSTS[@]}"; do
   HOST=${HOSTS[$i]}
   ssh ${USER}@${HOST} -t 'sudo apt update && sudo apt install -y containerd'
-  ssh ${USER}@${HOST} -t 'sudo systemctl start containerd && sudo systemctl enable containerd'
+  ssh ${USER}@${HOST} -t 'sudo mkdir -p /etc/containerd'
+  ssh ${USER}@${HOST} -t 'containerd config default | sed -e "s#SystemdCgroup = false#SystemdCgroup = true#g" | sudo tee -a /etc/containerd/config.toml'
+  ssh ${USER}@${HOST} -t 'sudo systemctl restart containerd && sudo systemctl enable containerd'
   scp containerd.conf ${USER}@${HOST}:
   ssh ${USER}@${HOST} -t 'sudo chown -R root:root containerd.conf && sudo mv containerd.conf /etc/modules-load.d/containerd.conf'
   ssh ${USER}@${HOST} -t 'sudo modprobe overlay && sudo modprobe br_netfilter'

deploy/tenant-cloudinit.yaml

@@ -21,10 +21,12 @@ runcmd:
   - sudo modprobe overlay
   - sudo modprobe br_netfilter
   - sudo sysctl --system
-  - sudo systemctl start containerd
+  - sudo mkdir -p /etc/containerd
+  - containerd config default | sed -e 's#SystemdCgroup = false#SystemdCgroup = true#g' | sudo tee -a /etc/containerd/config.toml
+  - sudo systemctl restart containerd
   - sudo systemctl enable containerd
   - sudo curl -fsSLo /usr/share/keyrings/kubernetes-archive-keyring.gpg https://packages.cloud.google.com/apt/doc/apt-key.gpg
   - echo "deb [signed-by=/usr/share/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" | sudo tee /etc/apt/sources.list.d/kubernetes.list
   - sudo apt update
-  - sudo apt install -y kubelet kubeadm kubectl containerd
-  - sudo apt-mark hold kubelet kubeadm kubectl
+  - sudo apt install -y kubelet=1.23.5-00 kubeadm=1.23.5-00 kubectl=1.23.5-00
+  - sudo apt-mark hold kubelet kubeadm kubectl

docs/README.md

@@ -0,0 +1,13 @@
+# Kamaji documentation
+
+- [Core Concepts](./concepts.md)
+- [Architecture](./architecture.md)
+- [Getting started](./getting-started-with-kamaji.md)
+- Guides:
+  - [Deploy Kamaji](./kamaji-deployment-guide.md)
+  - [Deploy Kamaji on Azure](./kamaji-azure-deployment-guide.md)
+  - Deploy Kamaji on AWS
+  - Deploy Kamaji on GCP
+  - Deploy Kamaji on OpenStack
+- [Reference](./reference.md)
+- [Versioning](./versioning.md)

docs/concepts.md

@@ -1 +1,31 @@
-# Kamaji concepts
+# Core Concepts
+
+Kamaji is a Kubernetes Operator. It turns any Kubernetes cluster into an _“admin cluster”_ to orchestrate other Kubernetes clusters called _“tenant clusters”_. 
+
+## Tenant Control Plane
+What makes Kamaji special is that the Control Plane of a _“tenant cluster”_ is just one or more regular pods running in a namespace of the _“admin cluster”_ instead of a dedicated set of Virtual Machines. This solution makes running control planes at scale cheaper and easier to deploy and operate. The Tenant Control Plane components are packaged in the same way they are running in bare metal or virtual nodes. We leverage the `kubeadm` code to set up the control plane components as they were running on their own server. The unchanged images of upstream `kube-apiserver`, `kube-scheduler`, and `kube-controller-manager` are used.
+
+High Availability and rolling updates of the Tenant Control Plane pods are provided by a regular Deployment. Autoscaling based on the metrics is available. A Service is used to espose the Tenant Control Plane outside of the _“admin cluster”_. The `LoadBalancer` service type is used, `NodePort` and `ClusterIP` with an Ingress Controller are still viable options, depending on the case. 
+
+Kamaji offers a [Custom Resource Definition](https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/) to provide a declarative approach of managing a Tenant Control Plane. This *CRD* is called `TenantControlPlane`, or `tcp` in short.
+
+## Tenant worker nodes
+And what about the tenant worker nodes? They are just _"worker nodes"_, i.e. regular virtual or bare metal machines, connecting to the APIs server of the Tenant Control Plane. Kamaji's goal is to manage the lifecycle of hundreds of these _“tenant clusters”_, not only one, so how to add another tenant cluster to Kamaji? As you could expect, you have just deploys a new Tenant Control Plane in one of the _“admin cluster”_ namespace, and then joins the tenant worker nodes to it.
+
+All the tenant clusters built with Kamaji are fully compliant CNCF Kubernetes clusters and are compatible with the standard Kubernetes toolchains everybody knows and loves.
+
+## Save the state
+Putting the Tenant Control Plane in a pod is the easiest part. Also, we have to make sure each tenant cluster saves the state to be able to store and retrieve data. A dedicated `etcd` cluster for each tenant cluster doesn’t scale well for a managed service because `etcd` data persistence can be cumbersome at scale, rising the operational effort to mitigate it. So we have to find an alternative keeping in mind our goal for a resilient and cost-optimized solution at the same time. As we can deploy any Kubernetes cluster with an external `etcd` cluster, we explored this option for the tenant control planes. On the admin cluster, we deploy a multi-tenant `etcd` cluster storing the state of multiple tenant clusters.
+
+With this solution, the resiliency is guaranteed by the usual `etcd` mechanism, and the pods' count remains under control, so it solves the main goal of resiliency and costs optimization. The trade-off here is that we have to operate an external `etcd` cluster, in addition to `etcd` of the _“admin cluster”_ and manage the access to be sure that each _“tenant cluster”_ uses only its data. Also, there are limits in size in `etcd`, defaulted to 2GB and configurable to a maximum of 8GB. We’re solving this issue by pooling multiple `etcd` togheter and sharding the Tenant Control Planes.
+
+Optionally, Kamaji offers the possibility of using a different storage system than `etcd` to save the state of the tenants' clusters, like MySQL or PostgreSQL compatible databases, thanks to the [kine](https://github.com/k3s-io/kine) integration. 
+
+## Requirements of design
+These are requirements of design behind Kamaji:
+
+- Communication between the _“admin cluster”_ and a _“tenant cluster”_ is unidirectional. The _“admin cluster”_ manages a _“tenant cluster”_, but a _“tenant cluster”_ has no awareness of the _“admin cluster”_.
+- Communication between different _“tenant clusters”_ is not allowed.
+- The worker nodes of tenant should not run anything beyond tenant's workloads.
+
+Goals and scope may vary as the project evolves.

docs/getting-started-with-kamaji.md

@@ -1,604 +1,173 @@
-# Setup a Kamaji environment
-This getting started guide will lead you through the process of creating a basic working Kamaji setup.
+# Setup a minimal Kamaji for development
 
-Kamaji requires:
+This document explains how to deploy a minimal Kamaji setup on [KinD](https://kind.sigs.k8s.io/) for development scopes. Please refer to the [Kamaji documentation](../README.md) for understanding all the terms used in this guide, as for example: `admin cluster` and `tenant control plane`.
 
-- (optional) a bootstrap node;
-- a multi-tenant `etcd` cluster made of 3 nodes hosting the datastore for the `Tenant`s' clusters
-- a Kubernetes cluster, running the admin and Tenant Control Planes
-- an arbitrary number of machines hosting `Tenant`s' workloads
+## Pre-requisites
 
-> In this guide, we assume all machines are running `Ubuntu 20.04`.
+We assume you have installed on your workstation:
 
-  * [Prepare the bootstrap workspace](#prepare-the-bootstrap-workspace)
-  * [Access Admin cluster](#access-admin-cluster)
-  * [Setup external multi-tenant etcd](#setup-external-multi-tenant-etcd)
-  * [Setup internal multi-tenant etcd](#setup-internal-multi-tenant-etcd)
-  * [Install Kamaji controller](#install-kamaji-controller)
-  * [Setup Tenant cluster](#setup-tenant-cluster)
+- [Docker](https://docs.docker.com/engine/install/)
+- [KinD](https://kind.sigs.k8s.io/)
+- [kubectl](https://kubernetes.io/docs/tasks/tools/)
+- [kubeadm](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/install-kubeadm/)
+- [jq](https://stedolan.github.io/jq/)
+- [openssl](https://www.openssl.org/)
+- [cfssl](https://github.com/cloudflare/cfssl)
+- [cfssljson](https://github.com/cloudflare/cfssl)
 
-## Prepare the bootstrap workspace
-This getting started guide is supposed to be run from a remote or local bootstrap machine.
-First, prepare the workspace directory:
+## Setup Kamaji on KinD
 
-```
-git clone https://github.com/clastix/kamaji
-cd kamaji/deploy
-```
+The instance of Kamaji is made of a single node hosting:
 
-Throughout the instructions, shell variables are used to indicate values that you should adjust to your own environment.
+- admin control-plane
+- admin worker
+- multi-tenant etcd cluster
 
-### Install required tools
-On the bootstrap machine, install all the required tools to work with a Kamaji setup.
+### Standard
 
-#### cfssl and cfssljson
-The `cfssl` and `cfssljson` command line utilities will be used in addition to `kubeadm` to provision the PKI Infrastructure and generate TLS certificates.
-
-```
-wget -q --show-progress --https-only --timestamping \
-https://storage.googleapis.com/kubernetes-the-hard-way/cfssl/1.4.1/linux/cfssl \
-https://storage.googleapis.com/kubernetes-the-hard-way/cfssl/1.4.1/linux/cfssljson
-
-chmod +x cfssl cfssljson
-sudo mv cfssl cfssljson /usr/local/bin/
-```
-
-#### Kubernetes tools
-Install `kubeadm` and `kubectl`
+You can install your KinD cluster, ETCD multi-tenant cluster and Kamaji operator with a **single command**:
 
 ```bash
-sudo apt update && sudo apt install -y apt-transport-https ca-certificates curl && \
-sudo curl -fsSLo /usr/share/keyrings/kubernetes-archive-keyring.gpg https://packages.cloud.google.com/apt/doc/apt-key.gpg && \
-echo "deb [signed-by=/usr/share/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" | sudo tee /etc/apt/sources.list.d/kubernetes.list && \
-sudo apt update && sudo apt install -y kubeadm kubectl --allow-change-held-packages && \
-sudo apt-mark hold kubeadm kubectl
+$ make -C deploy/kind
 ```
 
-#### etcdctl
-For administration of the `etcd` cluster, download and install the `etcdctl` CLI utility on the bootstrap machine
+Now you can [create your first `TenantControlPlane`](#deploy-tenant-control-plane).
+
+### Data store-specific
+
+#### ETCD
+
+The multi-tenant etcd cluster is deployed as statefulset into the Kamaji node.
+
+Run `make reqs` to setup Kamaji's requisites on KinD:
 
 ```bash
-ETCD_VER=v3.5.1
-ETCD_URL=https://storage.googleapis.com/etcd
-curl -L ${ETCD_URL}/${ETCD_VER}/etcd-${ETCD_VER}-linux-amd64.tar.gz -o etcd-${ETCD_VER}-linux-amd64.tar.gz
-tar xzvf etcd-${ETCD_VER}-linux-amd64.tar.gz etcd-${ETCD_VER}-linux-amd64/etcdctl
-sudo cp etcd-${ETCD_VER}-linux-amd64/etcdctl /usr/bin/etcdctl
-rm -rf etcd-${ETCD_VER}-linux-amd64*
+$ make -C deploy/kind reqs
 ```
 
-Verify `etcdctl` version is installed
+At this moment you will have your KinD up and running and ETCD cluster in multitenant mode.
+
+Now you're ready to [install Kamaji operator](#install-kamaji).
+
+#### Kine
+
+> The MySQL-compatible cluster provisioning is omitted here.
+
+Kamaji offers the possibility of using a different storage system than `ETCD` for the tenants, like MySQL or PostgreSQL compatible databases.
+
+Read it more in the provided [guide](../deploy/kine/README.md).
+
+
+
+Assuming you adjusted the [Kamaji manifest](./config/install.yaml) to connect to Kine and compatible database using the proper driver, you can now install it.
+
+### Install Kamaji
 
 ```bash
-etcdctl version
-etcdctl version: 3.5.1
-API version: 3.5
+$ kubectl apply -f config/install.yaml
 ```
 
+### Deploy Tenant Control Plane
 
-## Access Admin cluster
-In Kamaji, an Admin Cluster is a regular Kubernetes cluster which hosts zero to many Tenant Cluster Control Planes running as pods. The admin cluster acts as management cluster for all the Tenant clusters and implements Monitoring, Logging, and Governance of all the Kamaji setup, including all Tenant clusters. 
-
-Any regular and conformant Kubernetes v1.22+ cluster can be turned into a Kamaji setup. Currently we tested:
-
-- [Kubernetes installed with `kubeadm`](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/).
-- [Azure AKS managed service](./kamaji-on-azure.md).
-- [KinD for local development](./kind/README.md).
-
-The admin cluster should provide:
-
-- CNI module installed, eg. Calico
-- Support for LoadBalancer Service Type, eg. MetalLB
-- Ingress Controller
-- CSI module installed with StorageClass for multi-tenant `etcd`
-- Monitoring Stack, eg. Prometheus and Grafana
-
-Make sure you have a `kubeconfig` file with admin permissions on the cluster you want to turn into a Kamaji Admin Cluster.
-
-## Setup external multi-tenant etcd
-In this section, we're going to setup a multi-tenant `etcd` cluster on dedicated nodes. Alternatively, if you want to use an internal `etcd` cluster as Kubernetes StatefulSet, jump [here](#setup-internal-multi-tenant-etcd).
-
-### Ensure host access
-From the bootstrap machine load the environment for external `etcd` setup:
+Now it is the moment of deploying your first tenant control plane.
 
 ```bash
-source kamaji-external-etcd.env
-```
-
-The installer requires a user that has access to all hosts. In order to run the installer as a non-root user, first configure passwordless sudo rights each host:
-
-Generate an SSH key on the host you run the installer on:
-
-```bash
-ssh-keygen -t rsa
-```
-
-> Do not use a password.
-
-Distribute the key to the other cluster hosts.
-
-Depending on your environment, use a bash loop:
-
-```bash
-HOSTS=(${ETCD0} ${ETCD1} ${ETCD2})
-for i in "${!HOSTS[@]}"; do
-  HOST=${HOSTS[$i]}
-  ssh-copy-id -i ~/.ssh/id_rsa.pub $HOST;
-done
-```
-
-> Alternatively, inject the generated public key into machines metadata.
-
-Confirm that you can access each host from bootstrap machine:
-
-```bash
-HOSTS=(${ETCD0} ${ETCD1} ${ETCD2})
-for i in "${!HOSTS[@]}"; do
-  HOST=${HOSTS[$i]}
-  ssh ${USER}@${HOST} -t 'hostname';
-done
-```
-
-### Configure disk layout
-As per `etcd` [requirements](https://etcd.io/docs/v3.5/op-guide/hardware/#disks), back `etcd`’s storage with a SSD. A SSD usually provides lower write latencies and with less variance than a spinning disk, thus improving the stability and reliability of `etcd`.
-
-For each `etcd` machine, we assume an additional `sdb` disk of 10GB:
-
-```
-clastix@kamaji-etcd-00:~$ lsblk
-NAME    MAJ:MIN RM  SIZE RO TYPE MOUNTPOINT
-sda       8:0    0   16G  0 disk 
-├─sda1    8:1    0 15.9G  0 part /
-├─sda14   8:14   0    4M  0 part 
-└─sda15   8:15   0  106M  0 part /boot/efi
-sdb       8:16   0   10G  0 disk 
-sr0      11:0    1    4M  0 rom  
-```
-
-Create partition, format, and mount the `etcd` disk, by running the script below from the bootstrap machine:
-
-> If you already used the `etcd` disks, please make sure to wipe the partitions with `sudo wipefs --all --force /dev/sdb` before to attempt to recreate them.
-
-```bash
-for i in "${!ETCDHOSTS[@]}"; do
-  HOST=${ETCDHOSTS[$i]}
-  ssh ${USER}@${HOST} -t 'echo type=83 | sudo sfdisk -f -q /dev/sdb'
-  ssh ${USER}@${HOST} -t 'sudo mkfs -F -q -t ext4 /dev/sdb1'
-  ssh ${USER}@${HOST} -t 'sudo mkdir -p /var/lib/etcd'
-  ssh ${USER}@${HOST} -t 'sudo e2label /dev/sdb1 ETCD'
-  ssh ${USER}@${HOST} -t 'echo LABEL=ETCD /var/lib/etcd ext4 defaults 0 1 | sudo tee -a /etc/fstab'
-  ssh ${USER}@${HOST} -t 'sudo mount -a'
-  ssh ${USER}@${HOST} -t 'sudo lsblk -f'
-done
-```
-
-### Install prerequisites
-Use bash script `nodes-prerequisites.sh` to install all the dependencies on all the cluster nodes:
-
-- Install `containerd` as container runtime
-- Install `crictl`, the command line for working with `containerd`
-- Install `kubectl`, `kubelet`, and `kubeadm` in the desired version, eg. `v1.24.0`
-
-Run the installation script:
-
-```bash
-VERSION=v1.24.0
-./nodes-prerequisites.sh ${VERSION:1} ${HOSTS[@]}
-```
-
-### Configure kubelet
-
-On each `etcd` node, configure the `kubelet` service to start `etcd` static pods using `containerd` as container runtime, by running the script below from the bootstrap machine:
-
-```bash
-cat << EOF > 20-etcd-service-manager.conf
-[Service]
-ExecStart=
-ExecStart=/usr/bin/kubelet --address=127.0.0.1 --pod-manifest-path=/etc/kubernetes/manifests --cgroup-driver=systemd --container-runtime=remote --container-runtime-endpoint=/run/containerd/containerd.sock
-Restart=always
-EOF
-```
-
-```
-for i in "${!ETCDHOSTS[@]}"; do
-  HOST=${ETCDHOSTS[$i]}
-  scp 20-etcd-service-manager.conf ${USER}@${HOST}:
-  ssh ${USER}@${HOST} -t 'sudo chown -R root:root 20-etcd-service-manager.conf && sudo mv 20-etcd-service-manager.conf /etc/systemd/system/kubelet.service.d/20-etcd-service-manager.conf'
-  ssh ${USER}@${HOST} -t 'sudo systemctl daemon-reload'
-  ssh ${USER}@${HOST} -t 'sudo systemctl start kubelet'
-  ssh ${USER}@${HOST} -t 'sudo systemctl enable kubelet'
-done
-
-rm -f 20-etcd-service-manager.conf
-```
-
-### Create configuration
-Create temp directories to store files that will end up on `etcd` hosts:
-
-```bash
-mkdir -p /tmp/${ETCD0}/ /tmp/${ETCD1}/ /tmp/${ETCD2}/
-NAMES=("etcd00" "etcd01" "etcd02")
-
-for i in "${!ETCDHOSTS[@]}"; do
-HOST=${ETCDHOSTS[$i]}
-NAME=${NAMES[$i]}
-
-cat <<EOF | sudo tee /tmp/${HOST}/kubeadmcfg.yaml
-apiVersion: "kubeadm.k8s.io/v1beta2"
-kind: ClusterConfiguration
-etcd:
-  local:
-    serverCertSANs:
-    - "${HOST}"
-    peerCertSANs:
-    - "${HOST}"
-    extraArgs:
-      initial-cluster: ${NAMES[0]}=https://${ETCDHOSTS[0]}:2380,${NAMES[1]}=https://${ETCDHOSTS[1]}:2380,${NAMES[2]}=https://${ETCDHOSTS[2]}:2380
-      initial-cluster-state: new
-      name: ${NAME}
-      listen-peer-urls: https://${HOST}:2380
-      listen-client-urls: https://${HOST}:2379
-      advertise-client-urls: https://${HOST}:2379
-      initial-advertise-peer-urls: https://${HOST}:2380
-      auto-compaction-mode: periodic
-      auto-compaction-retention: 5m
-      quota-backend-bytes: '8589934592'
-EOF
-done
-```
-> Note:
->
-> ##### Etcd compaction
->
-> By enabling `etcd` authentication, it prevents the tenant apiservers (clients of `etcd`) to issue compaction requests. We set `etcd` to automatically compact the keyspace with the `--auto-compaction-*` option with a period of hours or minutes. When `--auto-compaction-mode=periodic` and `--auto-compaction-retention=5m` and writes per minute are about 1000, `etcd` compacts revision 5000 for every 5 minute. 
-> 
-> ##### Etcd storage quota
->
-> Currently, `etcd` is limited in storage size, defaulted to `2GB` and configurable with `--quota-backend-bytes` flag up to `8GB`. In Kamaji, we use a single `etcd` to store multiple tenant clusters, so we need to increase this size. Please, note `etcd` warns at startup if the configured value exceeds `8GB`.
-
-### Generate certificates 
-On the bootstrap machine, using `kubeadm` init phase, create and distribute `etcd` CA certificates:
-
-```bash
-sudo kubeadm init phase certs etcd-ca
-mkdir kamaji
-sudo cp -r /etc/kubernetes/pki/etcd kamaji
-sudo chown -R ${USER}. kamaji/etcd
-```
-
-For each `etcd` host: 
-
-```bash
-for i in "${!ETCDHOSTS[@]}"; do
-  HOST=${ETCDHOSTS[$i]}
-  sudo kubeadm init phase certs etcd-server --config=/tmp/${HOST}/kubeadmcfg.yaml
-  sudo kubeadm init phase certs etcd-peer --config=/tmp/${HOST}/kubeadmcfg.yaml
-  sudo kubeadm init phase certs etcd-healthcheck-client --config=/tmp/${HOST}/kubeadmcfg.yaml
-  sudo cp -R /etc/kubernetes/pki /tmp/${HOST}/
-  sudo find /etc/kubernetes/pki -not -name ca.crt -not -name ca.key -type f -delete
-done
-```
-
-### Startup the cluster
-Upload certificates on each `etcd` node and restart the `kubelet`
-
-```bash
-for i in "${!ETCDHOSTS[@]}"; do
-  HOST=${ETCDHOSTS[$i]}
-  sudo chown -R ${USER}. /tmp/${HOST}
-  scp -r /tmp/${HOST}/* ${USER}@${HOST}:
-  ssh ${USER}@${HOST} -t 'sudo chown -R root:root pki'
-  ssh ${USER}@${HOST} -t 'sudo mv pki /etc/kubernetes/'
-  ssh ${USER}@${HOST} -t 'sudo kubeadm init phase etcd local --config=kubeadmcfg.yaml'
-  ssh ${USER}@${HOST} -t 'sudo systemctl daemon-reload'
-  ssh ${USER}@${HOST} -t 'sudo systemctl restart kubelet'
-done
-```
-
-This will start the static `etcd` pod on each node and then the cluster gets formed.
-
-Generate certificates for the `root` user
-
-```bash
-cat > root-csr.json <<EOF  
-{
-  "CN": "root",
-  "key": {
-    "algo": "rsa",
-    "size": 2048
-  }
-}
-EOF
-```
-
-```bash
-cfssl gencert \
-  -ca=kamaji/etcd/ca.crt \
-  -ca-key=kamaji/etcd/ca.key \
-  -config=cfssl-cert-config.json \
-  -profile=client-authentication \
-  root-csr.json | cfssljson -bare root
-```
-
-```bash
-cp root.pem kamaji/etcd/root.crt
-cp root-key.pem kamaji/etcd/root.key
-rm root*
-```
-
-The result should be:
-
-```bash
-$ tree kamaji
-kamaji
-└── etcd
-    ├── ca.crt
-    ├── ca.key
-    ├── root.crt
-    └── root.key
-```
-
-Use the `root` user to check the just formed `etcd` cluster is in health state 
-
-```bash
-export ETCDCTL_CACERT=kamaji/etcd/ca.crt
-export ETCDCTL_CERT=kamaji/etcd/root.crt
-export ETCDCTL_KEY=kamaji/etcd/root.key
-export ETCDCTL_ENDPOINTS=https://${ETCD0}:2379
-
-etcdctl member list -w table
-```
-
-The result should be something like this:
-
-```
-+------------------+---------+--------+----------------------------+----------------------------+------------+
-|        ID        | STATUS  |  NAME  |         PEER ADDRS         |        CLIENT ADDRS        | IS LEARNER |
-+------------------+---------+--------+----------------------------+----------------------------+------------+
-| 72657d6307364226 | started | etcd01 | https://192.168.32.11:2380 | https://192.168.32.11:2379 |      false |
-| 91eb892c5ee87610 | started | etcd00 | https://192.168.32.10:2380 | https://192.168.32.10:2379 |      false |
-| e9971c576949c34e | started | etcd02 | https://192.168.32.12:2380 | https://192.168.32.12:2379 |      false |
-+------------------+---------+--------+----------------------------+----------------------------+------------+
-```
-
-### Enable multi-tenancy
-The `root` user has full access to `etcd`, must be created before activating authentication. The `root` user must have the `root` role and is allowed to change anything inside `etcd`.
-
-```bash
-etcdctl user add --no-password=true root
-etcdctl role add root
-etcdctl user grant-role root root
-etcdctl auth enable
-```
-
-### Cleanup
-If you want to get rid of the etcd cluster, for each node, login and clean it:
-
-```bash
-HOSTS=(${ETCD0} ${ETCD1} ${ETCD2})
-for i in "${!HOSTS[@]}"; do
-  HOST=${HOSTS[$i]}
-  ssh ${USER}@${HOST} -t 'sudo kubeadm reset -f';
-  ssh ${USER}@${HOST} -t 'sudo systemctl reboot';
-done
-```
-
-## Setup internal multi-tenant etcd
-If you opted for an internal etcd cluster running in the Kamaji admin cluster, follow steps below.
-
-From the bootstrap machine load the environment for internal `etcd` setup:
-
-```bash
-source kamaji-internal-etcd.env
-```
-
-### Generate certificates 
-On the bootstrap machine, using `kubeadm` init phase, create the `etcd` CA certificates:
-
-```bash
-sudo kubeadm init phase certs etcd-ca
-mkdir kamaji
-sudo cp -r /etc/kubernetes/pki/etcd kamaji
-sudo chown -R ${USER}. kamaji/etcd
-```
-
-Generate the `etcd` certificates for peers:
-
-```
-cat << EOF | tee kamaji/etcd/peer-csr.json
-{
-  "CN": "etcd",
-  "key": {
-    "algo": "rsa",
-    "size": 2048
-  },
-  "hosts": [
-    "127.0.0.1",
-    "etcd-0",
-    "etcd-0.etcd",
-    "etcd-0.etcd.${ETCD_NAMESPACE}.svc",
-    "etcd-0.etcd.${ETCD_NAMESPACE}.svc.cluster.local",
-    "etcd-1",
-    "etcd-1.etcd",
-    "etcd-1.etcd.${ETCD_NAMESPACE}.svc",
-    "etcd-1.etcd.${ETCD_NAMESPACE}.svc.cluster.local",
-    "etcd-2",
-    "etcd-2.etcd",
-    "etcd-2.etcd.${ETCD_NAMESPACE}.svc",
-    "etcd-2.etcd.${ETCD_NAMESPACE}.cluster.local"
-  ]
-}
-EOF
-
-cfssl gencert -ca=kamaji/etcd/ca.crt -ca-key=kamaji/etcd/ca.key \
-  -config=cfssl-cert-config.json \
-  -profile=peer-authentication kamaji/etcd/peer-csr.json | cfssljson -bare kamaji/etcd/peer
-
-```
-
-Generate the `etcd` certificates for server:
-
-```
-cat << EOF | tee kamaji/etcd/server-csr.json
-{
-  "CN": "etcd",
-  "key": {
-    "algo": "rsa",
-    "size": 2048
-  },
-  "hosts": [
-    "127.0.0.1",
-    "etcd-server",
-    "etcd-server.${ETCD_NAMESPACE}.svc",
-    "etcd-server.${ETCD_NAMESPACE}.svc.cluster.local",
-    "etcd-0.etcd.${ETCD_NAMESPACE}.svc.cluster.local",
-    "etcd-1.etcd.${ETCD_NAMESPACE}.svc.cluster.local",
-    "etcd-2.etcd.${ETCD_NAMESPACE}.svc.cluster.local"
-  ]
-}
-EOF
-
-cfssl gencert -ca=kamaji/etcd/ca.crt -ca-key=kamaji/etcd/ca.key \
-  -config=cfssl-cert-config.json \
-  -profile=peer-authentication kamaji/etcd/server-csr.json | cfssljson -bare kamaji/etcd/server
-```
-
-Generate certificates for the `root` user of the `etcd`
-
-```
-cat << EOF | tee kamaji/etcd/root-csr.json
-{
-  "CN": "root",
-  "key": {
-    "algo": "rsa",
-    "size": 2048
-  }
-}
-EOF
-
-cfssl gencert -ca=kamaji/etcd/ca.crt -ca-key=kamaji/etcd/ca.key \
-  -config=cfssl-cert-config.json \
-  -profile=client-authentication kamaji/etcd/root-csr.json | cfssljson -bare kamaji/etcd/root	
-```
-
-Install the `etcd` in the Kamaji admin cluster
-
-```bash
-kubectl create namespace ${ETCD_NAMESPACE}
-
-kubectl -n ${ETCD_NAMESPACE} create secret generic etcd-certs \
-  --from-file=kamaji/etcd/ca.crt \
-  --from-file=kamaji/etcd/ca.key \
-  --from-file=kamaji/etcd/peer-key.pem --from-file=kamaji/etcd/peer.pem \
-  --from-file=kamaji/etcd/server-key.pem --from-file=kamaji/etcd/server.pem
-
-kubectl -n ${ETCD_NAMESPACE} apply -f etcd/etcd-cluster.yaml
-```
-
-Install an `etcd` client to interact with the `etcd` server
-
-```bash
-kubectl -n ${ETCD_NAMESPACE} create secret tls root-certs \
-  --key=kamaji/etcd/root-key.pem \
-  --cert=kamaji/etcd/root.pem
-
-kubectl -n ${ETCD_NAMESPACE} apply -f etcd/etcd-client.yaml
-```
-
-Wait the etcd instances discover each other and the cluster is formed:
-
-```bash
-kubectl -n ${ETCD_NAMESPACE} wait pod --for=condition=ready -l app=etcd --timeout=120s
-echo -n "\nChecking endpoint's health..."
-
-kubectl -n ${ETCD_NAMESPACE} exec etcd-root-client -- /bin/bash -c "etcdctl endpoint health 1>/dev/null 2>/dev/null; until [ \$$? -eq 0 ]; do sleep 10; printf "."; etcdctl endpoint health 1>/dev/null 2>/dev/null; done;"
-echo -n "\netcd cluster's health:\n"
-
-kubectl -n ${ETCD_NAMESPACE} exec etcd-root-client -- /bin/bash -c "etcdctl endpoint health"
-echo -n "\nWaiting for all members..."
-
-kubectl -n ${ETCD_NAMESPACE} exec etcd-root-client -- /bin/bash -c "until [ \$$(etcdctl member list 2>/dev/null | wc -l) -eq 3 ]; do sleep 10; printf '.'; done;"
-	@echo -n "\netcd's members:\n"
-
-kubectl -n ${ETCD_NAMESPACE} exec etcd-root-client -- /bin/bash -c "etcdctl member list -w table"
-```
-
-### Enable multi-tenancy
-The `root` user has full access to `etcd`, must be created before activating authentication. The `root` user must have the `root` role and is allowed to change anything inside `etcd`.
-
-```bash
-kubectl -n ${ETCD_NAMESPACE} exec etcd-root-client -- etcdctl user add --no-password=true root
-kubectl -n ${ETCD_NAMESPACE} exec etcd-root-client -- etcdctl role add root
-kubectl -n ${ETCD_NAMESPACE} exec etcd-root-client -- etcdctl user grant-role root root
-kubectl -n ${ETCD_NAMESPACE} exec etcd-root-client -- etcdctl auth enable
-```
-
-
-## Install Kamaji controller
-Currently, the behaviour of the Kamaji controller for Tenant Control Plane is controlled by (in this order):
-
-- CLI flags
-- Environment variables
-- Configuration file `kamaji.yaml` built into the image
-
-By default Kamaji search for the configuration file and uses parameters found inside of it. In case some environment variable are passed, this will override configuration file parameters. In the end, if also a CLI flag is passed, this will override both env vars and config file as well.
-
-There are multiple ways to deploy the Kamaji controller:
-
-- Use the single YAML file installer
-- Use Kustomize with Makefile
-- Use the Kamaji Helm Chart
-
-The Kamaji controller needs to access the multi-tenant `etcd` in order to provision the access for tenant `kube-apiserver`.
-
-Create the secrets containing the `etcd` certificates
-
-```bash
-kubectl create namespace kamaji-system
-kubectl -n kamaji-system create secret generic etcd-certs \
-  --from-file=kamaji/etcd/ca.crt \
-  --from-file=kamaji/etcd/ca.key
-
-kubectl -n kamaji-system create secret tls root-client-certs \
-  --cert=kamaji/etcd/root.crt \
-  --key=kamaji/etcd/root.key
-```
-
-### Install with a single manifest
-Install with the single YAML file installer:
-
-```bash
-kubectl -n kamaji-system apply -f ../config/install.yaml
-```
-
-Make sure to patch the `etcd` endpoints of the Kamaji controller, according to your environment:
-
-```bash
-cat > patch-deploy.yaml <<EOF 
+$ kubectl apply -f - <<EOF
+apiVersion: kamaji.clastix.io/v1alpha1
+kind: TenantControlPlane
+metadata:
+  name: tenant1
 spec:
-  template:
-    spec:
-      containers:
-      - name: manager
-        args:
-        - --health-probe-bind-address=:8081
-        - --metrics-bind-address=127.0.0.1:8080
-        - --leader-elect
-        - --etcd-endpoints=${ETCD0}:2379,${ETCD1}:2379,${ETCD2}:2379
+  controlPlane:
+    deployment:
+      replicas: 2
+      additionalMetadata:
+        annotations:
+          environment.clastix.io: tenant1
+          tier.clastix.io: "0"
+        labels:
+          tenant.clastix.io: tenant1
+          kind.clastix.io: deployment
+    service:
+      additionalMetadata:
+        annotations:
+          environment.clastix.io: tenant1
+          tier.clastix.io: "0"
+        labels:
+          tenant.clastix.io: tenant1
+          kind.clastix.io: service
+      serviceType: NodePort
+  kubernetes:
+    version: "v1.23.4"
+    kubelet:
+      cgroupfs: cgroupfs
+    admissionControllers:
+    - LimitRanger
+    - ResourceQuota
+  networkProfile:
+    address: "172.18.0.2"
+    port: 31443
+    certSANs:
+    - "test.clastixlabs.io"
+    serviceCidr: "10.96.0.0/16"
+    podCidr: "10.244.0.0/16"
+    dnsServiceIPs: 
+    - "10.96.0.10"
+  addons:
+    coreDNS: {}
+    kubeProxy: {}
 EOF
-
-kubectl -n kamaji-system patch \
-  deployment kamaji-controller-manager \
-  --patch-file patch-deploy.yaml
 ```
 
-The Kamaji Tenant Control Plane controller is now running on the Admin Cluster:
+> Check networkProfile fields according to your installation
+> To let Kamaji works in kind, you have indicate that the service must be [NodePort](https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport)
+
+### Get Kubeconfig
+
+Let's retrieve kubeconfig and store in `/tmp/kubeconfig`
 
 ```bash
-kubectl -n kamaji-system get deploy
-NAME                          READY   UP-TO-DATE   AVAILABLE   AGE
-operator-controller-manager   1/1     1            1           14h
+$ kubectl get secrets tenant1-admin-kubeconfig -o json \
+ | jq -r '.data["admin.conf"]' \
+ | base64 -d > /tmp/kubeconfig
+ ```
+
+It can be export it, to facilitate the next tasks:
+
+```bash
+$ export KUBECONFIG=/tmp/kubeconfig
 ```
 
-## Setup Tenant Cluster
-Now you are getting an Admin Cluster available to run multiple Tenant Control Planes, deployed by the Kamaji controller. Please, refer to the Kamaji Tenant Deployment [guide](./kamaji-tenant-deployment-guide.md).
+### Install CNI
 
+We highly recommend to install [kindnet](https://github.com/aojea/kindnet) as CNI for your kamaji TCP.
+
+```bash
+$ kubectl create -f https://raw.githubusercontent.com/aojea/kindnet/master/install-kindnet.yaml
+```
+
+### Join worker nodes
+
+```bash
+$ make kamaji-kind-worker-join
+```
+
+> To add more worker nodes, run again the command above.
+
+Check out the node:
+
+```bash
+$ kubectl get nodes
+NAME           STATUS   ROLES    AGE   VERSION
+d2d4b468c9de   Ready    <none>   44s   v1.23.4
+```
+
+> For more complex scenarios (exposing port, different version and so on), run `join-node.bash`
+
+Tenant control plane provision has been finished in a minimal Kamaji setup based on KinD. Therefore, you could develop, test and make your own experiments with Kamaji.
+
+## Cleanup
+
+```bash
+$ make destroy
+```

docs/kamaji-azure-deployment-guide.md

@@ -1,42 +1,48 @@
 # Setup Kamaji on Azure
-
-In this section, we're going to setup Kamaji on MS Azure:
+This guide will lead you through the process of creating a working Kamaji setup on on MS Azure. It requires:
 
 - one bootstrap local workstation
-- a regular AKS cluster as Kamaji Admin Cluster
-- a multi-tenant etcd internal cluster running on AKS
-- an arbitrary number of Azure virtual machines hosting `Tenant`s' workloads
+- an AKS Kubernetes cluster to run the Admin and Tenant Control Planes
+- an arbitrary number of Azure virtual machines to host `Tenant`s' workloads
 
-## Bootstrap machine
-This getting started guide is supposed to be run from a remote or local bootstrap machine.
-First, prepare the workspace directory:
+  * [Prepare the bootstrap workspace](#prepare-the-bootstrap-workspace)
+  * [Access Admin cluster](#access-admin-cluster)
+  * [Install Kamaji controller](#install-kamaji-controller)
+  * [Create Tenant Cluster](#create-tenant-cluster)
+  * [Cleanup](#cleanup)
 
-```
+## Prepare the bootstrap workspace
+This guide is supposed to be run from a remote or local bootstrap machine. First, clone the repo and prepare the workspace directory:
+
+```bash
 git clone https://github.com/clastix/kamaji
 cd kamaji/deploy
 ```
 
-1. Follow the instructions in [Prepare the bootstrap workspace](./getting-started-with-kamaji.md#prepare-the-bootstrap-workspace).
-2. Install the [Azure CLI](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli).
-3. Make sure you have a valid Azure subscription
-4. Login to Azure:
+We assume you have installed on your workstation:
+
+- [kubectl](https://kubernetes.io/docs/tasks/tools/)
+- [helm](https://helm.sh/docs/intro/install/)
+- [jq](https://stedolan.github.io/jq/)
+- [openssl](https://www.openssl.org/)
+- [Azure CLI](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli)
+
+Make sure you have a valid Azure subscription, and login to Azure:
 
 ```bash
 az account set --subscription "MySubscription"
 az login
 ```
-> Currently, the Kamaji setup, including Admin and Tenant clusters need to be deployed within the same Azure region. Cross-regions deployments are not (yet) validated.
+> Currently, the Kamaji setup, including Admin and Tenant clusters need to be deployed within the same Azure region. Cross-regions deployments are not supported.
 
-## Setup Admin cluster on AKS
-Throughout the instructions, shell variables are used to indicate values that you should adjust to your own Azure environment:
+## Access Admin cluster
+In Kamaji, an Admin Cluster is a regular Kubernetes cluster which hosts zero to many Tenant Cluster Control Planes. The admin cluster acts as management cluster for all the Tenant clusters and implements Monitoring, Logging, and Governance of all the Kamaji setup, including all Tenant clusters. For this guide, we're going to use an instance of Azure Kubernetes Service - AKS as the Admin Cluster.
+
+Throughout the following instructions, shell variables are used to indicate values that you should adjust to your own Azure environment:
 
 ```bash
 source kamaji-azure.env
-```
 
-> we use the Azure CLI to setup the Kamaji Admin cluster on AKS.
-
-```
 az group create \
   --name $KAMAJI_RG \
   --location $KAMAJI_REGION
@@ -66,45 +72,50 @@ And check you can access:
 kubectl cluster-info
 ```
 
-## Setup internal multi-tenant etcd
-Follow the instructions [here](./getting-started-with-kamaji.md#setup-internal-multi-tenant-etcd).
+## Install Kamaji
+There are multiple ways to deploy Kamaji, including a [single YAML file](../config/install.yaml) and [Helm Chart](../charts/kamaji).
 
-## Install Kamaji controller
-Follow the instructions [here](./getting-started-with-kamaji.md#install-kamaji-controller).
+### Multi-tenant datastore
+The Kamaji controller needs to access a multi-tenant datastore in order to save data of the tenants' clusters.
+Install a multi-tenant `etcd` in the admin cluster as three replicas StatefulSet with data persistence.
+The Helm [Chart](../charts/kamaji/) provides the installation of an internal `etcd`.
+However, an externally managed `etcd` is highly recommended.
+If you'd like to use an external one, you can specify the overrides by setting the value `etcd.deploy=false`.
 
-## Create Tenant Clusters
-To create a Tenant Cluster in Kamaji on AKS, we have to work on both the Kamaji and Azure infrastructure sides.
-
-```
-source kamaji-tenant-azure.env
-```
-
-### On Kamaji side
-With Kamaji on AKS, the tenant control plane is accessible:
-
-- from tenant work nodes through an internal loadbalancer as `https://${TENANT_ADDR}:${TENANT_PORT}`
-- from tenant admin user through an external loadbalancer `https://${TENANT_NAME}.${KAMAJI_REGION}.cloudapp.azure.com:443`
-
-#### Allocate an internal IP address for the Tenant Control Plane
-Currently, Kamaji has a known limitation, meaning the address `${TENANT_ADDR}:${TENANT_PORT}` must be known in advance before to create the Tenant Control Plane. Given this limitation, let's to reserve an IP address and port in the same virtual subnet used by the Kamaji admin cluster:
+Optionally, Kamaji offers the possibility of using a different storage system than `etcd` for the tenants' clusters, like MySQL or PostgreSQL compatible database, thanks to the [kine](https://github.com/k3s-io/kine) integration documented [here](../deploy/kine/README.md).
 
+### Install with Helm Chart
+Install with the `helm` in a dedicated namespace of the Admin cluster:
 
 ```bash
-export TENANT_ADDR=10.240.0.100
-export TENANT_PORT=6443
-export TENANT_DOMAIN=$KAMAJI_REGION.cloudapp.azure.com
+helm install --create-namespace --namespace kamaji-system kamaji clastix/kamaji
 ```
 
-> Make sure the `TENANT_ADDR` value does not overlap with already allocated IP addresses in the AKS virtual network. In the future, Kamaji will implement a dynamic IP allocation.
+The Kamaji controller and the multi-tenant `etcd` are now running:
 
+```bash
+kubectl -n kamaji-system get pods
+NAME                      READY   STATUS    RESTARTS       AGE
+etcd-0                    1/1     Running   0              120m
+etcd-1                    1/1     Running   0              120m
+etcd-2                    1/1     Running   0              119m
+kamaji-857fcdf599-4fb2p   2/2     Running   0              120m
+```
 
-#### Create the Tenant Control Plane
+You just turned your AKS cluster into a Kamaji cluster to run multiple Tenant Control Planes.
 
-Create the manifest for Tenant Control Plane:
+## Create Tenant Cluster
+
+### Tenant Control Plane
+With Kamaji on AKS, the tenant control plane is accessible:
+
+- from tenant worker nodes through an internal loadbalancer
+- from tenant admin user through an external loadbalancer responding to `https://${TENANT_NAME}.${TENANT_NAME}.${TENANT_DOMAIN}:443`
+
+Create a tenant control plane of example:
 
 ```yaml
 cat > ${TENANT_NAMESPACE}-${TENANT_NAME}-tcp.yaml <<EOF
----
 apiVersion: kamaji.clastix.io/v1alpha1
 kind: TenantControlPlane
 metadata:
@@ -113,24 +124,37 @@ metadata:
 spec:
   controlPlane:
     deployment:
-      replicas: 2
+      replicas: 3
       additionalMetadata:
-        annotations:
-          environment.clastix.io: ${TENANT_NAME}
         labels:
           tenant.clastix.io: ${TENANT_NAME}
-          kind.clastix.io: deployment
+      extraArgs:
+        apiServer: []
+        controllerManager: []
+        scheduler: []
+      resources:
+        apiServer:
+          requests:
+            cpu: 500m
+            memory: 512Mi
+          limits: {}
+        controllerManager:
+          requests:
+            cpu: 250m
+            memory: 256Mi
+          limits: {}
+        scheduler:
+          requests:
+            cpu: 250m
+            memory: 256Mi
+          limits: {} 
     service:
       additionalMetadata:
-        annotations:
-          environment.clastix.io: ${TENANT_NAME}
-          service.beta.kubernetes.io/azure-load-balancer-internal: "true"
         labels:
           tenant.clastix.io: ${TENANT_NAME}
-          kind.clastix.io: service
+        annotations:
+          service.beta.kubernetes.io/azure-load-balancer-internal: "true"
       serviceType: LoadBalancer
-    ingress:
-      enabled: false    
   kubernetes:
     version: ${TENANT_VERSION}
     kubelet:
@@ -139,13 +163,23 @@ spec:
       - ResourceQuota
       - LimitRanger
   networkProfile:
-    address: ${TENANT_ADDR}
     port: ${TENANT_PORT}
-    domain: ${TENANT_DOMAIN}
+    certSANs:
+    - ${TENANT_NAME}.${TENANT_DOMAIN}
     serviceCidr: ${TENANT_SVC_CIDR}
     podCidr: ${TENANT_POD_CIDR}
     dnsServiceIPs:
     - ${TENANT_DNS_SERVICE}
+  addons:
+    coreDNS: {}
+    kubeProxy: {}
+    konnectivity:
+      proxyPort: ${TENANT_PROXY_PORT}
+      resources:
+        requests:
+          cpu: 100m
+          memory: 128Mi
+        limits: {}
 ---
 apiVersion: v1
 kind: Service
@@ -163,44 +197,24 @@ spec:
     kamaji.clastix.io/soot: ${TENANT_NAME}
   type: LoadBalancer
 EOF
-```
 
-Make sure:
-
-- the `tcp.spec.controlPlane.service.serviceType=LoadBalancer` and the following annotation: `service.beta.kubernetes.io/azure-load-balancer-internal=true` is set. This tells AKS to expose the service within an Azure internal loadbalancer.
-
-- the public loadbalancer service has the following annotation: `service.beta.kubernetes.io/azure-dns-label-name=${TENANT_NAME}` to expose the Tenant Control Plane with domain name: `${TENANT_NAME}.${KAMAJI_REGION}.cloudapp.azure.com`.
-
-Create the Tenant Control Plane
-
-```
 kubectl create namespace ${TENANT_NAMESPACE}
 kubectl apply -f ${TENANT_NAMESPACE}-${TENANT_NAME}-tcp.yaml
 ```
 
-And check it out:
+Make sure:
 
-```
-$ kubectl get tcp
-NAME        VERSION   CONTROL-PLANE-ENDPOINT   KUBECONFIG                   PRIVATE   AGE
-tenant-00   v1.23.4   10.240.0.100:6443        tenant-00-admin-kubeconfig   true      46m
+- the following annotation: `service.beta.kubernetes.io/azure-load-balancer-internal=true` is set on the `tcp` service. It tells Azure to expose the service within an internal loadbalancer.
 
-$ kubectl get svc
-NAME               TYPE           CLUSTER-IP     EXTERNAL-IP      PORT(S)          AGE
-tenant-00          LoadBalancer   10.0.223.161   10.240.0.100     6443:31902/TCP   46m
-tenant-00-public   LoadBalancer   10.0.205.97    20.101.215.149   443:30697/TCP    19h
+- the following annotation: `service.beta.kubernetes.io/azure-dns-label-name=${TENANT_NAME}` is set the public loadbalancer service. It tells Azure to expose the Tenant Control Plane with domain name: `${TENANT_NAME}.${TENANT_DOMAIN}`.
 
-$ kubectl get deploy
-NAME        READY   UP-TO-DATE   AVAILABLE   AGE
-tenant-00   2/2     2            2           47m
-```
+### Working with Tenant Control Plane
 
-
-#### Working with Tenant Control Plane
 Check the access to the Tenant Control Plane:
 
 ```
 curl -k https://${TENANT_NAME}.${KAMAJI_REGION}.cloudapp.azure.com/healthz
+curl -k https://${TENANT_NAME}.${KAMAJI_REGION}.cloudapp.azure.com/version
 ```
 
 Let's retrieve the `kubeconfig` in order to work with it:
@@ -225,7 +239,7 @@ NAMESPACE     NAME         TYPE        CLUSTER-IP   EXTERNAL-IP   PORT(S)     AG
 default       kubernetes   ClusterIP   10.32.0.1    <none>        443/TCP     6m
 ```
 
-Check out how the Tenant Control Plane advertises itself to workloads:
+Check out how the Tenant Control Plane advertises itself:
 
 ```
 kubectl --kubeconfig=${TENANT_NAMESPACE}-${TENANT_NAME}.kubeconfig get ep
@@ -234,8 +248,6 @@ NAME         ENDPOINTS           AGE
 kubernetes   10.240.0.100:6443   57m
 ```
 
-Make sure it's `${TENANT_ADDR}:${TENANT_PORT}`.
-
 ### Prepare the Infrastructure for the Tenant virtual machines
 Kamaji provides Control Plane as a Service, so the tenant user can join his own virtual machines as worker nodes. Each tenant can place his virtual machines in a dedicated Azure virtual network.
 
@@ -332,12 +344,11 @@ az vmss scale \
 The current approach for joining nodes is to use the `kubeadm` one therefore, we will create a bootstrap token to perform the action:
 
 ```bash
-JOIN_CMD=$(echo "sudo kubeadm join ${TENANT_ADDR}:${TENANT_PORT} ")$(kubeadm --kubeconfig=${TENANT_NAMESPACE}-${TENANT_NAME}.kubeconfig token create --print-join-command |cut -d" " -f4-)
+JOIN_CMD=$(echo "sudo kubeadm join ${TENANT_ADDR}:6443 ")$(kubeadm --kubeconfig=${TENANT_NAMESPACE}-${TENANT_NAME}.kubeconfig token create --print-join-command |cut -d" " -f4-)
 ```
 
 A bash loop will be used to join all the available nodes.
 
-
 ```bash
 HOSTS=($(az vmss list-instance-public-ips \
     --resource-group $TENANT_RG \
@@ -355,13 +366,12 @@ done
 Checking the nodes:
 
 ```bash
-kubectl get nodes --kubeconfig=${CLUSTER_NAMESPACE}-${CLUSTER_NAME}.kubeconfig
+kubectl --kubeconfig=${TENANT_NAMESPACE}-${TENANT_NAME}.kubeconfig get nodes
 
-NAME                      STATUS      ROLES    AGE    VERSION
-kamaji-tenant-worker-00   NotReady    <none>   1m     v1.23.4
-kamaji-tenant-worker-01   NotReady    <none>   1m     v1.23.4
-kamaji-tenant-worker-02   NotReady    <none>   1m     v1.23.4
-kamaji-tenant-worker-03   NotReady    <none>   1m     v1.23.4
+NAME               STATUS     ROLES    AGE    VERSION
+tenant-00-000000   NotReady   <none>   112s   v1.23.5
+tenant-00-000002   NotReady   <none>   92s    v1.23.5
+tenant-00-000003   NotReady   <none>   71s    v1.23.5
 ```
 
 The cluster needs a [CNI](https://kubernetes.io/docs/concepts/extend-kubernetes/compute-storage-net/network-plugins/) plugin to get the nodes ready. In our case, we are going to install [calico](https://projectcalico.docs.tigera.io/about/about-calico).
@@ -374,7 +384,7 @@ kubectl apply -f calico-cni/calico-azure.yaml --kubeconfig=${TENANT_NAMESPACE}-$
 And after a while, `kube-system` pods will be running.
 
 ```bash
-kubectl get po -n kube-system --kubeconfig=${TENANT_NAMESPACE}-${TENANT_NAME}.kubeconfig
+kubectl --kubeconfig=${TENANT_NAMESPACE}-${TENANT_NAME}.kubeconfig get po -n kube-system 
 
 NAME                                       READY   STATUS    RESTARTS   AGE
 calico-kube-controllers-8594699699-dlhbj   1/1     Running   0          3m
@@ -389,15 +399,25 @@ kube-proxy-m48v4                           1/1     Running   0          3m
 And the nodes will be ready
 
 ```bash
-kubectl get nodes --kubeconfig=${TENANT_NAMESPACE}-${TENANT_NAME}.kubeconfig
+kubectl --kubeconfig=${TENANT_NAMESPACE}-${TENANT_NAME}.kubeconfig get nodes 
 
-NAME                      STATUS      ROLES    AGE    VERSION
-kamaji-tenant-worker-01   Ready       <none>   10m    v1.23.4
-kamaji-tenant-worker-02   Ready       <none>   10m    v1.23.4
+NAME               STATUS   ROLES    AGE     VERSION
+tenant-00-000000   Ready    <none>   3m38s   v1.23.5
+tenant-00-000002   Ready    <none>   3m18s   v1.23.5
+tenant-00-000003   Ready    <none>   2m57s   v1.23.5
 ```
 
-
 ## Cleanup
-To get rid of the Tenant infrastructure, remove the RESOURCE_GROUP:  `az group delete --name $TENANT_RG --yes --no-wait`.
+To get rid of the Tenant infrastructure, remove the RESOURCE_GROUP:
 
-To get rid of the Kamaji infrastructure, remove the RESOURCE_GROUP:  `az group delete --name $KAMAJI_RG --yes --no-wait`.
+```
+az group delete --name $TENANT_RG --yes --no-wait
+```
+
+To get rid of the Kamaji infrastructure, remove the RESOURCE_GROUP:
+
+```
+az group delete --name $KAMAJI_RG --yes --no-wait
+```
+
+That's all folks!

docs/kamaji-deployment-guide.md

@@ -0,0 +1,477 @@
+# Setup Kamaji
+This guide will lead you through the process of creating a working Kamaji setup on a generic Kubernetes cluster. It requires:
+
+- one bootstrap local workstation
+- a Kubernetes cluster 1.22+, to run the Admin and Tenant Control Planes
+- an arbitrary number of machines to host Tenants' workloads
+
+> In this guide, we assume the machines are running `Ubuntu 20.04`.
+
+  * [Prepare the bootstrap workspace](#prepare-the-bootstrap-workspace)
+  * [Access Admin cluster](#access-admin-cluster)
+  * [Install Kamaji controller](#install-kamaji-controller)
+  * [Create Tenant Cluster](#create-tenant-cluster)
+  * [Cleanup](#cleanup)
+
+## Prepare the bootstrap workspace
+This guide is supposed to be run from a remote or local bootstrap machine. First, clone the repo and prepare the workspace directory:
+
+```bash
+git clone https://github.com/clastix/kamaji
+cd kamaji/deploy
+```
+
+We assume you have installed on your workstation:
+
+- [kubectl](https://kubernetes.io/docs/tasks/tools/)
+- [helm](https://helm.sh/docs/intro/install/)
+- [jq](https://stedolan.github.io/jq/)
+- [openssl](https://www.openssl.org/)
+
+## Access Admin cluster
+In Kamaji, an Admin Cluster is a regular Kubernetes cluster which hosts zero to many Tenant Cluster Control Planes. The admin cluster acts as management cluster for all the Tenant clusters and implements Monitoring, Logging, and Governance of all the Kamaji setup, including all Tenant clusters. 
+
+Throughout the following instructions, shell variables are used to indicate values that you should adjust to your environment:
+
+```bash
+source kamaji.env
+```
+
+Any regular and conformant Kubernetes v1.22+ cluster can be turned into a Kamaji setup. To work properly, the admin cluster should provide:
+
+- CNI module installed, eg. [Calico](https://github.com/projectcalico/calico), [Cilium](https://github.com/cilium/cilium).
+- CSI module installed with a Storage Class for the Tenants' `etcd`.
+- Support for LoadBalancer Service Type, or alternatively, an Ingress Controller, eg. [ingress-nginx](https://github.com/kubernetes/ingress-nginx), [haproxy](https://github.com/haproxytech/kubernetes-ingress).
+- Monitoring Stack, eg. [Prometheus](https://github.com/prometheus-community).
+
+Make sure you have a `kubeconfig` file with admin permissions on the cluster you want to turn into Kamaji Admin Cluster.
+
+## Install Kamaji
+There are multiple ways to deploy Kamaji, including a [single YAML file](../config/install.yaml) and [Helm Chart](../charts/kamaji).
+
+### Multi-tenant datastore
+The Kamaji controller needs to access a multi-tenant datastore in order to save data of the tenants' clusters. Install a multi-tenant `etcd` in the admin cluster as three replicas StatefulSet with data persistence. The Helm [Chart](../charts/kamaji/) provides the installation of an internal `etcd`. However, an externally managed `etcd` is highly recommended. If you'd like to use an external one, you can specify the overrides by setting the value `etcd.deploy=false`.
+
+Optionally, Kamaji offers the possibility of using a different storage system than `etcd` for the tenants' clusters, like MySQL compatible database, thanks to the [kine](https://github.com/k3s-io/kine) integration [here](../deploy/kine/mysql/README.md).
+
+### Install with Helm Chart
+Install with the `helm` in a dedicated namespace of the Admin cluster:
+
+```bash
+helm install --create-namespace --namespace kamaji-system kamaji clastix/kamaji
+```
+
+The Kamaji controller and the multi-tenant `etcd` are now running:
+
+```bash
+kubectl -n kamaji-system get pods
+NAME                      READY   STATUS    RESTARTS       AGE
+etcd-0                    1/1     Running   0              120m
+etcd-1                    1/1     Running   0              120m
+etcd-2                    1/1     Running   0              119m
+kamaji-857fcdf599-4fb2p   2/2     Running   0              120m
+```
+
+You just turned your Kubernetes cluster into a Kamaji cluster to run multiple Tenant Control Planes.
+
+## Create Tenant Cluster
+
+### Tenant Control Plane
+
+A tenant control plane of example looks like:
+
+```yaml
+cat > ${TENANT_NAMESPACE}-${TENANT_NAME}-tcp.yaml <<EOF
+apiVersion: kamaji.clastix.io/v1alpha1
+kind: TenantControlPlane
+metadata:
+  name: ${TENANT_NAME}
+  namespace: ${TENANT_NAMESPACE}
+spec:
+  controlPlane:
+    deployment:
+      replicas: 3
+      additionalMetadata:
+        labels:
+          tenant.clastix.io: ${TENANT_NAME}
+      extraArgs:
+        apiServer: []
+        controllerManager: []
+        scheduler: []
+      resources:
+        apiServer:
+          requests:
+            cpu: 500m
+            memory: 512Mi
+          limits: {}
+        controllerManager:
+          requests:
+            cpu: 250m
+            memory: 256Mi
+          limits: {}
+        scheduler:
+          requests:
+            cpu: 250m
+            memory: 256Mi
+          limits: {}
+    service:
+      additionalMetadata:
+        labels:
+          tenant.clastix.io: ${TENANT_NAME}
+      serviceType: LoadBalancer
+  kubernetes:
+    version: ${TENANT_VERSION}
+    kubelet:
+      cgroupfs: systemd
+    admissionControllers:
+      - ResourceQuota
+      - LimitRanger
+  networkProfile:
+    port: ${TENANT_PORT}
+    certSANs:
+    - ${TENANT_NAME}.${TENANT_DOMAIN}
+    serviceCidr: ${TENANT_SVC_CIDR}
+    podCidr: ${TENANT_POD_CIDR}
+    dnsServiceIPs:
+    - ${TENANT_DNS_SERVICE}
+  addons:
+    coreDNS: {}
+    kubeProxy: {}
+    konnectivity:
+      proxyPort: ${TENANT_PROXY_PORT}
+      resources:
+        requests:
+          cpu: 100m
+          memory: 128Mi
+        limits: {}
+EOF
+
+kubectl create namespace ${TENANT_NAMESPACE}
+kubectl apply -f ${TENANT_NAMESPACE}-${TENANT_NAME}-tcp.yaml
+```
+
+After a few minutes, check the created resources in the tenants namespace and when ready it will look similar to the following:
+
+```command
+kubectl -n tenants get tcp,deploy,pods,svc
+NAME                                             VERSION   STATUS   CONTROL-PLANE-ENDPOINT   KUBECONFIG                   AGE
+tenantcontrolplane.kamaji.clastix.io/tenant-00   v1.23.1   Ready    192.168.32.240:6443      tenant-00-admin-kubeconfig   2m20s
+
+NAME                        READY   UP-TO-DATE   AVAILABLE   AGE
+deployment.apps/tenant-00   3/3     3            3           118s
+
+NAME                             READY   STATUS    RESTARTS   AGE
+pod/tenant-00-58847c8cdd-7hc4n   4/4     Running   0          82s
+pod/tenant-00-58847c8cdd-ft5xt   4/4     Running   0          82s
+pod/tenant-00-58847c8cdd-shc7t   4/4     Running   0          82s
+
+NAME                TYPE           CLUSTER-IP      EXTERNAL-IP      PORT(S)                         AGE
+service/tenant-00   LoadBalancer   10.32.132.241   192.168.32.240   6443:32152/TCP,8132:32713/TCP   2m20s
+```
+
+The regular Tenant Control Plane containers: `kube-apiserver`, `kube-controller-manager`, `kube-scheduler` are running unchanged in the `tcp` pods instead of dedicated machines and they are exposed through a service on the port `6443` of worker nodes in the Admin cluster.
+
+```yaml
+apiVersion: v1
+kind: Service
+metadata:
+  name: tenant-00
+spec:
+  clusterIP: 10.32.233.177
+  loadBalancerIP: 192.168.32.240
+  ports:
+  - name: kube-apiserver
+    nodePort: 31073
+    port: 6443
+    protocol: TCP
+    targetPort: 6443
+  - name: konnectivity-server
+    nodePort: 32125
+    port: 8132
+    protocol: TCP
+    targetPort: 8132
+  selector:
+    kamaji.clastix.io/soot: tenant-00
+  type: LoadBalancer
+```
+
+The `LoadBalancer` service type is used to expose the Tenant Control Plane. However, `NodePort` and `ClusterIP` with an Ingress Controller are still viable options, depending on the case. High Availability and rolling updates of the Tenant Control Plane are provided by the `tcp` Deployment and all the resources reconcilied by the Kamaji controller.
+
+### Konnectivity
+In addition to the standard control plane containers, Kamaji creates an instance of [konnectivity-server](https://kubernetes.io/docs/concepts/architecture/control-plane-node-communication/) running as sidecar container in the `tcp` pod and exposed on port `8132` of the `tcp` service.
+
+This is required when the tenant worker nodes are not reachable from the `tcp` pods. The Konnectivity service consists of two parts: the Konnectivity server in the tenant control plane pod and the Konnectivity agents running on the tenant worker nodes. After worker nodes joined the tenant control plane, the Konnectivity agents initiate connections to the Konnectivity server and maintain the network connections. After enabling the Konnectivity service, all control plane to worker nodes traffic goes through these connections.
+
+> In Kamaji, Konnectivity is enabled by default and can be disabled when not required.
+
+### Working with Tenant Control Plane
+
+Collect the external IP address of the `tcp` service:
+
+```bash
+TENANT_ADDR=$(kubectl -n ${TENANT_NAMESPACE} get svc ${TENANT_NAME} -o json | jq -r ."spec.loadBalancerIP")
+```
+
+and check it out:
+
+```bash
+curl -k https://${TENANT_ADDR}:${TENANT_PORT}/healthz
+curl -k https://${TENANT_ADDR}:${TENANT_PORT}/version
+```
+
+The `kubeconfig` required to access the Tenant Control Plane is stored in a secret:
+
+```bash
+kubectl get secrets -n ${TENANT_NAMESPACE} ${TENANT_NAME}-admin-kubeconfig -o json \
+  | jq -r '.data["admin.conf"]' \
+  | base64 -d \
+  > ${TENANT_NAMESPACE}-${TENANT_NAME}.kubeconfig
+```
+
+and let's check it out:
+
+```bash
+kubectl --kubeconfig=${TENANT_NAMESPACE}-${TENANT_NAME}.kubeconfig cluster-info
+
+Kubernetes control plane is running at https://192.168.32.240:6443
+CoreDNS is running at https://192.168.32.240:6443/api/v1/namespaces/kube-system/services/kube-dns:dns/proxy
+```
+
+Check out how the Tenant control Plane advertises itself to workloads:
+
+```bash
+kubectl --kubeconfig=${TENANT_NAMESPACE}-${TENANT_NAME}.kubeconfig get svc
+
+NAMESPACE     NAME         TYPE        CLUSTER-IP   EXTERNAL-IP   PORT(S)   AGE
+default       kubernetes   ClusterIP   10.32.0.1    <none>        443/TCP   6m
+```
+
+```bash
+kubectl --kubeconfig=${TENANT_NAMESPACE}-${TENANT_NAME}.kubeconfig get ep
+
+NAME         ENDPOINTS             AGE
+kubernetes   192.168.32.240:6443   18m
+```
+
+And make sure it is `${TENANT_ADDR}:${TENANT_PORT}`.
+
+### Preparing Worker Nodes to join
+
+Currently Kamaji does not provide any helper for creation of tenant worker nodes. You should get a set of machines from your infrastructure provider, turn them into worker nodes, and then join to the tenant control plane with the `kubeadm`. In the future, we'll provide integration with Cluster APIs and other IaC tools.
+
+Use bash script `nodes-prerequisites.sh` to install the dependencies on all the worker nodes:
+
+- Install `containerd` as container runtime
+- Install `crictl`, the command line for working with `containerd`
+- Install `kubectl`, `kubelet`, and `kubeadm` in the desired version
+
+> Warning: we assume worker nodes are machines running `Ubuntu 20.04`
+
+Run the installation script:
+
+```bash
+HOSTS=(${WORKER0} ${WORKER1} ${WORKER2})
+./nodes-prerequisites.sh ${TENANT_VERSION:1} ${HOSTS[@]}
+```
+
+### Join Command
+
+The current approach for joining nodes is to use the kubeadm one therefore, we will create a bootstrap token to perform the action. In order to facilitate the step, we will store the entire command of joining in a variable.
+
+```bash
+JOIN_CMD=$(echo "sudo ")$(kubeadm --kubeconfig=${TENANT_NAMESPACE}-${TENANT_NAME}.kubeconfig token create --print-join-command)
+```
+
+### Adding Worker Nodes
+
+A bash loop will be used to join all the available nodes.
+
+```bash
+for i in "${!HOSTS[@]}"; do
+  HOST=${HOSTS[$i]}
+  ssh ${USER}@${HOST} -t ${JOIN_CMD};
+done
+```
+
+Checking the nodes:
+
+```bash
+kubectl --kubeconfig=${TENANT_NAMESPACE}-${TENANT_NAME}.kubeconfig get nodes 
+
+NAME                  STATUS     ROLES    AGE   VERSION
+tenant-00-worker-00   NotReady   <none>   25s   v1.23.5
+tenant-00-worker-01   NotReady   <none>   17s   v1.23.5
+tenant-00-worker-02   NotReady   <none>   9s    v1.23.5
+```
+
+The cluster needs a [CNI](https://kubernetes.io/docs/concepts/extend-kubernetes/compute-storage-net/network-plugins/) plugin to get the nodes ready. In our case, we are going to install [calico](https://projectcalico.docs.tigera.io/about/about-calico).
+
+```bash
+kubectl apply -f calico-cni/calico-crd.yaml --kubeconfig=${TENANT_NAMESPACE}-${TENANT_NAME}.kubeconfig
+kubectl apply -f calico-cni/calico.yaml --kubeconfig=${TENANT_NAMESPACE}-${TENANT_NAME}.kubeconfig
+```
+
+And after a while, `kube-system` pods will be running.
+
+```bash
+kubectl --kubeconfig=${TENANT_NAMESPACE}-${TENANT_NAME}.kubeconfig get pods -n kube-system 
+
+NAME                                       READY   STATUS    RESTARTS   AGE
+calico-kube-controllers-8594699699-dlhbj   1/1     Running   0          3m
+calico-node-kxf6n                          1/1     Running   0          3m
+calico-node-qtdlw                          1/1     Running   0          3m
+coredns-64897985d-2v5lc                    1/1     Running   0          5m
+coredns-64897985d-nq276                    1/1     Running   0          5m
+kube-proxy-cwdww                           1/1     Running   0          3m
+kube-proxy-m48v4                           1/1     Running   0          3m
+```
+
+And the nodes will be ready
+
+```bash
+kubectl get nodes --kubeconfig=${TENANT_NAMESPACE}-${TENANT_NAME}.kubeconfig
+NAME                  STATUS   ROLES    AGE     VERSION
+tenant-00-worker-00   Ready    <none>   2m48s   v1.23.5
+tenant-00-worker-01   Ready    <none>   2m40s   v1.23.5
+tenant-00-worker-02   Ready    <none>   2m32s   v1.23.5
+```
+
+## Smoke test
+
+The tenant cluster is now ready to accept workloads.
+
+Export its `kubeconfig` file
+
+```bash
+export KUBECONFIG=${TENANT_NAMESPACE}-${TENANT_NAME}.kubeconfig
+```
+
+#### Deployment
+Deploy a `nginx` application on the tenant cluster
+
+```bash
+kubectl create deployment nginx --image=nginx
+```
+
+and check the `nginx` pod gets scheduled
+
+```bash
+kubectl get pods -o wide
+
+NAME                     READY   STATUS    RESTARTS   AGE   IP             NODE     
+nginx-6799fc88d8-4sgcb   1/1     Running   0          33s   172.12.121.1   worker02
+```
+
+#### Port Forwarding
+Verify the ability to access applications remotely using port forwarding.
+
+Retrieve the full name of the `nginx` pod:
+
+```bash
+POD_NAME=$(kubectl get pods -l app=nginx -o jsonpath="{.items[0].metadata.name}")
+```
+
+Forward port 8080 on your local machine to port 80 of the `nginx` pod:
+
+```bash
+kubectl port-forward $POD_NAME 8080:80
+
+Forwarding from 127.0.0.1:8080 -> 80
+Forwarding from [::1]:8080 -> 80
+```
+
+In a new terminal make an HTTP request using the forwarding address:
+
+```bash
+curl --head http://127.0.0.1:8080
+
+HTTP/1.1 200 OK
+Server: nginx/1.21.0
+Date: Sat, 19 Jun 2021 08:19:01 GMT
+Content-Type: text/html
+Content-Length: 612
+Last-Modified: Tue, 25 May 2021 12:28:56 GMT
+Connection: keep-alive
+ETag: "60aced88-264"
+Accept-Ranges: bytes
+```
+
+Switch back to the previous terminal and stop the port forwarding to the `nginx` pod.
+
+#### Logs
+Verify the ability to retrieve container logs.
+
+Print the `nginx` pod logs:
+
+```bash
+kubectl logs $POD_NAME
+...
+127.0.0.1 - - [19/Jun/2021:08:19:01 +0000] "HEAD / HTTP/1.1" 200 0 "-" "curl/7.68.0" "-"
+```
+
+#### Kubelet tunnel
+Verify the ability to execute commands in a container.
+
+Print the `nginx` version by executing the `nginx -v` command in the `nginx` container:
+
+```bash
+kubectl exec -ti $POD_NAME -- nginx -v
+nginx version: nginx/1.21.0
+```
+
+#### Services
+Verify the ability to expose applications using a service.
+
+Expose the `nginx` deployment using a `NodePort` service:
+
+```bash
+kubectl expose deployment nginx --port 80 --type NodePort
+```
+
+Retrieve the node port assigned to the `nginx` service:
+
+```bash
+NODE_PORT=$(kubectl get svc nginx \
+  --output=jsonpath='{range .spec.ports[0]}{.nodePort}')
+```
+
+Retrieve the IP address of a worker instance and make an HTTP request:
+
+```bash
+curl -I http://${WORKER0}:${NODE_PORT}
+
+HTTP/1.1 200 OK
+Server: nginx/1.21.0
+Date: Sat, 19 Jun 2021 09:29:01 GMT
+Content-Type: text/html
+Content-Length: 612
+Last-Modified: Tue, 25 May 2021 12:28:56 GMT
+Connection: keep-alive
+ETag: "60aced88-264"
+Accept-Ranges: bytes
+```
+
+## Cleanup
+Remove the worker nodes joined the tenant control plane
+
+```bash
+kubectl delete nodes --all --kubeconfig=${TENANT_NAMESPACE}-${TENANT_NAME}.kubeconfig
+```
+
+For each worker node, login and clean it
+
+```bash
+HOSTS=(${WORKER0} ${WORKER1} ${WORKER2})
+for i in "${!HOSTS[@]}"; do
+  HOST=${HOSTS[$i]}
+  ssh ${USER}@${HOST} -t 'sudo kubeadm reset -f';
+  ssh ${USER}@${HOST} -t 'sudo rm -rf /etc/cni/net.d';
+  ssh ${USER}@${HOST} -t 'sudo systemctl reboot';
+done
+```
+
+Delete the tenant control plane from kamaji
+
+```bash
+kubectl delete -f ${TENANT_NAMESPACE}-${TENANT_NAME}-tcp.yaml
+```

docs/kamaji-tenant-deployment-guide.md

@@ -1,361 +0,0 @@
-# Kamaji Tenant Deployment Guide
-
-This guide defines the necessary actions to generate a kubernetes tenant cluster, which can be considered made of a virtual kubernetes control plane, deployed by Kamaji, and joining worker nodes pool to start workloads.
-
-## Requirements
-
-* [Kubernetes](https://kubernetes.io) Admin Cluster having [Kamaji](./getting-started-with-kamaji.md) installed.
-* [kubectl](https://kubernetes.io/docs/tasks/tools/#kubectl)
-* [jq](https://stedolan.github.io/jq/)
-
-## Tenant Control Plane
-
-Kamaji offers a [CRD](https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/) to provide a declarative approach of managing tenant control planes. This *CRD* is called `TenantControlPlane`, or `tcp` in short.
-
-Use the command `kubectl explain tcp.spec` to understand the fields and their usage.
-
-### Variable Definitions
-Throughout the instructions, shell variables are used to indicate values that you should adjust to your own environment:
-
-```bash
-source kamaji-tenant.env
-```
-
-### Creation
-
-Create a tenant control plane of example
-
-```yaml
-cat > ${TENANT_NAMESPACE}-${TENANT_NAME}-tcp.yaml <<EOF
-apiVersion: kamaji.clastix.io/v1alpha1
-kind: TenantControlPlane
-metadata:
-  name: ${TENANT_NAME}
-  namespace: ${TENANT_NAMESPACE}
-spec:
-  controlPlane:
-    deployment:
-      replicas: 2
-      additionalMetadata:
-        annotations:
-          environment.clastix.io: ${TENANT_NAME}
-        labels:
-          tenant.clastix.io: ${TENANT_NAME}
-          kind.clastix.io: deployment
-    service:
-      additionalMetadata:
-        annotations:
-          environment.clastix.io: ${TENANT_NAME}
-        labels:
-          tenant.clastix.io: ${TENANT_NAME}
-          kind.clastix.io: service
-      serviceType: LoadBalancer
-    ingress:
-      enabled: false
-  kubernetes:
-    version: ${TENANT_VERSION}
-    kubelet:
-      cgroupfs: systemd
-    admissionControllers:
-      - ResourceQuota
-      - LimitRanger
-  networkProfile:
-    address: ${TENANT_ADDR}
-    port: ${TENANT_PORT}
-    domain: ${TENANT_DOMAIN}
-    serviceCidr: ${TENANT_SVC_CIDR}
-    podCidr: ${TENANT_POD_CIDR}
-    dnsServiceIPs:
-    - ${TENANT_DNS_SERVICE}
-EOF
-```
-
-```bash
-kubectl create namespace ${TENANT_NAMESPACE}
-kubectl apply -f ${TENANT_NAMESPACE}-${TENANT_NAME}-tcp.yaml
-```
-
-A tenant control plane control plane is now running as deployment and it is exposed through a service.
-
-Check if control plane of the tenant is reachable and in healty state
-
-```bash
-curl -k https://${TENANT_ADDR}:${TENANT_PORT}/healthz
-curl -k https://${TENANT_ADDR}:${TENANT_PORT}/version
-```
-
-The tenant control plane components, i.e. `kube-apiserver`, `kube-scheduler`, and `kube-controller-manager` are running as containers in the same pods. The `kube-scheduler`, and `kube-controller-manager` connect the `kube-apiserver` throught localhost: `https://127.0.0.1.${TENANT_PORT}`
-
-Let's retrieve the `kubeconfig` files in order to check:
-
-```bash
-kubectl get secrets -n ${TENANT_NAMESPACE} ${TENANT_NAME}-scheduler-kubeconfig -o json \
-  | jq -r '.data["scheduler.conf"]' \
-  | base64 -d \
-  > ${TENANT_NAMESPACE}-${TENANT_NAME}-scheduler.kubeconfig
-```
-
-```bash
-kubectl get secrets -n ${TENANT_NAMESPACE} ${TENANT_NAME}-controller-manager-kubeconfig -o json \
-  | jq -r '.data["controller-manager.conf"]' \
-  | base64 -d \
-  > ${TENANT_NAMESPACE}-${TENANT_NAME}-controller-manager.kubeconfig
-```
-
-## Working with Tenant Control Plane
-
-A new Tenant cluster will be available at this moment but, it will not be useful without having worker nodes joined to it.
-
-### Getting Tenant Control Plane Kubeconfig
-
-Let's retrieve the `kubeconfig` in order to work with the tenant control plane.
-
-```bash
-kubectl get secrets -n ${TENANT_NAMESPACE} ${TENANT_NAME}-admin-kubeconfig -o json \
-  | jq -r '.data["admin.conf"]' \
-  | base64 -d \
-  > ${TENANT_NAMESPACE}-${TENANT_NAME}.kubeconfig
-```
-
-and let's check it out:
-
-```bash
-kubectl --kubeconfig=${TENANT_NAMESPACE}-${TENANT_NAME}.kubeconfig get svc
-
-NAMESPACE     NAME         TYPE        CLUSTER-IP   EXTERNAL-IP   PORT(S)   AGE
-default       kubernetes   ClusterIP   10.32.0.1    <none>        443/TCP   6m
-```
-
-Check out how the Tenant control Plane advertises itself to workloads:
-
-```bash
-kubectl --kubeconfig=${TENANT_NAMESPACE}-${TENANT_NAME}.kubeconfig get ep
-
-NAME         ENDPOINTS             AGE
-kubernetes   192.168.32.150:6443   18m
-```
-
-Make sure it's `${TENANT_ADDR}:${TENANT_PORT}`.
-
-### Preparing Worker Nodes to join
-
-Currently Kamaji does not provide any helper for creation of tenant worker nodes. You should get a set of machines from your infrastructure provider, turn them into worker nodes, and then join to the tenant control plane with the `kubeadm`. In the future, we'll provide integration with Cluster APIs and other IaC tools.
-
-Use bash script `nodes-prerequisites.sh` to install the dependencies on all the worker nodes:
-
-- Install `containerd` as container runtime
-- Install `crictl`, the command line for working with `containerd`
-- Install `kubectl`, `kubelet`, and `kubeadm` in the desired version
-
-> Warning: we assume worker nodes are machines running `Ubuntu 20.04`
-
-Run the installation script:
-
-```bash
-HOSTS=(${WORKER0} ${WORKER1} ${WORKER2} ${WORKER3})
-./nodes-prerequisites.sh ${TENANT_VERSION:1} ${HOSTS[@]}
-```
-
-### Join Command
-
-The current approach for joining nodes is to use the kubeadm one therefore, we will create a bootstrap token to perform the action. In order to facilitate the step, we will store the entire command of joining in a variable.
-
-```bash
-JOIN_CMD=$(echo "sudo ")$(kubeadm --kubeconfig=${TENANT_NAMESPACE}-${TENANT_NAME}.kubeconfig token create --print-join-command)
-```
-
-### Adding Worker Nodes
-
-A bash loop will be used to join all the available nodes.
-
-```bash
-HOSTS=(${WORKER0} ${WORKER1} ${WORKER2} ${WORKER3})
-for i in "${!HOSTS[@]}"; do
-  HOST=${HOSTS[$i]}
-  ssh ${USER}@${HOST} -t ${JOIN_CMD};
-done
-```
-
-Checking the nodes:
-
-```bash
-kubectl --kubeconfig=${TENANT_NAMESPACE}-${TENANT_NAME}.kubeconfig get nodes 
-
-NAME                      STATUS      ROLES    AGE    VERSION
-kamaji-tenant-worker-00   NotReady    <none>   1m     v1.23.4
-kamaji-tenant-worker-01   NotReady    <none>   1m     v1.23.4
-kamaji-tenant-worker-02   NotReady    <none>   1m     v1.23.4
-kamaji-tenant-worker-03   NotReady    <none>   1m     v1.23.4
-```
-
-The cluster needs a [CNI](https://kubernetes.io/docs/concepts/extend-kubernetes/compute-storage-net/network-plugins/) plugin to get the nodes ready. In our case, we are going to install [calico](https://projectcalico.docs.tigera.io/about/about-calico).
-
-```bash
-kubectl apply -f calico-cni/calico-crd.yaml --kubeconfig=${TENANT_NAMESPACE}-${TENANT_NAME}.kubeconfig
-kubectl apply -f calico-cni/calico.yaml --kubeconfig=${TENANT_NAMESPACE}-${TENANT_NAME}.kubeconfig
-```
-
-And after a while, `kube-system` pods will be running.
-
-```bash
-kubectl --kubeconfig=${TENANT_NAMESPACE}-${TENANT_NAME}.kubeconfig get pods -n kube-system 
-
-NAME                                       READY   STATUS    RESTARTS   AGE
-calico-kube-controllers-8594699699-dlhbj   1/1     Running   0          3m
-calico-node-kxf6n                          1/1     Running   0          3m
-calico-node-qtdlw                          1/1     Running   0          3m
-coredns-64897985d-2v5lc                    1/1     Running   0          5m
-coredns-64897985d-nq276                    1/1     Running   0          5m
-kube-proxy-cwdww                           1/1     Running   0          3m
-kube-proxy-m48v4                           1/1     Running   0          3m
-```
-
-And the nodes will be ready
-
-```bash
-kubectl get nodes --kubeconfig=${TENANT_NAMESPACE}-${TENANT_NAME}.kubeconfig
-
-NAME                      STATUS      ROLES    AGE    VERSION
-kamaji-tenant-worker-01   Ready       <none>   10m    v1.23.4
-kamaji-tenant-worker-02   Ready       <none>   10m    v1.23.4
-```
-
-## Smoke test
-
-The tenant cluster is now ready to accept workloads.
-
-Export its `kubeconfig` file
-
-```bash
-export KUBECONFIG=${TENANT_NAMESPACE}-${TENANT_NAME}.kubeconfig
-```
-
-#### Deployment
-Deploy a `nginx` application on the tenant cluster
-
-```bash
-kubectl create deployment nginx --image=nginx
-```
-
-and check the `nginx` pod gets scheduled
-
-```bash
-kubectl get pods -o wide
-
-NAME                     READY   STATUS    RESTARTS   AGE   IP             NODE     
-nginx-6799fc88d8-4sgcb   1/1     Running   0          33s   172.12.121.1   worker02
-```
-
-#### Port Forwarding
-Verify the ability to access applications remotely using port forwarding.
-
-Retrieve the full name of the `nginx` pod:
-
-```bash
-POD_NAME=$(kubectl get pods -l app=nginx -o jsonpath="{.items[0].metadata.name}")
-```
-
-Forward port 8080 on your local machine to port 80 of the `nginx` pod:
-
-```bash
-kubectl port-forward $POD_NAME 8080:80
-
-Forwarding from 127.0.0.1:8080 -> 80
-Forwarding from [::1]:8080 -> 80
-```
-
-In a new terminal make an HTTP request using the forwarding address:
-
-```bash
-curl --head http://127.0.0.1:8080
-
-HTTP/1.1 200 OK
-Server: nginx/1.21.0
-Date: Sat, 19 Jun 2021 08:19:01 GMT
-Content-Type: text/html
-Content-Length: 612
-Last-Modified: Tue, 25 May 2021 12:28:56 GMT
-Connection: keep-alive
-ETag: "60aced88-264"
-Accept-Ranges: bytes
-```
-
-Switch back to the previous terminal and stop the port forwarding to the `nginx` pod.
-
-#### Logs
-Verify the ability to retrieve container logs.
-
-Print the `nginx` pod logs:
-
-```bash
-kubectl logs $POD_NAME
-...
-127.0.0.1 - - [19/Jun/2021:08:19:01 +0000] "HEAD / HTTP/1.1" 200 0 "-" "curl/7.68.0" "-"
-```
-
-#### Kubelet tunnel
-Verify the ability to execute commands in a container.
-
-Print the `nginx` version by executing the `nginx -v` command in the `nginx` container:
-
-```bash
-kubectl exec -ti $POD_NAME -- nginx -v
-nginx version: nginx/1.21.0
-```
-
-#### Services
-Verify the ability to expose applications using a service.
-
-Expose the `nginx` deployment using a `NodePort` service:
-
-```bash
-kubectl expose deployment nginx --port 80 --type NodePort
-```
-
-Retrieve the node port assigned to the `nginx` service:
-
-```bash
-NODE_PORT=$(kubectl get svc nginx \
-  --output=jsonpath='{range .spec.ports[0]}{.nodePort}')
-```
-
-Retrieve the IP address of a worker instance and make an HTTP request:
-
-```bash
-curl -I http://${WORKER0}:${NODE_PORT}
-
-HTTP/1.1 200 OK
-Server: nginx/1.21.0
-Date: Sat, 19 Jun 2021 09:29:01 GMT
-Content-Type: text/html
-Content-Length: 612
-Last-Modified: Tue, 25 May 2021 12:28:56 GMT
-Connection: keep-alive
-ETag: "60aced88-264"
-Accept-Ranges: bytes
-```
-
-## Cleanup Tenant cluster
-Remove the worker nodes joined the tenant control plane
-
-```bash
-kubectl delete nodes --all --kubeconfig=${TENANT_NAMESPACE}-${TENANT_NAME}.kubeconfig
-```
-
-For each worker node, login and clean it
-
-```bash
-HOSTS=(${WORKER0} ${WORKER1} ${WORKER2} ${WORKER3})
-for i in "${!HOSTS[@]}"; do
-  HOST=${HOSTS[$i]}
-  ssh ${USER}@${HOST} -t 'sudo kubeadm reset -f';
-  ssh ${USER}@${HOST} -t 'sudo rm -rf /etc/cni/net.d';
-  ssh ${USER}@${HOST} -t 'sudo systemctl reboot';
-done
-```
-
-Delete the tenant control plane from kamaji
-
-```bash
-kubectl delete -f ${TENANT_NAMESPACE}-${TENANT_NAME}-tcp.yaml
-```

docs/reference.md

@@ -1,12 +1,12 @@
 ## Configuration
 
-Currently **kamaji** supports (in this order):
+Currently **Kamaji** supports (in this order):
 
 * CLI flags
 * Environment variables
 * Configuration files
 
-By default **kamaji** search for the configuration file (`kamaji.yaml`) and uses parameters found inside of it. In case some environment variable are passed, this will override configuration file parameters. In the end, if also a CLI flag is passed, this will override both env vars and config file as well.
+By default **Kamaji** search for the configuration file (`kamaji.yaml`) and uses parameters found inside of it. In case some environment variable are passed, this will override configuration file parameters. In the end, if also a CLI flag is passed, this will override both env vars and config file as well.
 
 This is easily explained in this way:
 
@@ -21,8 +21,13 @@ Available flags are the following:
 --etcd-client-secret-name            Name of the secret which contains ETCD client certificates. (default: "root-client-certs")
 --etcd-client-secret-namespace       Name of the namespace where the secret which contains ETCD client certificates is. (default: "kamaji")
 --etcd-compaction-interval           ETCD Compaction interval (i.e. "5m0s"). (default: "0" (disabled))
---etcd-endpoints                     Comma-separated list with ETCD endpoints (i.e. etcd-0.etcd.kamaji.svc.cluster.local,etcd-1.etcd.kamaji.svc.cluster.local,etcd-2.etcd.kamaji.svc.cluster.local)
+--etcd-endpoints                     Comma-separated list with ETCD endpoints (i.e. https://etcd-0.etcd.kamaji.svc.cluster.local,https://etcd-1.etcd.kamaji.svc.cluster.local,https://etcd-2.etcd.kamaji.svc.cluster.local)
+--etcd-storage-type                  ETCD Storage type (i.e. "etcd", "kine-mysql", "kine-postgresql"). (default: "etcd")
 --health-probe-bind-address string   The address the probe endpoint binds to. (default ":8081")
+--kine-port int                      Port where the DB used by Kine is listening to.
+--kine-host string                   Host where the DB used by Kine is working.
+--kine-secret-name                   Name of the secret which contains the Kine configuration. (default: "kine-secret")
+--kine-secret-name                   Name of the namespace where the secret which contains the Kine configuration. (default: "kamaji-system")
 --kubeconfig string                  Paths to a kubeconfig. Only required if out-of-cluster.
 --leader-elect                       Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.
 --metrics-bind-address string        The address the metric endpoint binds to. (default ":8080")
@@ -43,9 +48,14 @@ Available environment variables are:
 | `KAMAJI_ETCD_CLIENT_SECRET_NAMESPACE`      | Name of the namespace where the secret which contains ETCD client certificates is. (default: "kamaji")  |
 | `KAMAJI_ETCD_COMPACTION_INTERVAL`      | ETCD Compaction interval (i.e. "5m0s"). (default: "0" (disabled))  |
 | `KAMAJI_ETCD_ENDPOINTS`      | Comma-separated list with ETCD endpoints (i.e. etcd-server-1:2379,etcd-server-2:2379). (default: "etcd-server:2379")  |
+| `KAMAJI_ETCD_STORAGE_TYPE`      | ETCD Storage type (i.e. "etcd", "kine-mysql"). (default: "etcd")  |
 | `KAMAJI_ETCD_SERVERS`      | Comma-separated list with ETCD servers (i.e. etcd-0.etcd.kamaji.svc.cluster.local,etcd-1.etcd.kamaji.svc.cluster.local,etcd-2.etcd.kamaji.svc.cluster.local)  |
 | `KAMAJI_METRICS_BIND_ADDRESS`      | The address the metric endpoint binds to. (default ":8080")  |
 | `KAMAJI_HEALTH_PROBE_BIND_ADDRESS` | The address the probe endpoint binds to. (default ":8081")   |
+| `KAMAJI_KINE_MYSQL_HOST` | Host where MySQL is running(default "localhost")   |
+| `KAMAJI_KINE_MYSQL_PORT` | Port where MySQL is running (default: 3306)   |
+| `KAMAJI_KINE_MYSQL_SECRET_NAME` | Name of the secret where the necessary configuration and certificates are. (default: "mysql-config")   |
+| `KAMAJI_KINE_MYSQL_SECRET_NAMESPACE` | Name of the namespace of the secret where the necessary configuration and certificates are. (default: "kamaji-system")   |
 | `KAMAJI_LEADER_ELECTION`           | Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager. |
 | `KAMAJI_TMP_DIRECTORY`           | Directory which will be used to work with temporary files. (default "/tmp/kamaji") |
 
@@ -80,4 +90,43 @@ $ make yaml-installation-file
 
 ```
 
-It will generate a yaml installation file at `config/installation.yaml`. It should be customize accordingly.
+It will generate a yaml installation file at `config/install.yaml`. It should be customize accordingly.
+
+## Tenant Control Planes
+
+**Kamaji** offers a [CRD](https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/) to provide a declarative approach of managing tenant control planes. This *CRD* is called `TenantControlPlane`, or `tcp` in short. Use the command `kubectl explain tcp.spec` to understand the fields and their usage.
+
+### Add-ons
+
+**Kamaji** provides optional installations into the deployed tenant control plane through add-ons. Is it possible to enable/disable them through the `tcp` definition.
+
+### Core DNS
+
+```yaml
+addons:
+    coreDNS: {}
+```
+
+### Kube-Proxy
+
+```yaml
+addons:
+    kubeProxy: {}
+```
+
+### Konnectivity
+
+```yaml
+addons:
+  konnectivity:
+    proxyPort: 31132 # mandatory
+    version: v0.0.31
+    resources:
+      requests:
+        cpu: 100m
+        memory: 128Mi
+      limits:
+        cpu: 100m
+        memory: 128Mi
+    serverImage: us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-server
+    agentImage: us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-agent

docs/versioning.md

@@ -0,0 +1,8 @@
+# Versioning and support
+In Kamaji, there are different components that might require independent versioning and support level:
+
+|Kamaji|Admin Cluster|Tenant Cluster (min)|Tenant Cluster (max)|Konnectivity|Tenant etcd |
+|------|-------------|--------------------|--------------------|------------|------------|
+|0.0.1 |1.22.0+      |1.21.0              |1.23.x              |0.0.32      |3.5.4       | 
+
+Other combinations might work but have not been tested.

e2e/suite_test.go

@@ -1,16 +1,16 @@
 // Copyright 2022 Clastix Labs
 // SPDX-License-Identifier: Apache-2.0
 
-package controllers
+package e2e
 
 import (
-	"path/filepath"
 	"testing"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 	"k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/rest"
+	"k8s.io/utils/pointer"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/envtest"
 	"sigs.k8s.io/controller-runtime/pkg/envtest/printer"
@@ -23,9 +23,11 @@ import (
 // These tests use Ginkgo (BDD-style Go testing framework). Refer to
 // http://onsi.github.io/ginkgo/ to learn more about Ginkgo.
 
-var cfg *rest.Config // nolint
-var k8sClient client.Client
-var testEnv *envtest.Environment
+var (
+	cfg       *rest.Config
+	k8sClient client.Client
+	testEnv   *envtest.Environment
+)
 
 func TestAPIs(t *testing.T) {
 	RegisterFailHandler(Fail)
@@ -40,11 +42,12 @@ var _ = BeforeSuite(func() {
 
 	By("bootstrapping test environment")
 	testEnv = &envtest.Environment{
-		CRDDirectoryPaths:     []string{filepath.Join("..", "config", "crd", "bases")},
-		ErrorIfCRDPathMissing: true,
+		UseExistingCluster: pointer.Bool(true),
 	}
 
-	cfg, err := testEnv.Start()
+	var err error
+
+	cfg, err = testEnv.Start()
 	Expect(err).NotTo(HaveOccurred())
 	Expect(cfg).NotTo(BeNil())
 

e2e/tenant_control_plane_ready_test.go

@@ -0,0 +1,83 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package e2e
+
+import (
+	"context"
+	"time"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+)
+
+var _ = Describe("Deploy a TenantControlPlane resource", func() {
+	// Fill TenantControlPlane object
+	tcp := kamajiv1alpha1.TenantControlPlane{
+		TypeMeta: metav1.TypeMeta{},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "tcp-clusterip",
+			Namespace: "default",
+		},
+		Spec: kamajiv1alpha1.TenantControlPlaneSpec{
+			ControlPlane: kamajiv1alpha1.ControlPlane{
+				Deployment: kamajiv1alpha1.DeploymentSpec{
+					Replicas: 1,
+				},
+				Service: kamajiv1alpha1.ServiceSpec{
+					ServiceType: "ClusterIP",
+				},
+			},
+			NetworkProfile: kamajiv1alpha1.NetworkProfileSpec{
+				Address: "172.18.0.2",
+			},
+			Kubernetes: kamajiv1alpha1.KubernetesSpec{
+				Version: "v1.23.6",
+				Kubelet: kamajiv1alpha1.KubeletSpec{
+					CGroupFS: "cgroupfs",
+				},
+				AdmissionControllers: kamajiv1alpha1.AdmissionControllers{
+					"LimitRanger",
+					"ResourceQuota",
+				},
+			},
+			Addons: kamajiv1alpha1.AddonsSpec{},
+		},
+	}
+
+	// Create a TenantControlPlane resource into the cluster
+	JustBeforeEach(func() {
+		Expect(k8sClient.Create(context.Background(), &tcp)).NotTo(HaveOccurred())
+	})
+
+	// Delete the TenantControlPlane resource after test is finished
+	JustAfterEach(func() {
+		PrintTenantControlPlaneInfo(&tcp)
+		PrintKamajiLogs()
+		Expect(k8sClient.Delete(context.Background(), &tcp)).Should(Succeed())
+	})
+
+	// Check if TenantControlPlane resource has been created
+	It("Should be Ready", func() {
+		Eventually(func() kamajiv1alpha1.KubernetesVersionStatus {
+			err := k8sClient.Get(context.Background(), types.NamespacedName{
+				Name:      tcp.GetName(),
+				Namespace: tcp.GetNamespace(),
+			}, &tcp)
+			if err != nil {
+				return ""
+			}
+
+			// Check if Status field has been created on TenantControlPlane struct
+			if tcp.Status.Kubernetes.Version.Status == nil {
+				return ""
+			}
+
+			return *tcp.Status.Kubernetes.Version.Status
+		}, 5*time.Minute, time.Second).Should(Equal(kamajiv1alpha1.VersionReady))
+	})
+})

e2e/utils_test.go

@@ -0,0 +1,110 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package e2e
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io/ioutil"
+	"os/exec"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/kubernetes"
+
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+)
+
+func GetKindIPAddress() string {
+	ep := &corev1.Endpoints{}
+	Expect(k8sClient.Get(context.Background(), types.NamespacedName{Name: "kubernetes", Namespace: "default"}, ep)).ToNot(HaveOccurred())
+
+	return ep.Subsets[0].Addresses[0].IP
+}
+
+func PrintTenantControlPlaneInfo(tcp *kamajiv1alpha1.TenantControlPlane) {
+	kubectlExec := func(args ...string) {
+		cmd := exec.Command("kubectl")
+
+		var out bytes.Buffer
+		cmd.Stdout = &out
+		cmd.Args = args
+
+		Expect(cmd.Run()).ToNot(HaveOccurred())
+
+		for {
+			line, err := out.ReadString('\n')
+			if err != nil {
+				return
+			}
+
+			_, _ = fmt.Fprint(GinkgoWriter, ">>> ", line)
+		}
+	}
+
+	if CurrentGinkgoTestDescription().Failed {
+		_, _ = fmt.Fprintln(GinkgoWriter, "DEBUG: Tenant Control Plane definition")
+		kubectlExec(
+			fmt.Sprintf("--namespace=%s", tcp.GetNamespace()),
+			"get",
+			"tcp",
+			tcp.GetName(),
+		)
+		_, _ = fmt.Fprintln(GinkgoWriter, "DEBUG: Tenant Control Plane resources")
+		kubectlExec(
+			fmt.Sprintf("--namespace=%s", tcp.GetNamespace()),
+			"get",
+			"svc,deployment,pods,ep,configmap,secrets",
+		)
+		_, _ = fmt.Fprintln(GinkgoWriter, "DEBUG: Tenant Control Plane pods")
+		kubectlExec(
+			fmt.Sprintf("--namespace=%s", tcp.GetNamespace()),
+			"describe",
+			"pods",
+		)
+	}
+}
+
+func PrintKamajiLogs() {
+	if CurrentGinkgoTestDescription().Failed {
+		clientset, err := kubernetes.NewForConfig(cfg)
+		Expect(err).ToNot(HaveOccurred())
+
+		list, err := clientset.CoreV1().Pods("kamaji-system").List(context.Background(), metav1.ListOptions{
+			LabelSelector: "app.kubernetes.io/component=controller-manager",
+		})
+		Expect(err).ToNot(HaveOccurred())
+		Expect(list.Items).To(HaveLen(1))
+
+		request := clientset.CoreV1().Pods("kamaji-system").GetLogs(list.Items[0].GetName(), &corev1.PodLogOptions{
+			Container: "manager",
+			SinceSeconds: func() *int64 {
+				seconds := int64(CurrentGinkgoTestDescription().Duration.Seconds())
+
+				return &seconds
+			}(),
+			Timestamps: true,
+		})
+
+		podLogs, err := request.Stream(context.Background())
+		Expect(err).ToNot(HaveOccurred())
+
+		defer podLogs.Close()
+
+		podBytes, err := ioutil.ReadAll(podLogs)
+		Expect(err).ToNot(HaveOccurred())
+
+		_, _ = fmt.Fprintln(GinkgoWriter, "DEBUG: retrieving Kamaji Pod logs")
+
+		for _, line := range bytes.Split(podBytes, []byte("\n")) {
+			_, _ = fmt.Fprintln(GinkgoWriter, ">>> ", string(line))
+		}
+
+		_, _ = fmt.Fprintln(GinkgoWriter, "DEBUG: end of Kamaji Pod logs")
+	}
+}

e2e/worker_kubeadm_join_test.go

@@ -0,0 +1,167 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package e2e
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"strings"
+	"time"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/testcontainers/testcontainers-go"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/kubernetes/cmd/kubeadm/app/cmd"
+	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/util"
+
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+)
+
+var _ = Describe("starting a kind worker with kubeadm", func() {
+	ctx := context.Background()
+
+	var tcp kamajiv1alpha1.TenantControlPlane
+
+	var workerContainer testcontainers.Container
+
+	var kubeconfigFile *os.File
+
+	JustBeforeEach(func() {
+		tcp = kamajiv1alpha1.TenantControlPlane{
+			TypeMeta: metav1.TypeMeta{},
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "worker-nodes-join",
+				Namespace: "default",
+			},
+			Spec: kamajiv1alpha1.TenantControlPlaneSpec{
+				ControlPlane: kamajiv1alpha1.ControlPlane{
+					Deployment: kamajiv1alpha1.DeploymentSpec{
+						Replicas: 1,
+					},
+					Service: kamajiv1alpha1.ServiceSpec{
+						ServiceType: "NodePort",
+					},
+				},
+				NetworkProfile: kamajiv1alpha1.NetworkProfileSpec{
+					Address: GetKindIPAddress(),
+					Port:    31443,
+				},
+				Kubernetes: kamajiv1alpha1.KubernetesSpec{
+					Version: "v1.23.6",
+					Kubelet: kamajiv1alpha1.KubeletSpec{
+						CGroupFS: "cgroupfs",
+					},
+					AdmissionControllers: kamajiv1alpha1.AdmissionControllers{
+						"LimitRanger",
+						"ResourceQuota",
+					},
+				},
+				Addons: kamajiv1alpha1.AddonsSpec{},
+			},
+		}
+		Expect(k8sClient.Create(ctx, &tcp)).NotTo(HaveOccurred())
+
+		var err error
+
+		workerContainer, err = testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
+			ContainerRequest: testcontainers.ContainerRequest{
+				Name:       fmt.Sprintf("%s-worker-node", tcp.GetName()),
+				Image:      fmt.Sprintf("kindest/node:%s", tcp.Spec.Kubernetes.Version),
+				Mounts:     testcontainers.ContainerMounts{testcontainers.BindMount("/lib/modules", "/lib/modules")},
+				Networks:   []string{"kind"},
+				Privileged: true,
+			},
+			Started: true,
+		})
+		Expect(err).ToNot(HaveOccurred())
+
+		kubeconfigFile, err = ioutil.TempFile("", "kamaji")
+		Expect(err).ToNot(HaveOccurred())
+	})
+
+	JustAfterEach(func() {
+		PrintTenantControlPlaneInfo(&tcp)
+		PrintKamajiLogs()
+		Expect(workerContainer.Terminate(ctx)).ToNot(HaveOccurred())
+		Expect(k8sClient.Delete(ctx, &tcp)).Should(Succeed())
+		Expect(os.Remove(kubeconfigFile.Name())).ToNot(HaveOccurred())
+	})
+
+	It("should join the Tenant Control Plane cluster", func() {
+		By("waiting for the Tenant Control Plane being ready", func() {
+			Eventually(func() kamajiv1alpha1.KubernetesVersionStatus {
+				err := k8sClient.Get(ctx, types.NamespacedName{Name: tcp.GetName(), Namespace: tcp.GetNamespace()}, &tcp)
+				if err != nil {
+					return ""
+				}
+
+				if tcp.Status.Kubernetes.Version.Status == nil {
+					return ""
+				}
+
+				return *tcp.Status.Kubernetes.Version.Status
+			}, 5*time.Minute, time.Second).Should(Equal(kamajiv1alpha1.VersionReady))
+		})
+
+		By("downloading Tenant Control Plane kubeconfig", func() {
+			secret := &corev1.Secret{}
+
+			Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: tcp.GetNamespace(), Name: tcp.Status.KubeConfig.Admin.SecretName}, secret)).NotTo(HaveOccurred())
+
+			_, err := kubeconfigFile.Write(secret.Data["admin.conf"])
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		var joinCommandBuffer *bytes.Buffer
+
+		By("generating kubeadm join command", func() {
+			joinCommandBuffer = bytes.NewBuffer([]byte(""))
+
+			config, err := clientcmd.BuildConfigFromFlags("", kubeconfigFile.Name())
+			Expect(err).ToNot(HaveOccurred())
+
+			clientset, err := kubernetes.NewForConfig(config)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(cmd.RunCreateToken(joinCommandBuffer, clientset, "", util.DefaultInitConfiguration(), true, "", kubeconfigFile.Name())).ToNot(HaveOccurred())
+		})
+
+		By("executing the command in the worker node", func() {
+			cmds := append(strings.Split(strings.TrimSpace(joinCommandBuffer.String()), " "), "--ignore-preflight-errors", "SystemVerification")
+
+			exitCode, err := workerContainer.Exec(ctx, cmds)
+			Expect(exitCode).To(Equal(0))
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		By("waiting for nodes", func() {
+			config, err := clientcmd.BuildConfigFromFlags("", kubeconfigFile.Name())
+			Expect(err).ToNot(HaveOccurred())
+
+			clientset, err := kubernetes.NewForConfig(config)
+			Expect(err).ToNot(HaveOccurred())
+
+			Eventually(func() string {
+				nodes, err := clientset.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
+				if err != nil {
+					return ""
+				}
+
+				if len(nodes.Items) == 0 {
+					return ""
+				}
+
+				return nodes.Items[0].GetName()
+			}, time.Minute, time.Second).Should(Equal(workerContainer.GetContainerID()[:12]))
+		})
+	})
+})

e2e/worker_tcp_change_port_test.go

@@ -0,0 +1,162 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package e2e
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"time"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/version"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
+
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+)
+
+var _ = Describe("validating kubeconfig", func() {
+	ctx := context.Background()
+
+	var tcp *kamajiv1alpha1.TenantControlPlane
+
+	var kubeconfigFile *os.File
+
+	JustBeforeEach(func() {
+		tcp = &kamajiv1alpha1.TenantControlPlane{
+			TypeMeta: metav1.TypeMeta{},
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "kubeconfig",
+				Namespace: "default",
+			},
+			Spec: kamajiv1alpha1.TenantControlPlaneSpec{
+				ControlPlane: kamajiv1alpha1.ControlPlane{
+					Deployment: kamajiv1alpha1.DeploymentSpec{
+						Replicas: 1,
+					},
+					Service: kamajiv1alpha1.ServiceSpec{
+						ServiceType: "NodePort",
+					},
+				},
+				NetworkProfile: kamajiv1alpha1.NetworkProfileSpec{
+					Address: GetKindIPAddress(),
+					Port:    30001,
+				},
+				Kubernetes: kamajiv1alpha1.KubernetesSpec{
+					Version: "v1.23.6",
+					Kubelet: kamajiv1alpha1.KubeletSpec{
+						CGroupFS: "cgroupfs",
+					},
+					AdmissionControllers: kamajiv1alpha1.AdmissionControllers{
+						"LimitRanger",
+						"ResourceQuota",
+					},
+				},
+				Addons: kamajiv1alpha1.AddonsSpec{},
+			},
+		}
+		Expect(k8sClient.Create(ctx, tcp)).NotTo(HaveOccurred())
+
+		var err error
+
+		kubeconfigFile, err = ioutil.TempFile("", "kamaji")
+		Expect(err).ToNot(HaveOccurred())
+	})
+
+	JustAfterEach(func() {
+		PrintKamajiLogs()
+		PrintTenantControlPlaneInfo(tcp)
+		Expect(k8sClient.Delete(ctx, tcp)).Should(Succeed())
+		Expect(os.Remove(kubeconfigFile.Name())).ToNot(HaveOccurred())
+	})
+
+	It("return kubernetes version", func() {
+		for _, port := range []int32{30002, 30003, 30004} {
+			Eventually(func() string {
+				By(fmt.Sprintf("ensuring TCP port is set to %d", port), func() {
+					Eventually(func() (err error) {
+						if err = k8sClient.Get(ctx, types.NamespacedName{Namespace: tcp.GetNamespace(), Name: tcp.GetName()}, tcp); err != nil {
+							_, _ = fmt.Fprintln(GinkgoWriter, "DEBUG: cannot retrieve TCP:", err.Error())
+
+							return err
+						}
+
+						tcp.Spec.NetworkProfile.Port = port
+
+						return k8sClient.Update(ctx, tcp)
+					}, time.Minute, 5*time.Second).ShouldNot(HaveOccurred())
+				})
+
+				By("ensuring port change is defined in the TCP status", func() {
+					Eventually(func() int32 {
+						if err := k8sClient.Get(ctx, types.NamespacedName{Namespace: tcp.GetNamespace(), Name: tcp.GetName()}, tcp); err != nil {
+							_, _ = fmt.Fprintln(GinkgoWriter, "DEBUG: cannot retrieve TCP:", err.Error())
+
+							return 0
+						}
+
+						return tcp.Status.Kubernetes.Service.Port
+					}, time.Minute, 5*time.Second).Should(Equal(port))
+				})
+
+				By("ensuring downloading the updated kubeconfig", func() {
+					Eventually(func() (err error) {
+						if err := k8sClient.Get(ctx, types.NamespacedName{Namespace: tcp.GetNamespace(), Name: tcp.GetName()}, tcp); err != nil {
+							_, _ = fmt.Fprintln(GinkgoWriter, "DEBUG: cannot retrieve TCP:", err.Error())
+
+							return err
+						}
+
+						secret := &corev1.Secret{}
+
+						if err = k8sClient.Get(ctx, types.NamespacedName{Namespace: tcp.GetNamespace(), Name: tcp.Status.KubeConfig.Admin.SecretName}, secret); err != nil {
+							_, _ = fmt.Fprintln(GinkgoWriter, "DEBUG: cannot retrieve kubeconfig secret name:", err.Error())
+
+							return err
+						}
+
+						_, err = kubeconfigFile.Write(secret.Data["admin.conf"])
+
+						return err
+					}, time.Minute, 5*time.Second).ShouldNot(HaveOccurred())
+				})
+
+				var version version.Info
+
+				By("retrieving TCP version using the kubeconfig", func() {
+					config, err := clientcmd.BuildConfigFromFlags("", kubeconfigFile.Name())
+					if err != nil {
+						_, _ = fmt.Fprintln(GinkgoWriter, "DEBUG: cannot generate REST configuration:", err.Error())
+
+						return
+					}
+
+					clientset, err := kubernetes.NewForConfig(config)
+					if err != nil {
+						_, _ = fmt.Fprintln(GinkgoWriter, "DEBUG: cannot generate clientset:", err.Error())
+
+						return
+					}
+
+					serverVersion, err := clientset.ServerVersion()
+					if err != nil {
+						_, _ = fmt.Fprintln(GinkgoWriter, "DEBUG: cannot retrieve server version:", err.Error())
+
+						return
+					}
+
+					version = *serverVersion
+				})
+
+				return version.GitVersion
+			}, 3*time.Minute, 5*time.Second).Should(Equal(tcp.Spec.Kubernetes.Version))
+		}
+	})
+})

go.mod

@@ -1,20 +1,25 @@
 module github.com/clastix/kamaji
 
-go 1.17
+go 1.18
 
 require (
 	github.com/go-logr/logr v1.2.0
-	github.com/google/uuid v1.1.2
+	github.com/go-pg/pg/v10 v10.10.6
+	github.com/go-sql-driver/mysql v1.6.0
+	github.com/google/uuid v1.3.0
+	github.com/json-iterator/go v1.1.12
 	github.com/onsi/ginkgo v1.16.5
 	github.com/onsi/gomega v1.17.0
 	github.com/pkg/errors v0.9.1
 	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.10.1
+	github.com/testcontainers/testcontainers-go v0.13.0
 	go.etcd.io/etcd/api/v3 v3.5.1
 	go.etcd.io/etcd/client/v3 v3.5.1
 	google.golang.org/grpc v1.43.0
 	k8s.io/api v0.23.5
 	k8s.io/apimachinery v0.23.5
+	k8s.io/apiserver v0.23.5
 	k8s.io/client-go v0.23.5
 	k8s.io/cluster-bootstrap v0.0.0
 	k8s.io/component-base v0.23.5
@@ -28,6 +33,7 @@ require (
 require (
 	cloud.google.com/go v0.99.0 // indirect
 	cloud.google.com/go/storage v1.18.2 // indirect
+	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
 	github.com/Azure/go-autorest v14.2.0+incompatible // indirect
 	github.com/Azure/go-autorest/autorest v0.11.18 // indirect
 	github.com/Azure/go-autorest/autorest/adal v0.9.13 // indirect
@@ -35,53 +41,94 @@ require (
 	github.com/Azure/go-autorest/logger v0.2.1 // indirect
 	github.com/Azure/go-autorest/tracing v0.6.0 // indirect
 	github.com/Microsoft/go-winio v0.4.17 // indirect
+	github.com/Microsoft/hcsshim v0.8.23 // indirect
+	github.com/PuerkitoBio/purell v1.1.1 // indirect
+	github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/blang/semver v3.5.1+incompatible // indirect
+	github.com/cenkalti/backoff/v4 v4.1.2 // indirect
 	github.com/census-instrumentation/opencensus-proto v0.3.0 // indirect
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
 	github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4 // indirect
 	github.com/cncf/xds/go v0.0.0-20211216145620-d92e9ce0af51 // indirect
+	github.com/containerd/cgroups v1.0.1 // indirect
+	github.com/containerd/containerd v1.5.9 // indirect
 	github.com/coredns/caddy v1.1.0 // indirect
 	github.com/coredns/corefile-migration v1.0.14 // indirect
 	github.com/coreos/go-semver v0.3.0 // indirect
 	github.com/coreos/go-systemd/v22 v22.3.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/docker/distribution v2.7.1+incompatible // indirect
+	github.com/docker/docker v20.10.11+incompatible // indirect
+	github.com/docker/go-connections v0.4.0 // indirect
+	github.com/docker/go-units v0.4.0 // indirect
 	github.com/envoyproxy/go-control-plane v0.10.1 // indirect
 	github.com/envoyproxy/protoc-gen-validate v0.6.2 // indirect
 	github.com/evanphx/json-patch v4.12.0+incompatible // indirect
 	github.com/form3tech-oss/jwt-go v3.2.3+incompatible // indirect
 	github.com/fsnotify/fsnotify v1.5.1 // indirect
+	github.com/go-errors/errors v1.0.1 // indirect
 	github.com/go-logr/zapr v1.2.0 // indirect
+	github.com/go-openapi/jsonpointer v0.19.5 // indirect
+	github.com/go-openapi/jsonreference v0.19.5 // indirect
+	github.com/go-openapi/swag v0.19.14 // indirect
+	github.com/go-pg/zerochecker v0.2.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/google/btree v1.0.1 // indirect
 	github.com/google/go-cmp v0.5.6 // indirect
 	github.com/google/gofuzz v1.1.0 // indirect
+	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
 	github.com/googleapis/gax-go/v2 v2.1.1 // indirect
 	github.com/googleapis/gnostic v0.5.5 // indirect
 	github.com/googleapis/google-cloud-go-testing v0.0.0-20210719221736-1c9a4c676720 // indirect
+	github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/imdario/mergo v0.3.12 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
-	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/jinzhu/inflection v1.0.0 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
+	github.com/lithammer/dedent v1.1.0 // indirect
 	github.com/magiconair/properties v1.8.5 // indirect
+	github.com/mailru/easyjson v0.7.6 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
 	github.com/mitchellh/mapstructure v1.4.3 // indirect
+	github.com/moby/sys/mount v0.2.0 // indirect
+	github.com/moby/sys/mountinfo v0.5.0 // indirect
+	github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
+	github.com/morikuni/aec v1.0.0 // indirect
 	github.com/nxadm/tail v1.4.8 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/image-spec v1.0.2 // indirect
+	github.com/opencontainers/runc v1.0.2 // indirect
 	github.com/pelletier/go-toml v1.9.4 // indirect
+	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_golang v1.12.1 // indirect
 	github.com/prometheus/client_model v0.2.0 // indirect
 	github.com/prometheus/common v0.32.1 // indirect
 	github.com/prometheus/procfs v0.7.3 // indirect
+	github.com/sirupsen/logrus v1.8.1 // indirect
 	github.com/spf13/afero v1.7.0 // indirect
 	github.com/spf13/cast v1.4.1 // indirect
 	github.com/spf13/cobra v1.4.0 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
+	github.com/stretchr/testify v1.7.0 // indirect
 	github.com/subosito/gotenv v1.2.0 // indirect
+	github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc // indirect
+	github.com/vmihailenco/bufpool v0.1.11 // indirect
+	github.com/vmihailenco/msgpack/v5 v5.3.4 // indirect
+	github.com/vmihailenco/tagparser v0.1.2 // indirect
+	github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
+	github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca // indirect
 	go.etcd.io/etcd/client/pkg/v3 v3.5.1 // indirect
 	go.opencensus.io v0.23.0 // indirect
+	go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect
 	go.uber.org/atomic v1.7.0 // indirect
 	go.uber.org/multierr v1.6.0 // indirect
 	go.uber.org/zap v1.19.1 // indirect
@@ -100,15 +147,19 @@ require (
 	google.golang.org/protobuf v1.27.1 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/ini.v1 v1.66.2 // indirect
+	gopkg.in/square/go-jose.v2 v2.5.1 // indirect
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
 	k8s.io/apiextensions-apiserver v0.23.5 // indirect
-	k8s.io/apiserver v0.23.5 // indirect
+	k8s.io/cli-runtime v0.23.5 // indirect
 	k8s.io/klog/v2 v2.60.1 // indirect
 	k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 // indirect
 	k8s.io/system-validators v1.6.0 // indirect
+	mellium.im/sasl v0.3.0 // indirect
 	sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 // indirect
+	sigs.k8s.io/kustomize/api v0.10.1 // indirect
+	sigs.k8s.io/kustomize/kyaml v0.13.0 // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect
 	sigs.k8s.io/yaml v1.3.0 // indirect
 )

helm/README.md

@@ -1,15 +0,0 @@
-# Deploy with Helm
-
-## Pre-requisites
-
-1. Deploy a [multi-tenant Etcd cluster](https://github.com/clastix/kamaji-internal/blob/master/deploy/getting-started-with-kamaji.md#setup-internal-multi-tenant-etcd)
-
-```
-make -C ../deploy/etcd
-```
-
-## Install
-
-```
-helm upgrade --install --namespace kamaji-system --create-namespace kamaji ./kamaji
-```

helm/kamaji/README.md

@@ -1,100 +0,0 @@
-# kamaji
-
-![Version: 0.1.0](https://img.shields.io/badge/Version-0.1.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.1.0](https://img.shields.io/badge/AppVersion-0.1.0-informational?style=flat-square)
-
-A Kubernetes distribution aimed to build and operate a Managed Kubernetes service with a fraction of operational burde.
-
-**Homepage:** <https://github.com/clastix/kamaji-internal/tree/master/helm/kamaji>
-
-## Installing the Chart
-
-To install the chart with the release name `kamaji`:
-
-### Pre-requisites
-
-1. Deploy a [multi-tenant Etcd cluster](https://github.com/clastix/kamaji-internal/blob/master/deploy/getting-started-with-kamaji.md#setup-internal-multi-tenant-etcd)
-2. Create the `Secret` containing the Etcd CA cert keypair:
-
-```
-kubectl -n kamaji-system create secret generic etcd-certs \
-  --from-file=/path/to/etcd/ca.crt \
-  --from-file=/path/to/etcd/ca.key
-```
-
-3. Create a `Secret` containing the Etcd root user client cert keypair:
-
-```
-kubectl -n kamaji-system create secret tls root-client-certs \
-  --cert=/path/to/etcd/root.pem \
-  --key=/path/to/etcd/root-key.pem
-```
-
-### Install Kamaji
-
-```console
-helm upgrade --install --namespace kamaji-system --create-namespace kamaji .
-```
-
-## Maintainers
-
-| Name | Email | Url |
-| ---- | ------ | --- |
-| Gonzalo Gabriel Jiménez Fuentes | <iam@mendrugory.com> |  |
-| Dario Tranchitella | <dario@tranchitella.eu> |  |
-| Massimiliano Giovagnoli | <me@maxgio.it> |  |
-
-## Source Code
-
-* <https://github.com/clastix/kamaji-internal>
-
-## Requirements
-
-Kubernetes: `>=1.18`
-
-## Values
-
-| Key | Type | Default | Description |
-|-----|------|---------|-------------|
-| affinity | object | `{}` | Kubernetes affinity rules to apply to Kamaji controller pods |
-| configPath | string | `"./kamaji.yaml"` | Configuration file path alternative. (default "./kamaji.yaml") |
-| etcd.caSecret.name | string | `"etcd-certs"` | Name of the secret which contains CA's certificate and private key. (default: "etcd-certs") |
-| etcd.caSecret.namespace | string | `"kamaji-system"` | Namespace of the secret which contains CA's certificate and private key. (default: "kamaji") |
-| etcd.clientSecret.name | string | `"root-client-certs"` | Name of the secret which contains ETCD client certificates. (default: "root-client-certs") |
-| etcd.clientSecret.namespace | string | `"kamaji-system"` | Name of the namespace where the secret which contains ETCD client certificates is. (default: "kamaji") |
-| etcd.compactionInterval | int | `0` | ETCD Compaction interval (e.g. "5m0s"). (default: "0" (disabled)) |
-| etcd.endpoints | string | `"etcd-0.etcd.kamaji-system.svc.cluster.local:2379,etcd-1.etcd.kamaji-system.svc.cluster.local:2379,etcd-2.etcd.kamaji-system.svc.cluster.local:2379"` | (string) Comma-separated list of the endpoints of the etcd cluster's members. |
-| extraArgs | list | `[]` | A list of extra arguments to add to the kamaji controller default ones |
-| fullnameOverride | string | `""` |  |
-| healthProbeBindAddress | string | `":8081"` | The address the probe endpoint binds to. (default ":8081") |
-| image.pullPolicy | string | `"IfNotPresent"` |  |
-| image.repository | string | `"quay.io/clastix/kamaji"` | The container image of the Kamaji controller. |
-| image.tag | string | `"latest"` |  |
-| imagePullSecrets | list | `[]` |  |
-| ingress.annotations | object | `{}` |  |
-| ingress.className | string | `""` | Name of the ingress class to route through this controller. |
-| ingress.enabled | bool | `false` | Whether to expose the Kamaji controller through an Ingress. |
-| ingress.hosts[0].host | string | `"chart-example.local"` |  |
-| ingress.hosts[0].paths[0].path | string | `"/"` |  |
-| ingress.hosts[0].paths[0].pathType | string | `"ImplementationSpecific"` |  |
-| ingress.tls | list | `[]` |  |
-| livenessProbe | object | `{"httpGet":{"path":"/healthz","port":"healthcheck"},"initialDelaySeconds":15,"periodSeconds":20}` | The livenessProbe for the controller container |
-| loggingDevel.enable | bool | `false` | (string) Development Mode defaults(encoder=consoleEncoder,logLevel=Debug,stackTraceLevel=Warn). Production Mode defaults(encoder=jsonEncoder,logLevel=Info,stackTraceLevel=Error) (default false) |
-| metricsBindAddress | string | `":8080"` | (string) The address the metric endpoint binds to. (default ":8080") |
-| nameOverride | string | `""` |  |
-| nodeSelector | object | `{}` | Kubernetes node selector rules to schedule Kamaji controller |
-| podAnnotations | object | `{}` | The annotations to apply to the Kamaji controller pods. |
-| podSecurityContext | object | `{"runAsNonRoot":true}` | The securityContext to apply to the Kamaji controller pods. |
-| readinessProbe | object | `{"httpGet":{"path":"/readyz","port":"healthcheck"},"initialDelaySeconds":5,"periodSeconds":10}` | The readinessProbe for the controller container |
-| replicaCount | int | `1` | The number of the pod replicas for the Kamaji controller. |
-| resources.limits.cpu | string | `"200m"` |  |
-| resources.limits.memory | string | `"100Mi"` |  |
-| resources.requests.cpu | string | `"100m"` |  |
-| resources.requests.memory | string | `"20Mi"` |  |
-| securityContext | object | `{"allowPrivilegeEscalation":false}` | The securityContext to apply to the Kamaji controller container only. It does not apply to the Kamaji RBAC proxy container. |
-| service.port | int | `8443` |  |
-| service.type | string | `"ClusterIP"` |  |
-| serviceAccount.annotations | object | `{}` |  |
-| serviceAccount.create | bool | `true` |  |
-| serviceAccount.name | string | `"kamaji-controller-manager"` |  |
-| temporaryDirectoryPath | string | `"/tmp/kamaji"` | Directory which will be used to work with temporary files. (default "/tmp/kamaji") |
-| tolerations | list | `[]` | Kubernetes node taints that the Kamaji controller pods would tolerate |

helm/kamaji/README.md.gotmpl

@@ -1,45 +0,0 @@
-{{ template "chart.header" . }}
-{{ template "chart.deprecationWarning" . }}
-
-{{ template "chart.badgesSection" . }}
-
-{{ template "chart.description" . }}
-
-{{ template "chart.homepageLine" . }}
-
-## Installing the Chart
-
-To install the chart with the release name `kamaji`:
-
-### Pre-requisites
-
-1. Deploy a [multi-tenant Etcd cluster](https://github.com/clastix/kamaji-internal/blob/master/deploy/getting-started-with-kamaji.md#setup-internal-multi-tenant-etcd)
-2. Create the `Secret` containing the Etcd CA cert keypair:
-
-```
-kubectl -n kamaji-system create secret generic etcd-certs \
-  --from-file=/path/to/etcd/ca.crt \
-  --from-file=/path/to/etcd/ca.key
-```
-
-3. Create a `Secret` containing the Etcd root user client cert keypair:
-
-```
-kubectl -n kamaji-system create secret tls root-client-certs \
-  --cert=/path/to/etcd/root.pem \
-  --key=/path/to/etcd/root-key.pem
-```
-
-### Install Kamaji
-
-```console
-helm upgrade --install --namespace kamaji-system --create-namespace kamaji .
-```
-
-{{ template "chart.maintainersSection" . }}
-
-{{ template "chart.sourcesSection" . }}
-
-{{ template "chart.requirementsSection" . }}
-
-{{ template "chart.valuesSection" . }}

helm/kamaji/crds/tenantcontrolplane.yaml

@@ -1,716 +0,0 @@
-apiVersion: apiextensions.k8s.io/v1
-kind: CustomResourceDefinition
-metadata:
-  annotations:
-    controller-gen.kubebuilder.io/version: v0.6.1
-  creationTimestamp: null
-  name: tenantcontrolplanes.kamaji.clastix.io
-spec:
-  group: kamaji.clastix.io
-  names:
-    kind: TenantControlPlane
-    listKind: TenantControlPlaneList
-    plural: tenantcontrolplanes
-    shortNames:
-    - tcp
-    singular: tenantcontrolplane
-  scope: Namespaced
-  versions:
-  - additionalPrinterColumns:
-    - description: Kubernetes version
-      jsonPath: .spec.kubernetes.version
-      name: Version
-      type: string
-    - description: Tenant Control Plane Endpoint (API server)
-      jsonPath: .status.controlPlaneEndpoint
-      name: Control-Plane-Endpoint
-      type: string
-    - description: Secret which contains admin kubeconfig
-      jsonPath: .status.kubeconfig.admin.secretName
-      name: Kubeconfig
-      type: string
-    - description: Age
-      jsonPath: .metadata.creationTimestamp
-      name: Age
-      type: date
-    name: v1alpha1
-    schema:
-      openAPIV3Schema:
-        description: TenantControlPlane is the Schema for the tenantcontrolplanes API
-        properties:
-          apiVersion:
-            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
-            type: string
-          kind:
-            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
-            type: string
-          metadata:
-            type: object
-          spec:
-            description: TenantControlPlaneSpec defines the desired state of TenantControlPlane
-            properties:
-              controlPlane:
-                description: ControlPlane defines how the Tenant Control Plane Kubernetes resources must be created in the Admin Cluster, such as the number of Pod replicas, the Service resource, or the Ingress.
-                properties:
-                  deployment:
-                    description: Defining the options for the deployed Tenant Control Plane as Deployment resource.
-                    properties:
-                      additionalMetadata:
-                        description: AdditionalMetadata defines which additional metadata, such as labels and annotations, must be attached to the created resource.
-                        properties:
-                          annotations:
-                            additionalProperties:
-                              type: string
-                            type: object
-                          labels:
-                            additionalProperties:
-                              type: string
-                            type: object
-                        type: object
-                      replicas:
-                        default: 2
-                        format: int32
-                        type: integer
-                    type: object
-                  ingress:
-                    description: Defining the options for an Optional Ingress which will expose API Server of the Tenant Control Plane
-                    properties:
-                      additionalMetadata:
-                        description: AdditionalMetadata defines which additional metadata, such as labels and annotations, must be attached to the created resource.
-                        properties:
-                          annotations:
-                            additionalProperties:
-                              type: string
-                            type: object
-                          labels:
-                            additionalProperties:
-                              type: string
-                            type: object
-                        type: object
-                      enabled:
-                        type: boolean
-                      hostname:
-                        description: Hostname is an optional field which will be used as Ingress's Host. If it is not defined, Ingress's host will be "<tenant>.<namespace>.<domain>", where domain is specified under NetworkProfileSpec
-                        type: string
-                      ingressClassName:
-                        type: string
-                    required:
-                    - enabled
-                    type: object
-                  service:
-                    description: Defining the options for the Tenant Control Plane Service resource.
-                    properties:
-                      additionalMetadata:
-                        description: AdditionalMetadata defines which additional metadata, such as labels and annotations, must be attached to the created resource.
-                        properties:
-                          annotations:
-                            additionalProperties:
-                              type: string
-                            type: object
-                          labels:
-                            additionalProperties:
-                              type: string
-                            type: object
-                        type: object
-                      serviceType:
-                        description: ServiceType allows specifying how to expose the Tenant Control Plane.
-                        enum:
-                        - ClusterIP
-                        - NodePort
-                        - LoadBalancer
-                        type: string
-                    required:
-                    - serviceType
-                    type: object
-                required:
-                - service
-                type: object
-              kubernetes:
-                description: Kubernetes specification for tenant control plane
-                properties:
-                  admissionControllers:
-                    default:
-                    - CertificateApproval
-                    - CertificateSigning
-                    - CertificateSubjectRestriction
-                    - DefaultIngressClass
-                    - DefaultStorageClass
-                    - DefaultTolerationSeconds
-                    - LimitRanger
-                    - MutatingAdmissionWebhook
-                    - NamespaceLifecycle
-                    - PersistentVolumeClaimResize
-                    - Priority
-                    - ResourceQuota
-                    - RuntimeClass
-                    - ServiceAccount
-                    - StorageObjectInUseProtection
-                    - TaintNodesByCondition
-                    - ValidatingAdmissionWebhook
-                    description: 'List of enabled Admission Controllers for the Tenant cluster. Full reference available here: https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers'
-                    items:
-                      enum:
-                      - AlwaysAdmit
-                      - AlwaysDeny
-                      - AlwaysPullImages
-                      - CertificateApproval
-                      - CertificateSigning
-                      - CertificateSubjectRestriction
-                      - DefaultIngressClass
-                      - DefaultStorageClass
-                      - DefaultTolerationSeconds
-                      - DenyEscalatingExec
-                      - DenyExecOnPrivileged
-                      - DenyServiceExternalIPs
-                      - EventRateLimit
-                      - ExtendedResourceToleration
-                      - ImagePolicyWebhook
-                      - LimitPodHardAntiAffinityTopology
-                      - LimitRanger
-                      - MutatingAdmissionWebhook
-                      - NamespaceAutoProvision
-                      - NamespaceExists
-                      - NamespaceLifecycle
-                      - NodeRestriction
-                      - OwnerReferencesPermissionEnforcement
-                      - PersistentVolumeClaimResize
-                      - PersistentVolumeLabel
-                      - PodNodeSelector
-                      - PodSecurity
-                      - PodSecurityPolicy
-                      - PodTolerationRestriction
-                      - Priority
-                      - ResourceQuota
-                      - RuntimeClass
-                      - SecurityContextDeny
-                      - ServiceAccount
-                      - StorageObjectInUseProtection
-                      - TaintNodesByCondition
-                      - ValidatingAdmissionWebhook
-                      type: string
-                    type: array
-                  kubelet:
-                    properties:
-                      cgroupfs:
-                        description: CGroupFS defines the  cgroup driver for Kubelet https://kubernetes.io/docs/tasks/administer-cluster/kubeadm/configure-cgroup-driver/
-                        enum:
-                        - systemd
-                        - cgroupfs
-                        type: string
-                    type: object
-                  version:
-                    description: Kubernetes Version for the tenant control plane
-                    type: string
-                required:
-                - kubelet
-                - version
-                type: object
-              networkProfile:
-                description: NetworkProfile specifies how the network is
-                properties:
-                  address:
-                    description: Address where API server of will be exposed
-                    type: string
-                  dnsServiceIPs:
-                    items:
-                      type: string
-                    type: array
-                  domain:
-                    description: Domain of the tenant control plane
-                    type: string
-                  podCidr:
-                    description: CIDR for Kubernetes Pods
-                    type: string
-                  port:
-                    default: 6443
-                    description: Port where API server of will be exposed
-                    format: int32
-                    type: integer
-                  serviceCidr:
-                    description: Kubernetes Service
-                    type: string
-                required:
-                - address
-                - dnsServiceIPs
-                - domain
-                - podCidr
-                - port
-                - serviceCidr
-                type: object
-            required:
-            - controlPlane
-            - kubernetes
-            type: object
-          status:
-            description: TenantControlPlaneStatus defines the observed state of TenantControlPlane
-            properties:
-              certificates:
-                description: Certificates contains information about the different certificates that are necessary to run a kubernetes control plane
-                properties:
-                  apiServer:
-                    description: CertificatePrivateKeyPair defines the status
-                    properties:
-                      lastUpdate:
-                        format: date-time
-                        type: string
-                      secretName:
-                        type: string
-                    type: object
-                  apiServerKubeletClient:
-                    description: CertificatePrivateKeyPair defines the status
-                    properties:
-                      lastUpdate:
-                        format: date-time
-                        type: string
-                      secretName:
-                        type: string
-                    type: object
-                  ca:
-                    description: CertificatePrivateKeyPair defines the status
-                    properties:
-                      lastUpdate:
-                        format: date-time
-                        type: string
-                      secretName:
-                        type: string
-                    type: object
-                  etcd:
-                    description: ETCDAPIServerCertificate defines the observed state of ETCD Certificate for API server
-                    properties:
-                      apiServer:
-                        description: ETCDAPIServerCertificate defines the observed state of ETCD Certificate for API server
-                        properties:
-                          lastUpdate:
-                            format: date-time
-                            type: string
-                          secretName:
-                            type: string
-                        type: object
-                      ca:
-                        description: ETCDAPIServerCertificate defines the observed state of ETCD Certificate for API server
-                        properties:
-                          lastUpdate:
-                            format: date-time
-                            type: string
-                          secretName:
-                            type: string
-                        type: object
-                    type: object
-                  frontProxyCA:
-                    description: CertificatePrivateKeyPair defines the status
-                    properties:
-                      lastUpdate:
-                        format: date-time
-                        type: string
-                      secretName:
-                        type: string
-                    type: object
-                  frontProxyClient:
-                    description: CertificatePrivateKeyPair defines the status
-                    properties:
-                      lastUpdate:
-                        format: date-time
-                        type: string
-                      secretName:
-                        type: string
-                    type: object
-                  sa:
-                    description: CertificatePrivateKeyPair defines the status
-                    properties:
-                      lastUpdate:
-                        format: date-time
-                        type: string
-                      secretName:
-                        type: string
-                    type: object
-                type: object
-              controlPlaneEndpoint:
-                description: ControlPlaneEndpoint contains the status of the kubernetes control plane
-                type: string
-              kubeadmPhase:
-                description: KubeadmPhase contains the status of the kubeadm phases action
-                properties:
-                  addonCoreDNS:
-                    description: KubeadmPhasesStatus contains the status of of a kubeadm phase action
-                    properties:
-                      kubeadmConfigResourceVersion:
-                        type: string
-                      lastUpdate:
-                        format: date-time
-                        type: string
-                    type: object
-                  addonKubeProxy:
-                    description: KubeadmPhasesStatus contains the status of of a kubeadm phase action
-                    properties:
-                      kubeadmConfigResourceVersion:
-                        type: string
-                      lastUpdate:
-                        format: date-time
-                        type: string
-                    type: object
-                  bootstrapToken:
-                    description: KubeadmPhasesStatus contains the status of of a kubeadm phase action
-                    properties:
-                      kubeadmConfigResourceVersion:
-                        type: string
-                      lastUpdate:
-                        format: date-time
-                        type: string
-                    type: object
-                  uploadConfigKubeadm:
-                    description: KubeadmPhasesStatus contains the status of of a kubeadm phase action
-                    properties:
-                      kubeadmConfigResourceVersion:
-                        type: string
-                      lastUpdate:
-                        format: date-time
-                        type: string
-                    type: object
-                  uploadConfigKubelet:
-                    description: KubeadmPhasesStatus contains the status of of a kubeadm phase action
-                    properties:
-                      kubeadmConfigResourceVersion:
-                        type: string
-                      lastUpdate:
-                        format: date-time
-                        type: string
-                    type: object
-                required:
-                - addonCoreDNS
-                - addonKubeProxy
-                - bootstrapToken
-                - uploadConfigKubeadm
-                - uploadConfigKubelet
-                type: object
-              kubeadmconfig:
-                description: KubeadmConfig contains the status of the configuration required by kubeadm
-                properties:
-                  configmapName:
-                    type: string
-                  lastUpdate:
-                    format: date-time
-                    type: string
-                  resourceVersion:
-                    type: string
-                required:
-                - resourceVersion
-                type: object
-              kubeconfig:
-                description: KubeConfig contains information about the kubenconfigs that control plane pieces need
-                properties:
-                  admin:
-                    description: TenantControlPlaneKubeconfigsStatus contains information about a the generated kubeconfig
-                    properties:
-                      lastUpdate:
-                        format: date-time
-                        type: string
-                      secretName:
-                        type: string
-                    type: object
-                  controlerManager:
-                    description: TenantControlPlaneKubeconfigsStatus contains information about a the generated kubeconfig
-                    properties:
-                      lastUpdate:
-                        format: date-time
-                        type: string
-                      secretName:
-                        type: string
-                    type: object
-                  scheduler:
-                    description: TenantControlPlaneKubeconfigsStatus contains information about a the generated kubeconfig
-                    properties:
-                      lastUpdate:
-                        format: date-time
-                        type: string
-                      secretName:
-                        type: string
-                    type: object
-                type: object
-              kubernetesResources:
-                description: KubernetesResources contains information about the reconciliation of the required Kubernetes resources deployed in the admin cluster
-                properties:
-                  deployment:
-                    description: KubernetesDeploymentStatus defines the status for the Tenant Control Plane Deployment in the management cluster.
-                    properties:
-                      availableReplicas:
-                        description: Total number of available pods (ready for at least minReadySeconds) targeted by this deployment.
-                        format: int32
-                        type: integer
-                      collisionCount:
-                        description: Count of hash collisions for the Deployment. The Deployment controller uses this field as a collision avoidance mechanism when it needs to create the name for the newest ReplicaSet.
-                        format: int32
-                        type: integer
-                      conditions:
-                        description: Represents the latest available observations of a deployment's current state.
-                        items:
-                          description: DeploymentCondition describes the state of a deployment at a certain point.
-                          properties:
-                            lastTransitionTime:
-                              description: Last time the condition transitioned from one status to another.
-                              format: date-time
-                              type: string
-                            lastUpdateTime:
-                              description: The last time this condition was updated.
-                              format: date-time
-                              type: string
-                            message:
-                              description: A human readable message indicating details about the transition.
-                              type: string
-                            reason:
-                              description: The reason for the condition's last transition.
-                              type: string
-                            status:
-                              description: Status of the condition, one of True, False, Unknown.
-                              type: string
-                            type:
-                              description: Type of deployment condition.
-                              type: string
-                          required:
-                          - status
-                          - type
-                          type: object
-                        type: array
-                      name:
-                        description: The name of the Deployment for the given cluster.
-                        type: string
-                      namespace:
-                        description: The namespace which the Deployment for the given cluster is deployed.
-                        type: string
-                      observedGeneration:
-                        description: The generation observed by the deployment controller.
-                        format: int64
-                        type: integer
-                      readyReplicas:
-                        description: readyReplicas is the number of pods targeted by this Deployment with a Ready Condition.
-                        format: int32
-                        type: integer
-                      replicas:
-                        description: Total number of non-terminated pods targeted by this deployment (their labels match the selector).
-                        format: int32
-                        type: integer
-                      unavailableReplicas:
-                        description: Total number of unavailable pods targeted by this deployment. This is the total number of pods that are still required for the deployment to have 100% available capacity. They may either be pods that are running but not yet available or pods that still have not been created.
-                        format: int32
-                        type: integer
-                      updatedReplicas:
-                        description: Total number of non-terminated pods targeted by this deployment that have the desired template spec.
-                        format: int32
-                        type: integer
-                    required:
-                    - name
-                    - namespace
-                    type: object
-                  ingress:
-                    description: KubernetesIngressStatus defines the status for the Tenant Control Plane Ingress in the management cluster.
-                    properties:
-                      loadBalancer:
-                        description: LoadBalancer contains the current status of the load-balancer.
-                        properties:
-                          ingress:
-                            description: Ingress is a list containing ingress points for the load-balancer. Traffic intended for the service should be sent to these ingress points.
-                            items:
-                              description: 'LoadBalancerIngress represents the status of a load-balancer ingress point: traffic intended for the service should be sent to an ingress point.'
-                              properties:
-                                hostname:
-                                  description: Hostname is set for load-balancer ingress points that are DNS based (typically AWS load-balancers)
-                                  type: string
-                                ip:
-                                  description: IP is set for load-balancer ingress points that are IP based (typically GCE or OpenStack load-balancers)
-                                  type: string
-                                ports:
-                                  description: Ports is a list of records of service ports If used, every port defined in the service should have an entry in it
-                                  items:
-                                    properties:
-                                      error:
-                                        description: 'Error is to record the problem with the service port The format of the error shall comply with the following rules: - built-in error values shall be specified in this file and those shall use   CamelCase names - cloud provider specific error values must have names that comply with the   format foo.example.com/CamelCase. --- The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)'
-                                        maxLength: 316
-                                        pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
-                                        type: string
-                                      port:
-                                        description: Port is the port number of the service port of which status is recorded here
-                                        format: int32
-                                        type: integer
-                                      protocol:
-                                        default: TCP
-                                        description: 'Protocol is the protocol of the service port of which status is recorded here The supported values are: "TCP", "UDP", "SCTP"'
-                                        type: string
-                                    required:
-                                    - port
-                                    - protocol
-                                    type: object
-                                  type: array
-                                  x-kubernetes-list-type: atomic
-                              type: object
-                            type: array
-                        type: object
-                      name:
-                        description: The name of the Ingress for the given cluster.
-                        type: string
-                      namespace:
-                        description: The namespace which the Ingress for the given cluster is deployed.
-                        type: string
-                    required:
-                    - name
-                    - namespace
-                    type: object
-                  service:
-                    description: KubernetesServiceStatus defines the status for the Tenant Control Plane Service in the management cluster.
-                    properties:
-                      conditions:
-                        description: Current service state
-                        items:
-                          description: "Condition contains details for one aspect of the current state of this API Resource. --- This struct is intended for direct use as an array at the field path .status.conditions.  For example, type FooStatus struct{     // Represents the observations of a foo's current state.     // Known .status.conditions.type are: \"Available\", \"Progressing\", and \"Degraded\"     // +patchMergeKey=type     // +patchStrategy=merge     // +listType=map     // +listMapKey=type     Conditions []metav1.Condition `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"` \n     // other fields }"
-                          properties:
-                            lastTransitionTime:
-                              description: lastTransitionTime is the last time the condition transitioned from one status to another. This should be when the underlying condition changed.  If that is not known, then using the time when the API field changed is acceptable.
-                              format: date-time
-                              type: string
-                            message:
-                              description: message is a human readable message indicating details about the transition. This may be an empty string.
-                              maxLength: 32768
-                              type: string
-                            observedGeneration:
-                              description: observedGeneration represents the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date with respect to the current state of the instance.
-                              format: int64
-                              minimum: 0
-                              type: integer
-                            reason:
-                              description: reason contains a programmatic identifier indicating the reason for the condition's last transition. Producers of specific condition types may define expected values and meanings for this field, and whether the values are considered a guaranteed API. The value should be a CamelCase string. This field may not be empty.
-                              maxLength: 1024
-                              minLength: 1
-                              pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
-                              type: string
-                            status:
-                              description: status of the condition, one of True, False, Unknown.
-                              enum:
-                              - "True"
-                              - "False"
-                              - Unknown
-                              type: string
-                            type:
-                              description: type of condition in CamelCase or in foo.example.com/CamelCase. --- Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be useful (see .node.status.conditions), the ability to deconflict is important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)
-                              maxLength: 316
-                              pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
-                              type: string
-                          required:
-                          - lastTransitionTime
-                          - message
-                          - reason
-                          - status
-                          - type
-                          type: object
-                        type: array
-                        x-kubernetes-list-map-keys:
-                        - type
-                        x-kubernetes-list-type: map
-                      loadBalancer:
-                        description: LoadBalancer contains the current status of the load-balancer, if one is present.
-                        properties:
-                          ingress:
-                            description: Ingress is a list containing ingress points for the load-balancer. Traffic intended for the service should be sent to these ingress points.
-                            items:
-                              description: 'LoadBalancerIngress represents the status of a load-balancer ingress point: traffic intended for the service should be sent to an ingress point.'
-                              properties:
-                                hostname:
-                                  description: Hostname is set for load-balancer ingress points that are DNS based (typically AWS load-balancers)
-                                  type: string
-                                ip:
-                                  description: IP is set for load-balancer ingress points that are IP based (typically GCE or OpenStack load-balancers)
-                                  type: string
-                                ports:
-                                  description: Ports is a list of records of service ports If used, every port defined in the service should have an entry in it
-                                  items:
-                                    properties:
-                                      error:
-                                        description: 'Error is to record the problem with the service port The format of the error shall comply with the following rules: - built-in error values shall be specified in this file and those shall use   CamelCase names - cloud provider specific error values must have names that comply with the   format foo.example.com/CamelCase. --- The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)'
-                                        maxLength: 316
-                                        pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
-                                        type: string
-                                      port:
-                                        description: Port is the port number of the service port of which status is recorded here
-                                        format: int32
-                                        type: integer
-                                      protocol:
-                                        default: TCP
-                                        description: 'Protocol is the protocol of the service port of which status is recorded here The supported values are: "TCP", "UDP", "SCTP"'
-                                        type: string
-                                    required:
-                                    - port
-                                    - protocol
-                                    type: object
-                                  type: array
-                                  x-kubernetes-list-type: atomic
-                              type: object
-                            type: array
-                        type: object
-                      name:
-                        description: The name of the Service for the given cluster.
-                        type: string
-                      namespace:
-                        description: The namespace which the Service for the given cluster is deployed.
-                        type: string
-                      port:
-                        description: The port where the service is running
-                        format: int32
-                        type: integer
-                    required:
-                    - name
-                    - namespace
-                    - port
-                    type: object
-                type: object
-              storage:
-                description: Storage Status contains information about Kubernetes storage system
-                properties:
-                  etcd:
-                    description: ETCDStatus defines the observed state of ETCDStatus
-                    properties:
-                      role:
-                        properties:
-                          exists:
-                            type: boolean
-                          name:
-                            type: string
-                          permissions:
-                            items:
-                              properties:
-                                key:
-                                  type: string
-                                rangeEnd:
-                                  type: string
-                                type:
-                                  type: integer
-                              type: object
-                            type: array
-                        required:
-                        - exists
-                        - name
-                        type: object
-                      user:
-                        properties:
-                          exists:
-                            type: boolean
-                          name:
-                            type: string
-                          roles:
-                            items:
-                              type: string
-                            type: array
-                        required:
-                        - exists
-                        - name
-                        type: object
-                    type: object
-                type: object
-            type: object
-        type: object
-    served: true
-    storage: true
-    subresources:
-      status: {}
-status:
-  acceptedNames:
-    kind: ""
-    plural: ""
-  conditions: []
-  storedVersions: []

helm/kamaji/templates/NOTES.txt

@@ -1,22 +0,0 @@
-1. Get the application URL by running these commands:
-{{- if .Values.ingress.enabled }}
-{{- range $host := .Values.ingress.hosts }}
-  {{- range .paths }}
-  http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }}
-  {{- end }}
-{{- end }}
-{{- else if contains "NodePort" .Values.service.type }}
-  export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "kamaji.fullname" . }})
-  export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
-  echo http://$NODE_IP:$NODE_PORT
-{{- else if contains "LoadBalancer" .Values.service.type }}
-     NOTE: It may take a few minutes for the LoadBalancer IP to be available.
-           You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "kamaji.fullname" . }}'
-  export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "kamaji.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}")
-  echo http://$SERVICE_IP:{{ .Values.service.port }}
-{{- else if contains "ClusterIP" .Values.service.type }}
-  export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "kamaji.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
-  export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}")
-  echo "Visit http://127.0.0.1:8080 to use your application"
-  kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT
-{{- end }}

helm/kamaji/templates/ingress.yaml

@@ -1,61 +0,0 @@
-{{- if .Values.ingress.enabled -}}
-{{- $fullName := include "kamaji.fullname" . -}}
-{{- $svcPort := .Values.service.port -}}
-{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }}
-  {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }}
-  {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}}
-  {{- end }}
-{{- end }}
-{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}}
-apiVersion: networking.k8s.io/v1
-{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}}
-apiVersion: networking.k8s.io/v1beta1
-{{- else -}}
-apiVersion: extensions/v1beta1
-{{- end }}
-kind: Ingress
-metadata:
-  name: {{ $fullName }}
-  labels:
-    {{- include "kamaji.labels" . | nindent 4 }}
-  {{- with .Values.ingress.annotations }}
-  annotations:
-    {{- toYaml . | nindent 4 }}
-  {{- end }}
-spec:
-  {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }}
-  ingressClassName: {{ .Values.ingress.className }}
-  {{- end }}
-  {{- if .Values.ingress.tls }}
-  tls:
-    {{- range .Values.ingress.tls }}
-    - hosts:
-        {{- range .hosts }}
-        - {{ . | quote }}
-        {{- end }}
-      secretName: {{ .secretName }}
-    {{- end }}
-  {{- end }}
-  rules:
-    {{- range .Values.ingress.hosts }}
-    - host: {{ .host | quote }}
-      http:
-        paths:
-          {{- range .paths }}
-          - path: {{ .path }}
-            {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }}
-            pathType: {{ .pathType }}
-            {{- end }}
-            backend:
-              {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }}
-              service:
-                name: {{ $fullName }}
-                port:
-                  number: {{ $svcPort }}
-              {{- else }}
-              serviceName: {{ $fullName }}
-              servicePort: {{ $svcPort }}
-              {{- end }}
-          {{- end }}
-    {{- end }}
-{{- end }}