chore(helm): releasing v1.0.0

Signed-off-by: bsctl <adriano@clastix.io>

.github/workflows/ci.yaml

@@ -14,12 +14,12 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-go@v3
         with:
-          go-version: '1.19'
+          go-version: '1.22'
           check-latest: true
       - name: Run golangci-lint
         uses: golangci/golangci-lint-action@v3.2.0
         with:
-          version: v1.49.0
+          version: v1.54.2
           only-new-issues: false
           args: --timeout 5m --config .golangci.yml
   diff:
@@ -31,7 +31,7 @@ jobs:
           fetch-depth: 0
       - uses: actions/setup-go@v3
         with:
-          go-version: '1.19'
+          go-version: '1.22'
           check-latest: true
       - run: make yaml-installation-file
       - name: Checking if YAML installer file is not aligned

.github/workflows/docker-ci.yml

@@ -12,12 +12,14 @@ jobs:
 
       - name: Checkout
         uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
 
       - name: Generate build-args
         id: build-args
         run: |
           # Declare vars for internal use
-          VERSION=$(git describe --abbrev=0 --tags)
+          VERSION=$(make get_version)
           GIT_HEAD_COMMIT=$(git rev-parse --short HEAD)
           GIT_TAG_COMMIT=$(git rev-parse --short $VERSION)
           GIT_MODIFIED_1=$(git diff $GIT_HEAD_COMMIT $GIT_TAG_COMMIT --quiet && echo "" || echo ".dev")
@@ -85,7 +87,13 @@ jobs:
           platforms: linux/amd64,linux/arm64,linux/arm
           push: true
           tags: ${{ steps.meta.outputs.tags }}
-          build-args:
+          build-args: |
+            GIT_LAST_TAG=${{ env.GIT_LAST_TAG }}
+            GIT_HEAD_COMMIT=${{ env.GIT_HEAD_COMMIT }}
+            GIT_TAG_COMMIT=${{ env.GIT_TAG_COMMIT }}
+            GIT_MODIFIED=${{ env.GIT_MODIFIED }}
+            GIT_REPO=${{ env.GIT_REPO }}
+            BUILD_DATE=${{ env.BUILD_DATE }}
 
       - name: Image digest
         run: echo ${{ steps.build-release.outputs.digest }}

.github/workflows/e2e.yaml

@@ -13,6 +13,7 @@ on:
       - 'main.go'
       - 'Makefile'
       - 'internal/**'
+      - 'cmd/**'
   pull_request:
     branches: [ "*" ]
     paths:
@@ -25,6 +26,7 @@ on:
       - 'main.go'
       - 'Makefile'
       - 'internal/**'
+      - 'cmd/**'
 
 jobs:
   kind:
@@ -36,7 +38,7 @@ jobs:
           fetch-depth: 0
       - uses: actions/setup-go@v3
         with:
-          go-version: '1.19'
+          go-version: '1.22'
           check-latest: true
       - run: |
           sudo apt-get update

.gitignore

@@ -24,10 +24,16 @@ bin
 *~
 .vscode
 
+# Tilt files.
+.tiltbuild
+
 **/*.kubeconfig
 **/*.crt
 **/*.key
 **/*.pem
 **/*.csr
-**/server-csr.json
 .DS_Store
+
+**/server-csr.json
+!deploy/kine/mysql/server-csr.json
+!deploy/kine/nats/server-csr.json

.golangci.yml

@@ -3,7 +3,7 @@ linters-settings:
     sections:
       - standard
       - default
-      - prefix(github.com/clastix/kamaji)
+      - prefix(github.com/clastix/kamaji/)
   goheader:
     template: |-
       Copyright 2022 Clastix Labs
@@ -11,6 +11,7 @@ linters-settings:
 
 linters:
   disable:
+    - depguard
     - wrapcheck
     - gomnd
     - scopelint
@@ -36,6 +37,8 @@ linters:
     - funlen
     - dupl
     - cyclop
+    - gocognit
+    - nestif
     # deprecated linters
     - deadcode
     - golint

ADOPTERS.md

@@ -0,0 +1,28 @@
+# Adopters
+
+This is a list of companies that have adopted Kamaji.
+Feel free to open a Pull-Request to get yours listed.
+
+### Adopter list (alphabetically)
+
+| Type | Name | Since | Website | Use-Case |
+|:-|:-|:-|:-|:-|
+| Vendor | DCloud | 2024 | [link](https://dcloud.co.id) | DCloud is an Indonesian Cloud Provider using Kamaji to build and offer [Managed Kubernetes Service](https://dcloud.co.id/dkubes.html). |
+| End-user | Sicuro Tech Lab | 2024 | [link](https://sicurotechlab.it/) | Sicuro Tech Lab offers cloud infrastructure for Web Agencies and uses kamaji to provide managed k8s services. |
+| R&D | TIM | 2024 | [link](https://www.gruppotim.it) | TIM is an Italian telecommunications company using Kamaji for experimental research and development purposes. |
+| End-user | KINX | 2024 | [link](https://kinx.net/?lang=en) | KINX is an Internet infrastructure service provider and will use kamaji for its new [Managed Kubernetes Service](https://kinx.net/service/cloud/kubernetes/intro/?lang=en). |
+| End-user | sevensphere | 2023 | [link](https://www.sevensphere.io) | Sevensphere provides consulting services for end-user companies / cloud providers and uses Kamaji for designing cloud/on-premises Kubernetes-as-a-Service platform. |
+| Vendor | Ænix | 2023 | [link](https://aenix.io/) | Ænix provides consulting services for cloud providers and uses Kamaji for running Kubernetes-as-a-Service in free PaaS platform [Cozystack](https://cozystack.io). |
+| Vendor | Netsons | 2023 | [link](https://www.netsons.com) | Netsons is an Italian hosting and cloud provider and uses Kamaji in its [Managed Kubernetes](https://www.netsons.com/kubernetes) offering. |
+| Vendor | Aknostic | 2023 | [link](https://aknostic.com) | Aknostic is a cloud-native consultancy company using Kamaji to build a Kubernetes based PaaS. |
+
+
+### Adopter Types
+
+**End-user**: The organization runs Kamaji in production in some way.
+
+**Integration**: The organization has a product that integrates with Kamaji, but does not contain Kamaji.
+
+**Vendor**: The organization packages Kamaji in their product and sells it as part of their product.
+
+**R&D**: Company that exploring innovative technologies  and solutions for research and development purposes.

Dockerfile

@@ -1,5 +1,5 @@
 # Build the manager binary
-FROM golang:1.19 as builder
+FROM golang:1.22 as builder
 
 WORKDIR /workspace
 # Copy the Go Modules manifests

Makefile

@@ -3,7 +3,7 @@
 # To re-generate a bundle for another specific version without changing the standard setup, you can:
 # - use the VERSION as arg of the bundle target (e.g make bundle VERSION=0.0.2)
 # - use environment variables to overwrite this value (e.g export VERSION=0.0.2)
-VERSION ?= 0.1.1
+VERSION ?= 1.0.0
 
 # CHANNELS define the bundle channels used in the bundle.
 # Add a new line here if you would like to change its default config. (E.g CHANNELS = "candidate,fast,stable")
@@ -85,11 +85,11 @@ kind: ## Download kind locally if necessary.
 
 CONTROLLER_GEN = $(shell pwd)/bin/controller-gen
 controller-gen: ## Download controller-gen locally if necessary.
-	$(call go-install-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.9.2)
+	$(call go-install-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.14.0)
 
 GOLANGCI_LINT = $(shell pwd)/bin/golangci-lint
 golangci-lint: ## Download golangci-lint locally if necessary.
-	$(call go-install-tool,$(GOLANGCI_LINT),github.com/golangci/golangci-lint/cmd/golangci-lint@v1.49.0)
+	$(call go-install-tool,$(GOLANGCI_LINT),github.com/golangci/golangci-lint/cmd/golangci-lint@v1.54.2)
 
 KUSTOMIZE = $(shell pwd)/bin/kustomize
 kustomize: ## Download kustomize locally if necessary.
@@ -132,7 +132,11 @@ datastore-postgres:
 	$(MAKE) NAME=gold _datastore-postgres
 
 _datastore-etcd:
-	$(HELM) upgrade --install etcd-$(NAME) clastix/kamaji-etcd --create-namespace -n etcd-system --set datastore.enabled=true
+	$(HELM) upgrade --install etcd-$(NAME) clastix/kamaji-etcd --create-namespace -n etcd-system --set datastore.enabled=true --set fullnameOverride=etcd-$(NAME)
+
+_datastore-nats:
+	$(MAKE) NAME=$(NAME) NAMESPACE=nats-system -C deploy/kine/nats nats
+	kubectl apply -f $(shell pwd)/config/samples/kamaji_v1alpha1_datastore_nats_$(NAME).yaml
 
 datastore-etcd: helm
 	$(HELM) repo add clastix https://clastix.github.io/charts
@@ -141,7 +145,15 @@ datastore-etcd: helm
 	$(MAKE) NAME=silver _datastore-etcd
 	$(MAKE) NAME=gold _datastore-etcd
 
-datastores: datastore-mysql datastore-etcd datastore-postgres ## Install all Kamaji DataStores with multiple drivers, and different tiers.
+datastore-nats: helm
+	$(HELM) repo add nats https://nats-io.github.io/k8s/helm/charts/
+	$(HELM) repo update
+	$(MAKE) NAME=bronze _datastore-nats
+	$(MAKE) NAME=silver _datastore-nats
+	$(MAKE) NAME=gold _datastore-nats
+	$(MAKE) NAME=notls _datastore-nats
+
+datastores: datastore-mysql datastore-etcd datastore-postgres datastore-nats ## Install all Kamaji DataStores with multiple drivers, and different tiers.
 
 ##@ Build
 
@@ -154,6 +166,10 @@ GIT_MODIFIED    ?= $$(echo "$(GIT_MODIFIED_1)$(GIT_MODIFIED_2)")
 GIT_REPO        ?= $$(git config --get remote.origin.url)
 BUILD_DATE      ?= $$(git log -1 --format="%at" | xargs -I{} date -d @{} +%Y-%m-%dT%H:%M:%S)
 
+.PHONY: get_version
+get_version:
+	@echo -n v$(VERSION)
+
 build: generate fmt vet ## Build manager binary.
 	go build -o bin/manager main.go
 
@@ -165,7 +181,7 @@ docker-build: ## Build docker image with the manager.
 		--build-arg GIT_TAG_COMMIT=$(GIT_TAG_COMMIT) \
 		--build-arg GIT_MODIFIED=$(GIT_MODIFIED) \
 		--build-arg GIT_REPO=$(GIT_REPO) \
-		--build-arg GIT_LAST_TAG=$(VERSION) \
+		--build-arg GIT_LAST_TAG=v$(VERSION) \
 		--build-arg BUILD_DATE=$(BUILD_DATE)
 
 docker-push: ## Push docker image with the manager.
@@ -291,7 +307,7 @@ env:
 
 .PHONY: e2e
 e2e: env load helm ginkgo cert-manager ## Create a KinD cluster, install Kamaji on it and run the test suite.
-	$(HELM) upgrade --debug --install kamaji ./charts/kamaji --create-namespace --namespace kamaji-system --set "image.pullPolicy=Never"
+	$(HELM) upgrade --debug --install kamaji ./charts/kamaji --create-namespace --namespace kamaji-system --set "image.pullPolicy=Never" --set "telemetry.disabled=true"
 	$(MAKE) datastores
 	$(GINKGO) -v ./e2e
 

PROJECT

@@ -16,10 +16,6 @@ resources:
   kind: TenantControlPlane
   path: github.com/clastix/kamaji/api/v1alpha1
   version: v1alpha1
-  webhooks:
-    defaulting: true
-    validation: true
-    webhookVersion: v1
 - api:
     crdVersion: v1
   domain: clastix.io
@@ -27,8 +23,4 @@ resources:
   kind: DataStore
   path: github.com/clastix/kamaji/api/v1alpha1
   version: v1alpha1
-  webhooks:
-    defaulting: true
-    validation: true
-    webhookVersion: v1
 version: "3"

README.md

@@ -3,63 +3,158 @@
 <p align="left">
   <img src="https://img.shields.io/github/license/clastix/kamaji"/>
   <img src="https://img.shields.io/github/go-mod/go-version/clastix/kamaji"/>
-  <a href="https://github.com/clastix/kamaji/releases">
-    <img src="https://img.shields.io/github/v/release/clastix/kamaji"/>
-  </a>
+  <a href="https://github.com/clastix/kamaji/releases"><img src="https://img.shields.io/github/v/release/clastix/kamaji"/></a>
+  <img src="https://goreportcard.com/badge/github.com/clastix/kamaji">
+  <a href="https://kubernetes.slack.com/archives/C03GLTTMWNN"><img alt="#kamaji on Kubernetes Slack" src="https://img.shields.io/badge/slack-@kubernetes/kamaji-blue.svg?logo=slack"/></a>
 </p>
 
-**Kamaji** deploys and operates **Kubernetes** at scale with a fraction of the operational burden.
+![Logo](assets/logo-black.png#gh-light-mode-only)
+![Logo](assets/logo-white.png#gh-dark-mode-only)
 
-<p align="center" style="padding: 6px 6px">
-  <img src="assets/kamaji-logo.png" />
-</p>
+### 🤔 What is Kamaji?
 
-## Why we are building it?
-Global hyper-scalers are leading the Managed Kubernetes space, while other cloud providers, as well as large corporations, are struggling to offer the same experience to their DevOps teams because of the lack of the right tools. Also, current Kubernetes solutions are mainly designed with an enterprise-first approach and they are too costly when deployed at scale.
+**Kamaji** is a **Kubernetes Control Plane Manager** leveraging on the concept of [**Hosted Control Plane**](https://clastix.io/post/the-raise-of-hosted-control-plane-in-kubernetes/).
 
-**Kamaji** aims to solve these pains by leveraging multi-tenancy and simplifying how to run multiple control planes on the same infrastructure with a fraction of the operational burden.
+Kamaji's approach is based on running the Kubernetes Control Plane components in Pods instead of dedicated machines.
+This allows operating Kubernetes clusters at scale, with a fraction of the operational burden.
+Thanks to this approach, running multiple Control Planes can be cheaper and easier to deploy and operate.
 
-## How it works
-Kamaji turns any Kubernetes cluster into an _“admin cluster”_ to orchestrate other Kubernetes clusters called _“tenant clusters”_. What makes Kamaji special is that Control Planes of _“tenant clusters”_ are just regular pods running in the _“admin cluster”_ instead of dedicated Virtual Machines. This solution makes running control planes at scale cheaper and easier to deploy and operate.
+_Kamaji is like a fleet of Site Reliability Engineers with expertise codified into its logic, working 24/7 to keep up and running your Control Planes._
 
-![Architecture](docs/content/images/kamaji-light.png#gh-light-mode-only)
-![Architecture](docs/content/images/kamaji-dark.png#gh-dark-mode-only)
+<img src="docs/content/images/architecture.png"  width="600" style="display: block; margin: 0 auto">
 
-## Getting started
+### 📖 How it works
 
-Please refer to the [Getting Started guide](https://kamaji.clastix.io/getting-started/) to deploy a minimal setup of Kamaji on KinD.
+Kamaji is extending the Kubernetes API capabilities thanks to [Custom Resource Definitions](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/#customresourcedefinitions).
 
-## Features
+By installing Kamaji, two pairs of new APIs will be available:
 
-- **Self Service Kubernetes:** leave users the freedom to self-provision their Kubernetes clusters according to the assigned boundaries.
-- **Multi-cluster Management:** centrally manage multiple tenant clusters from a single admin cluster. Happy SREs. 
-- **Cheaper Control Planes:** place multiple tenant control planes on a single node, instead of having three nodes for a single control plane.
-- **Stronger Multi-Tenancy:** leave tenants to access the control plane with admin permissions while keeping the tenant isolated at the infrastructure level.
-- **Kubernetes Inception:** use Kubernetes to manage Kubernetes by re-using all the Kubernetes goodies you already know and love.
-- **Full APIs compliant:** tenant clusters are fully CNCF compliant built with upstream Kubernetes binaries. A user does not see differences between a Kamaji provisioned cluster and a dedicated cluster.
+- `TenantControlPlane`, the instance definition of your desired Kubernetes Control Plane
+- `Datastore`, the backing store used by one (or more) `TenantControlPlane`
 
-## Roadmap
+The `TenantControlPlane` (short-named as `tcp`) objects are Namespace-scoped and allows configuring every aspect of your desired Control Plane.
+Besides the Kubernetes configuration values, you can specify the Pod options such as limit, request, tolerations, node selector, etc.,
+as well as how these should be exposed (e.g.: using a `ClusterIP`, a `LoadBalancer`, or a `NodePort`).
 
-- [ ] Benchmarking and stress-test
-- [x] Support for dynamic address allocation on native Load Balancer
+The `TenantControlPlane` is the stateless definition of the Control Plane allowing to set up the required components for a full-fledged Kubernetest cluster.
+The state is managed by the `Datastore` API, a cluster-scoped resource which can hold the data of one or more Kubernetes clusters.
+
+> For further information about the API specifications and all the available options,
+> refer to the official [API reference](https://kamaji.clastix.io/reference/api/#tenantcontrolplane).
+
+### ⭐️ Main features
+
+- **Fast provisioning time**: depending on the infrastructure, Tenant Control Planes are up and ready to serve traffic in **16 seconds**.
+- **Streamlined update**: the rollout to a new Kubernetes version for a given Tenant Control Plane takes just **10 seconds**, with a Blue/Green deployment to avoid serving mixed Kubernetes versions.
+- **Resource optimization**: thanks to the Datastore decoupling, there's no need of odd number instances (e.g.: RAFT consensus) by allowing to save up to 60% of HW resources.
+- **Scale from zero to the moon**: scale down the instance when there's no usage, or automatically scale to support the traffic spikes reusing the Kubernetes patterns.
+- **Declarative approach, constant reconciliation**: thanks to the Operator pattern, drift detection happens in real-time, maintaining the desired state.
+- **Automated certificates management**: Kamaji leverages on `kubeadm` and the certificates are automatically created and rotated for you.
+- **Managing core addons**: Kamaji allows configuring automatically `kube-proxy`, `CoreDNS`, and `konnectivity`, with automatic remediation in case of user errors (e.g.: deleting the `CoreDNS` deployment).
+- **Auto Healing**: the `TenantControlPlane` objects in the management cluster are tracked by Kamaji, in case of deletion of those, everything is created in an idempotent way.
+- **Datastore multi-tenancy**: optionally, Kamaji allows running multiple Control Planes on the same _Datastore_ instance leveraging on the multi-tenancy of each driver, decreasing operations and optimizing costs.
+- **Overcoming `etcd` limitations**: optionally, Kamaji allows using a different _Datastore_ thanks to [`kine`](https://github.com/k3s-io/kine) by supporting `MySQL`, `PostgreSQL`, or `NATS` as an alternative.
+- **Simplifying mixed-networks setup**: thanks to [`Konnectivity`](https://kubernetes.io/docs/tasks/extend-kubernetes/setup-konnectivity/),
+  the Tenant Control Plane is connected to the worker nodes hosted in a different network, overcoming the no-NAT availability when dealing with nodes with a non routable IP address
+  (e.g.: worker nodes in a different infrastructure).
+
+### 🚀 Use cases
+
+- [**Creating a private Managed Kubernetes Service**](https://clastix.io/post/netsons-builds-a-managed-kubernetes-service-with-kamaji-and-open-stack/)
+- [**Building a Platform as a Service**](https://aenix.io/cozystack/)
+- [**Overcoming public Managed Kubernetes Services**](https://clastix.io/post/overcoming-eks-limitations-with-kamaji-on-aws/) such as EKS
+- [**Hybrid infrastructures**](https://clastix.io/post/bridging-the-gap-hybrid-kubernetes-clusters-with-remote-control-planes/):
+  host the Control Plane on the Cloud and worker nodes on prem or vice-versa, according to your needs.
+- [**Kubernetes at the edge**](https://clastix.io/post/edgevolution-unleashing-the-power-of-kubernetes-clusters-for-a-revolutionary-edge-computing-experience/):
+  take full advantage of the _Kubernetes API Server as a service_ paradigm.
+- **Kubernetes Control Plane as a Service:** centrally manage multiple Kubernetes clusters from a single management point (_Multi-Cluster management_). 
+- **High-density Control Plane:** place multiple control planes on the same infrastructure, instead of having dedicated machines for each control plane.
+- **Strong Multi-tenancy:** leave users to access the control plane with admin permissions while keeping them isolated at the infrastructure level.
+- **Kubernetes Inception:** use Kubernetes to manage Kubernetes with automation, high-availability, fault tolerance, and autoscaling out of the box. 
+- **Bring Your Own Device:** keep the control plane isolated from data plane. Worker nodes can join and run consistently from everywhere: cloud, edge, and data-center.
+- **Full CNCF compliant:** all clusters are built with upstream Kubernetes binaries, resulting in full CNCF compliant Kubernetes clusters.
+
+> 🤔 You'd like to do the same but don't know how?
+> 💡 [CLASTIX](https://clastix.io/) can help you with your needs!
+
+### 🧑‍💻‍ Production grade
+
+Kamaji is empowering several businesses, and it counts public adopters.
+Check out the [adopters](./ADOPTERS.md) file to learn more.
+
+> 🤗 If you're using Kamaji, share your love by opening a PR!
+
+### 🍦 Vanilla Kubernetes clusters
+
+Kamaji is **not** yet-another-Kubernetes distribution: you have full freedom on the technology stack to provide to end users.
+Kamaji is a perfect fit for Platform Engineering, hiding the complexity of the Control Plane management to developers and DevOps engineers.
+
+The provided Kubernetes Control Planes are [CNCF compliant clusters](https://kamaji.clastix.io/reference/conformance/).
+
+<img src="https://raw.githubusercontent.com/cncf/artwork/master/projects/kubernetes/certified-kubernetes/versionless/color/certified-kubernetes-color.png" style="display: block; width: 75px; margin: 0 auto">
+
+### 🐢 Cluster API support
+
+Kamaji is **not** a [Cluster API](https://cluster-api.sigs.k8s.io/) replacement, rather, it plays very well with it.
+
+Since Kamaji is just focusing on the Control Plane a [Kamaji's Cluster API Control Plane provider](https://github.com/clastix/cluster-api-control-plane-provider-kamaji) has been developed.
+
+### 🛣️ Roadmap
+
+- [x] Dynamic address on Load Balancer
 - [x] Zero Downtime Tenant Control Plane upgrade
-- [x] `konnectivity` integration
-- [ ] Provisioning of Tenant Control Plane through Cluster APIs
+- [x] [Join worker nodes from anywhere thanks to Konnectivity](https://kamaji.clastix.io/concepts/#konnectivity)
+- [x] [Alternative datastore MySQL, PostgreSQL, NATS](https://kamaji.clastix.io/guides/alternative-datastore/)
+- [x] [Pool of multiple datastores](https://kamaji.clastix.io/concepts/#datastores)
+- [x] [Seamless migration between datastores](https://kamaji.clastix.io/guides/datastore-migration/)
+- [ ] Automatic assignment to a datastore
+- [ ] Autoscaling of Tenant Control Plane
+- [x] [Provisioning through Cluster APIs](https://github.com/clastix/cluster-api-control-plane-provider-kamaji)
 - [ ] Terraform provider
-- [ ] Custom Prometheus metrics for monitoring and alerting
-- [x] `kine` integration for MySQL as datastore
-- [x] `kine` integration for PostgreSQL as datastore
-- [x] Pool of multiple datastores
-- [x] Seamless migration between datastore with the same driver
-- [ ] Automatic assigning of Tenant Control Plane to a datastore
-- [ ] Autoscaling of Tenant Control Plane pods
+- [ ] Custom Prometheus metrics
 
+### 🎥 Multimedia
 
-## Documentation
-Please, check the project's [documentation](https://kamaji.clastix.io/) for getting started with Kamaji.
+- Playlist ▶️ [Tutorials and How-Tos by Dario Tranchitella, CLASTIX](https://www.youtube.com/playlist?list=PLjiUjoV4Ws_3pNsUpTXI-KKk731nD2MQY)
+- YouTube ▶️ [Metal³ provisioning with Kamaji Hosted Control Planes by Huy Mai, Ericsson](https://youtu.be/u9sbURj6jXY?t=10536)
+- YouTube ▶️ [Hands-on introduction to Kamaji](https://www.youtube.com/watch?v=HhevxwQWQ88)
+- YouTube ▶️ [Scaling Kubernetes up to 1,000 Control Planes](https://www.youtube.com/watch?v=W_HXRXJh96U)
+- YouTube ▶️ [Equinix, Kamaji, and Cluster API](https://www.youtube.com/watch?v=TLBTqROj_wA)
+- YouTube ▶️ [Rancher & Kamaji: solving multitenancy challenges in the Kubernetes world](https://www.youtube.com/watch?v=VXHNrMmlF8U)
+- YouTube ▶️ [Enabling Self-Service Kubernetes clusters with Kamaji and Paralus](https://www.youtube.com/watch?v=JWA2LwZazM0)
 
-## Contributions
-Kamaji is Open Source with Apache 2 license and any contribution is welcome.
+### 🏷️ Versioning
 
-## Community
-Join the [Kubernetes Slack Workspace](https://slack.k8s.io/) and the [`#kamaji`](https://kubernetes.slack.com/archives/C03GLTTMWNN) channel to meet end-users and contributors.
+Versioning adheres to the [Semantic Versioning](http://semver.org/) principles.
+A full list of the available releases is available in the GitHub repository's [**Release** section](https://github.com/clastix/kamaji/releases).
+
+### 📄 Documentation
+
+Further documentation can be found on the official [Kamaji documentation website](https://kamaji.clastix.io/).
+
+### 🤝 Contributions
+
+Contributions are highly appreciated and very welcomed!
+
+In case of bugs, please, check if the issue has been already opened by checking the [GitHub Issues](https://github.com/clastix/kamaji/issues) section.
+In case it isn't, you can open a new one: a detailed report will help us to replicate it, assess it, and work on a fix.
+
+You can express your intention in working on the fix on your own.
+The commit messages are checked according to the described [semantics](https://github.com/projectcapsule/capsule/blob/main/CONTRIBUTING.md#semantics).
+Commits are used to generate the changelog, and their author will be referenced in it.
+
+In case of **✨ Feature Requests** please use the [Discussion's Feature Request section](https://github.com/clastix/kamaji/discussions/categories/feature-requests).
+
+### 📝 License
+
+The Kamaji Cluster API Control Plane provider is licensed under Apache 2.0.
+The code is provided as-is with no warranties.
+
+### 🛟 Commercial Support
+
+![CLASTIX](https://avatars.githubusercontent.com/u/39170129?s=50&v=4) [CLASTIX](https://clastix.io/) is the commercial company behind Kamaji and the Cluster API Control Plane provider.
+
+If you're looking to run Kamaji in production and would like to learn more, **CLASTIX** can help by offering [Open Source support plans](https://clastix.io/support),
+as well as providing a comprehensive Enterprise Platform named [CLASTIX Enterprise Platform](https://clastix.cloud/), built on top of the Kamaji and [Capsule](https://capsule.clastix.io/) project (now donated to CNCF as a Sandbox project).
+
+Feel free to get in touch with the provided [Contact form](https://clastix.io/contact).

api/v1alpha1/datastore_secret_webhook.go

@@ -1,57 +0,0 @@
-// Copyright 2022 Clastix Labs
-// SPDX-License-Identifier: Apache-2.0
-
-package v1alpha1
-
-import (
-	"context"
-	"fmt"
-	"strings"
-
-	"github.com/go-logr/logr"
-	corev1 "k8s.io/api/core/v1"
-	"k8s.io/apimachinery/pkg/fields"
-	"k8s.io/apimachinery/pkg/runtime"
-	"sigs.k8s.io/controller-runtime/pkg/client"
-)
-
-//+kubebuilder:webhook:path=/validate--v1-secret,mutating=false,failurePolicy=ignore,sideEffects=None,groups="",resources=secrets,verbs=delete,versions=v1,name=vdatastoresecrets.kb.io,admissionReviewVersions=v1
-
-type dataStoreSecretValidator struct {
-	log    logr.Logger
-	client client.Client
-}
-
-func (d *dataStoreSecretValidator) ValidateCreate(context.Context, runtime.Object) error {
-	return nil
-}
-
-func (d *dataStoreSecretValidator) ValidateUpdate(context.Context, runtime.Object, runtime.Object) error {
-	return nil
-}
-
-func (d *dataStoreSecretValidator) ValidateDelete(ctx context.Context, obj runtime.Object) error {
-	secret := obj.(*corev1.Secret) //nolint:forcetypeassert
-
-	dsList := &DataStoreList{}
-
-	if err := d.client.List(ctx, dsList, client.MatchingFieldsSelector{Selector: fields.OneTermEqualSelector(DatastoreUsedSecretNamespacedNameKey, fmt.Sprintf("%s/%s", secret.GetNamespace(), secret.GetName()))}); err != nil {
-		return err
-	}
-
-	if len(dsList.Items) > 0 {
-		var res []string
-
-		for _, ds := range dsList.Items {
-			res = append(res, ds.GetName())
-		}
-
-		return fmt.Errorf("the Secret is used by the following kamajiv1alpha1.DataStores and cannot be deleted (%s)", strings.Join(res, ", "))
-	}
-
-	return nil
-}
-
-func (d *dataStoreSecretValidator) Default(context.Context, runtime.Object) error {
-	return nil
-}

api/v1alpha1/datastore_types.go

@@ -8,7 +8,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
-// +kubebuilder:validation:Enum=etcd;MySQL;PostgreSQL
+// +kubebuilder:validation:Enum=etcd;MySQL;PostgreSQL;NATS
 
 type Driver string
 
@@ -16,6 +16,7 @@ var (
 	EtcdDriver           Driver = "etcd"
 	KineMySQLDriver      Driver = "MySQL"
 	KinePostgreSQLDriver Driver = "PostgreSQL"
+	KineNatsDriver       Driver = "NATS"
 )
 
 // +kubebuilder:validation:MinItems=1
@@ -33,7 +34,8 @@ type DataStoreSpec struct {
 	// This value is optional.
 	BasicAuth *BasicAuth `json:"basicAuth,omitempty"`
 	// Defines the TLS/SSL configuration required to connect to the data store in a secure way.
-	TLSConfig TLSConfig `json:"tlsConfig"`
+	// This value is optional.
+	TLSConfig *TLSConfig `json:"tlsConfig,omitempty"`
 }
 
 // TLSConfig contains the information used to connect to the data store using a secured connection.
@@ -42,7 +44,7 @@ type TLSConfig struct {
 	// The key reference is required since etcd authentication is based on certificates, and Kamaji is responsible in creating this.
 	CertificateAuthority CertKeyPair `json:"certificateAuthority"`
 	// Specifies the SSL/TLS key and private key pair used to connect to the data store.
-	ClientCertificate ClientCertificate `json:"clientCertificate"`
+	ClientCertificate *ClientCertificate `json:"clientCertificate,omitempty"`
 }
 
 type ClientCertificate struct {

api/v1alpha1/datastore_webhook.go

@@ -1,185 +0,0 @@
-// Copyright 2022 Clastix Labs
-// SPDX-License-Identifier: Apache-2.0
-
-package v1alpha1
-
-import (
-	"context"
-	"fmt"
-
-	"github.com/go-logr/logr"
-	corev1 "k8s.io/api/core/v1"
-	"k8s.io/apimachinery/pkg/api/errors"
-	"k8s.io/apimachinery/pkg/fields"
-	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/apimachinery/pkg/types"
-	ctrl "sigs.k8s.io/controller-runtime"
-	"sigs.k8s.io/controller-runtime/pkg/client"
-)
-
-//+kubebuilder:webhook:path=/mutate-kamaji-clastix-io-v1alpha1-datastore,mutating=true,failurePolicy=fail,sideEffects=None,groups=kamaji.clastix.io,resources=datastores,verbs=create;update,versions=v1alpha1,name=mdatastore.kb.io,admissionReviewVersions=v1
-//+kubebuilder:webhook:path=/validate-kamaji-clastix-io-v1alpha1-datastore,mutating=false,failurePolicy=fail,sideEffects=None,groups=kamaji.clastix.io,resources=datastores,verbs=create;update;delete,versions=v1alpha1,name=vdatastore.kb.io,admissionReviewVersions=v1
-
-func (in *DataStore) SetupWebhookWithManager(mgr ctrl.Manager) error {
-	secretValidator := &dataStoreSecretValidator{
-		log:    mgr.GetLogger().WithName("datastore-secret-webhook"),
-		client: mgr.GetClient(),
-	}
-
-	if err := ctrl.NewWebhookManagedBy(mgr).For(&corev1.Secret{}).WithValidator(secretValidator).Complete(); err != nil {
-		return err
-	}
-
-	dsValidator := &dataStoreValidator{
-		log:    mgr.GetLogger().WithName("datastore-webhook"),
-		client: mgr.GetClient(),
-	}
-
-	return ctrl.NewWebhookManagedBy(mgr).
-		For(in).
-		WithValidator(dsValidator).
-		WithDefaulter(dsValidator).
-		Complete()
-}
-
-type dataStoreValidator struct {
-	log    logr.Logger
-	client client.Client
-}
-
-func (d *dataStoreValidator) ValidateCreate(ctx context.Context, obj runtime.Object) error {
-	ds, ok := obj.(*DataStore)
-	if !ok {
-		return fmt.Errorf("expected *kamajiv1alpha1.DataStore")
-	}
-
-	if err := d.validate(ctx, ds); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (d *dataStoreValidator) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) error {
-	old, ok := oldObj.(*DataStore)
-	if !ok {
-		return fmt.Errorf("expected *kamajiv1alpha1.DataStore")
-	}
-
-	ds, ok := newObj.(*DataStore)
-	if !ok {
-		return fmt.Errorf("expected *kamajiv1alpha1.DataStore")
-	}
-
-	d.log.Info("validate update", "name", ds.GetName())
-
-	if ds.Spec.Driver != old.Spec.Driver {
-		return fmt.Errorf("driver of a DataStore cannot be changed")
-	}
-
-	if err := d.validate(ctx, ds); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (d *dataStoreValidator) ValidateDelete(ctx context.Context, obj runtime.Object) error {
-	ds, ok := obj.(*DataStore)
-	if !ok {
-		return fmt.Errorf("expected *kamajiv1alpha1.DataStore")
-	}
-
-	tcpList := &TenantControlPlaneList{}
-
-	if err := d.client.List(ctx, tcpList, client.MatchingFieldsSelector{Selector: fields.OneTermEqualSelector(TenantControlPlaneUsedDataStoreKey, ds.GetName())}); err != nil {
-		return err
-	}
-
-	if len(tcpList.Items) > 0 {
-		return fmt.Errorf("the DataStore is used by multiple TenantControlPlanes and cannot be removed")
-	}
-
-	return nil
-}
-
-func (d *dataStoreValidator) Default(context.Context, runtime.Object) error {
-	return nil
-}
-
-func (d *dataStoreValidator) validate(ctx context.Context, ds *DataStore) error {
-	if ds.Spec.BasicAuth != nil {
-		if err := d.validateBasicAuth(ctx, ds); err != nil {
-			return err
-		}
-	}
-
-	if err := d.validateTLSConfig(ctx, ds); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (d *dataStoreValidator) validateBasicAuth(ctx context.Context, ds *DataStore) error {
-	if err := d.validateContentReference(ctx, ds.Spec.BasicAuth.Password); err != nil {
-		return fmt.Errorf("basic-auth password is not valid, %w", err)
-	}
-
-	if err := d.validateContentReference(ctx, ds.Spec.BasicAuth.Username); err != nil {
-		return fmt.Errorf("basic-auth username is not valid, %w", err)
-	}
-
-	return nil
-}
-
-func (d *dataStoreValidator) validateTLSConfig(ctx context.Context, ds *DataStore) error {
-	if err := d.validateContentReference(ctx, ds.Spec.TLSConfig.CertificateAuthority.Certificate); err != nil {
-		return fmt.Errorf("CA certificate is not valid, %w", err)
-	}
-
-	if ds.Spec.Driver == EtcdDriver {
-		if ds.Spec.TLSConfig.CertificateAuthority.PrivateKey == nil {
-			return fmt.Errorf("CA private key is required when using the etcd driver")
-		}
-	}
-
-	if ds.Spec.TLSConfig.CertificateAuthority.PrivateKey != nil {
-		if err := d.validateContentReference(ctx, *ds.Spec.TLSConfig.CertificateAuthority.PrivateKey); err != nil {
-			return fmt.Errorf("CA private key is not valid, %w", err)
-		}
-	}
-
-	if err := d.validateContentReference(ctx, ds.Spec.TLSConfig.ClientCertificate.Certificate); err != nil {
-		return fmt.Errorf("client certificate is not valid, %w", err)
-	}
-
-	if err := d.validateContentReference(ctx, ds.Spec.TLSConfig.ClientCertificate.PrivateKey); err != nil {
-		return fmt.Errorf("client private key is not valid, %w", err)
-	}
-
-	return nil
-}
-
-func (d *dataStoreValidator) validateContentReference(ctx context.Context, ref ContentRef) error {
-	switch {
-	case len(ref.Content) > 0:
-		return nil
-	case ref.SecretRef == nil:
-		return fmt.Errorf("the Secret reference is mandatory when bare content is not specified")
-	case len(ref.SecretRef.SecretReference.Name) == 0:
-		return fmt.Errorf("the Secret reference name is mandatory")
-	case len(ref.SecretRef.SecretReference.Namespace) == 0:
-		return fmt.Errorf("the Secret reference namespace is mandatory")
-	}
-
-	if err := d.client.Get(ctx, types.NamespacedName{Name: ref.SecretRef.SecretReference.Name, Namespace: ref.SecretRef.SecretReference.Namespace}, &corev1.Secret{}); err != nil {
-		if errors.IsNotFound(err) {
-			return fmt.Errorf("secret %s/%s is not found", ref.SecretRef.SecretReference.Namespace, ref.SecretRef.SecretReference.Name)
-		}
-
-		return err
-	}
-
-	return nil
-}

api/v1alpha1/indexer_datastore_usedsecret.go

@@ -43,20 +43,22 @@ func (d *DatastoreUsedSecret) ExtractValue() client.IndexerFunc {
 			}
 		}
 
-		if ds.Spec.TLSConfig.CertificateAuthority.Certificate.SecretRef != nil {
-			res = append(res, d.namespacedName(*ds.Spec.TLSConfig.CertificateAuthority.Certificate.SecretRef))
-		}
+		if ds.Spec.TLSConfig != nil {
+			if ds.Spec.TLSConfig.CertificateAuthority.Certificate.SecretRef != nil {
+				res = append(res, d.namespacedName(*ds.Spec.TLSConfig.CertificateAuthority.Certificate.SecretRef))
+			}
 
-		if ds.Spec.TLSConfig.CertificateAuthority.PrivateKey != nil && ds.Spec.TLSConfig.CertificateAuthority.PrivateKey.SecretRef != nil {
-			res = append(res, d.namespacedName(*ds.Spec.TLSConfig.CertificateAuthority.PrivateKey.SecretRef))
-		}
+			if ds.Spec.TLSConfig.CertificateAuthority.PrivateKey != nil && ds.Spec.TLSConfig.CertificateAuthority.PrivateKey.SecretRef != nil {
+				res = append(res, d.namespacedName(*ds.Spec.TLSConfig.CertificateAuthority.PrivateKey.SecretRef))
+			}
 
-		if ds.Spec.TLSConfig.ClientCertificate.Certificate.SecretRef != nil {
-			res = append(res, d.namespacedName(*ds.Spec.TLSConfig.ClientCertificate.Certificate.SecretRef))
-		}
+			if ds.Spec.TLSConfig.ClientCertificate.Certificate.SecretRef != nil {
+				res = append(res, d.namespacedName(*ds.Spec.TLSConfig.ClientCertificate.Certificate.SecretRef))
+			}
 
-		if ds.Spec.TLSConfig.ClientCertificate.PrivateKey.SecretRef != nil {
-			res = append(res, d.namespacedName(*ds.Spec.TLSConfig.ClientCertificate.PrivateKey.SecretRef))
+			if ds.Spec.TLSConfig.ClientCertificate.PrivateKey.SecretRef != nil {
+				res = append(res, d.namespacedName(*ds.Spec.TLSConfig.ClientCertificate.PrivateKey.SecretRef))
+			}
 		}
 
 		return res

api/v1alpha1/tenantcontrolplane_registrysettings.go

@@ -0,0 +1,18 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package v1alpha1
+
+type RegistrySettings struct {
+	// +kubebuilder:default="registry.k8s.io"
+	Registry string `json:"registry,omitempty"`
+	// The tag to append to all the Control Plane container images.
+	// Optional.
+	TagSuffix string `json:"tagSuffix,omitempty"`
+	// +kubebuilder:default="kube-apiserver"
+	APIServerImage string `json:"apiServerImage,omitempty"`
+	// +kubebuilder:default="kube-controller-manager"
+	ControllerManagerImage string `json:"controllerManagerImage,omitempty"`
+	// +kubebuilder:default="kube-scheduler"
+	SchedulerImage string `json:"schedulerImage,omitempty"`
+}

api/v1alpha1/tenantcontrolplane_registrysettings_funcs.go

@@ -0,0 +1,30 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package v1alpha1
+
+import (
+	"fmt"
+)
+
+func (r *RegistrySettings) buildContainerImage(name, tag string) string {
+	image := fmt.Sprintf("%s/%s:%s", r.Registry, name, tag)
+
+	if len(r.TagSuffix) > 0 {
+		image += r.TagSuffix
+	}
+
+	return image
+}
+
+func (r *RegistrySettings) KubeAPIServerImage(version string) string {
+	return r.buildContainerImage(r.APIServerImage, version)
+}
+
+func (r *RegistrySettings) KubeSchedulerImage(version string) string {
+	return r.buildContainerImage(r.SchedulerImage, version)
+}
+
+func (r *RegistrySettings) KubeControllerManagerImage(version string) string {
+	return r.buildContainerImage(r.ControllerManagerImage, version)
+}

api/v1alpha1/tenantcontrolplane_status.go

@@ -183,11 +183,12 @@ type KubernetesStatus struct {
 	Ingress    *KubernetesIngressStatus   `json:"ingress,omitempty"`
 }
 
-// +kubebuilder:validation:Enum=Provisioning;Upgrading;Migrating;Ready;NotReady
+// +kubebuilder:validation:Enum=Provisioning;CertificateAuthorityRotating;Upgrading;Migrating;Ready;NotReady
 type KubernetesVersionStatus string
 
 var (
 	VersionProvisioning KubernetesVersionStatus = "Provisioning"
+	VersionCARotating   KubernetesVersionStatus = "CertificateAuthorityRotating"
 	VersionUpgrading    KubernetesVersionStatus = "Upgrading"
 	VersionMigrating    KubernetesVersionStatus = "Migrating"
 	VersionReady        KubernetesVersionStatus = "Ready"

api/v1alpha1/tenantcontrolplane_types.go

@@ -4,6 +4,7 @@
 package v1alpha1
 
 import (
+	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
@@ -32,7 +33,23 @@ type NetworkProfileSpec struct {
 	DNSServiceIPs []string `json:"dnsServiceIPs,omitempty"`
 }
 
+// +kubebuilder:validation:Enum=Hostname;InternalIP;ExternalIP;InternalDNS;ExternalDNS
+type KubeletPreferredAddressType string
+
+const (
+	NodeHostName    KubeletPreferredAddressType = "Hostname"
+	NodeInternalIP  KubeletPreferredAddressType = "InternalIP"
+	NodeExternalIP  KubeletPreferredAddressType = "ExternalIP"
+	NodeInternalDNS KubeletPreferredAddressType = "InternalDNS"
+	NodeExternalDNS KubeletPreferredAddressType = "ExternalDNS"
+)
+
 type KubeletSpec struct {
+	// Ordered list of the preferred NodeAddressTypes to use for kubelet connections.
+	// Default to Hostname, InternalIP, ExternalIP.
+	// +kubebuilder:default={"Hostname","InternalIP","ExternalIP"}
+	// +kubebuilder:validation:MinItems=1
+	PreferredAddressTypes []KubeletPreferredAddressType `json:"preferredAddressTypes,omitempty"`
 	// CGroupFS defines the  cgroup driver for Kubelet
 	// https://kubernetes.io/docs/tasks/administer-cluster/kubeadm/configure-cgroup-driver/
 	CGroupFS CGroupDriver `json:"cgroupfs,omitempty"`
@@ -76,27 +93,22 @@ type IngressSpec struct {
 	Hostname string `json:"hostname,omitempty"`
 }
 
-// ComponentResourceRequirements describes the compute resource requirements.
-type ComponentResourceRequirements struct {
-	// Limits describes the maximum amount of compute resources allowed.
-	// More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
-	Limits corev1.ResourceList `json:"limits,omitempty" protobuf:"bytes,1,rep,name=limits,casttype=ResourceList,castkey=ResourceName"`
-	// Requests describes the minimum amount of compute resources required.
-	// If Requests is omitted for a container, it defaults to Limits if that is explicitly specified,
-	// otherwise to an implementation-defined value.
-	// More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
-	Requests corev1.ResourceList `json:"requests,omitempty" protobuf:"bytes,2,rep,name=requests,casttype=ResourceList,castkey=ResourceName"`
-}
-
 type ControlPlaneComponentsResources struct {
-	APIServer         *ComponentResourceRequirements `json:"apiServer,omitempty"`
-	ControllerManager *ComponentResourceRequirements `json:"controllerManager,omitempty"`
-	Scheduler         *ComponentResourceRequirements `json:"scheduler,omitempty"`
+	APIServer         *corev1.ResourceRequirements `json:"apiServer,omitempty"`
+	ControllerManager *corev1.ResourceRequirements `json:"controllerManager,omitempty"`
+	Scheduler         *corev1.ResourceRequirements `json:"scheduler,omitempty"`
+	// Define the kine container resources.
+	// Available only if Kamaji is running using Kine as backing storage.
+	Kine *corev1.ResourceRequirements `json:"kine,omitempty"`
 }
 
 type DeploymentSpec struct {
+	// RegistrySettings allows to override the default images for the given Tenant Control Plane instance.
+	// It could be used to point to a different container registry rather than the public one.
+	// +kubebuilder:default={registry:"registry.k8s.io",apiServerImage:"kube-apiserver",controllerManagerImage:"kube-controller-manager",schedulerImage:"kube-scheduler"}
+	RegistrySettings RegistrySettings `json:"registrySettings,omitempty"`
 	// +kubebuilder:default=2
-	Replicas int32 `json:"replicas,omitempty"`
+	Replicas *int32 `json:"replicas,omitempty"`
 	// NodeSelector is a selector which must be true for the pod to fit on a node.
 	// Selector which must match a node's labels for the pod to be scheduled on that node.
 	// More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/
@@ -107,6 +119,10 @@ type DeploymentSpec struct {
 	// empty definition that uses the default runtime handler.
 	// More info: https://git.k8s.io/enhancements/keps/sig-node/585-runtime-class
 	RuntimeClassName string `json:"runtimeClassName,omitempty"`
+	// Strategy describes how to replace existing pods with new ones for the given Tenant Control Plane.
+	// Default value is set to Rolling Update, with a blue/green strategy.
+	// +kubebuilder:default={type:"RollingUpdate",rollingUpdate:{maxUnavailable:0,maxSurge:"100%"}}
+	Strategy appsv1.DeploymentStrategy `json:"strategy,omitempty"`
 	// If specified, the Tenant Control Plane pod's tolerations.
 	// More info: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/
 	Tolerations []corev1.Toleration `json:"tolerations,omitempty"`
@@ -122,9 +138,31 @@ type DeploymentSpec struct {
 	// (kube-apiserver, controller-manager, and scheduler).
 	Resources *ControlPlaneComponentsResources `json:"resources,omitempty"`
 	// ExtraArgs allows adding additional arguments to the Control Plane components,
-	// such as kube-apiserver, controller-manager, and scheduler.
-	ExtraArgs          *ControlPlaneExtraArgs `json:"extraArgs,omitempty"`
-	AdditionalMetadata AdditionalMetadata     `json:"additionalMetadata,omitempty"`
+	// such as kube-apiserver, controller-manager, and scheduler. WARNING - This option
+	// can override existing parameters and cause components to misbehave in unxpected ways.
+	// Only modify if you know what you are doing.
+	ExtraArgs             *ControlPlaneExtraArgs `json:"extraArgs,omitempty"`
+	AdditionalMetadata    AdditionalMetadata     `json:"additionalMetadata,omitempty"`
+	PodAdditionalMetadata AdditionalMetadata     `json:"podAdditionalMetadata,omitempty"`
+	// AdditionalInitContainers allows adding additional init containers to the Control Plane deployment.
+	AdditionalInitContainers []corev1.Container `json:"additionalInitContainers,omitempty"`
+	// AdditionalContainers allows adding additional containers to the Control Plane deployment.
+	AdditionalContainers []corev1.Container `json:"additionalContainers,omitempty"`
+	// AdditionalVolumes allows to add additional volumes to the Control Plane deployment.
+	AdditionalVolumes []corev1.Volume `json:"additionalVolumes,omitempty"`
+	// AdditionalVolumeMounts allows to mount an additional volume into each component of the Control Plane
+	// (kube-apiserver, controller-manager, and scheduler).
+	AdditionalVolumeMounts *AdditionalVolumeMounts `json:"additionalVolumeMounts,omitempty"`
+	// +kubebuilder:default="default"
+	// ServiceAccountName allows to specify the service account to be mounted to the pods of the Control plane deployment
+	ServiceAccountName string `json:"serviceAccountName,omitempty"`
+}
+
+// AdditionalVolumeMounts allows mounting additional volumes to the Control Plane components.
+type AdditionalVolumeMounts struct {
+	APIServer         []corev1.VolumeMount `json:"apiServer,omitempty"`
+	ControllerManager []corev1.VolumeMount `json:"controllerManager,omitempty"`
+	Scheduler         []corev1.VolumeMount `json:"scheduler,omitempty"`
 }
 
 // ControlPlaneExtraArgs allows specifying additional arguments to the Control Plane components.
@@ -157,6 +195,9 @@ type ImageOverrideTrait struct {
 }
 
 // ExtraArgs allows adding additional arguments to said component.
+// WARNING - This option can override existing konnectivity
+// parameters and cause konnectivity components to misbehave in
+// unxpected ways. Only modify if you know what you are doing.
 type ExtraArgs []string
 
 type KonnectivityServerSpec struct {
@@ -169,8 +210,8 @@ type KonnectivityServerSpec struct {
 	// +kubebuilder:default=registry.k8s.io/kas-network-proxy/proxy-server
 	Image string `json:"image,omitempty"`
 	// Resources define the amount of CPU and memory to allocate to the Konnectivity server.
-	Resources *ComponentResourceRequirements `json:"resources,omitempty"`
-	ExtraArgs ExtraArgs                      `json:"extraArgs,omitempty"`
+	Resources *corev1.ResourceRequirements `json:"resources,omitempty"`
+	ExtraArgs ExtraArgs                    `json:"extraArgs,omitempty"`
 }
 
 type KonnectivityAgentSpec struct {
@@ -179,8 +220,12 @@ type KonnectivityAgentSpec struct {
 	Image string `json:"image,omitempty"`
 	// Version for Konnectivity agent.
 	// +kubebuilder:default=v0.0.32
-	Version   string    `json:"version,omitempty"`
-	ExtraArgs ExtraArgs `json:"extraArgs,omitempty"`
+	Version string `json:"version,omitempty"`
+	// Tolerations for the deployed agent.
+	// Can be customized to start the konnectivity-agent even if the nodes are not ready or tainted.
+	// +kubebuilder:default={{key: "CriticalAddonsOnly", operator: "Exists"}}
+	Tolerations []corev1.Toleration `json:"tolerations,omitempty"`
+	ExtraArgs   ExtraArgs           `json:"extraArgs,omitempty"`
 }
 
 // KonnectivitySpec defines the spec for Konnectivity.
@@ -221,7 +266,7 @@ type TenantControlPlaneSpec struct {
 // +kubebuilder:object:root=true
 // +kubebuilder:subresource:status
 // +kubebuilder:subresource:scale:specpath=.spec.controlPlane.deployment.replicas,statuspath=.status.kubernetesResources.deployment.replicas,selectorpath=.status.kubernetesResources.deployment.selector
-// +kubebuilder:resource:shortName=tcp
+// +kubebuilder:resource:categories=kamaji,shortName=tcp
 // +kubebuilder:printcolumn:name="Version",type="string",JSONPath=".spec.kubernetes.version",description="Kubernetes version"
 // +kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.kubernetesResources.version.status",description="Status"
 // +kubebuilder:printcolumn:name="Control-Plane endpoint",type="string",JSONPath=".status.controlPlaneEndpoint",description="Tenant Control Plane Endpoint (API server)"

api/v1alpha1/tenantcontrolplane_webhook.go

@@ -1,166 +0,0 @@
-// Copyright 2022 Clastix Labs
-// SPDX-License-Identifier: Apache-2.0
-
-package v1alpha1
-
-import (
-	"context"
-	"fmt"
-	"strings"
-
-	"github.com/blang/semver"
-	"github.com/go-logr/logr"
-	"github.com/pkg/errors"
-	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/apimachinery/pkg/types"
-	ctrl "sigs.k8s.io/controller-runtime"
-	"sigs.k8s.io/controller-runtime/pkg/client"
-
-	"github.com/clastix/kamaji/internal/upgrade"
-)
-
-//+kubebuilder:webhook:path=/mutate-kamaji-clastix-io-v1alpha1-tenantcontrolplane,mutating=true,failurePolicy=fail,sideEffects=None,groups=kamaji.clastix.io,resources=tenantcontrolplanes,verbs=create;update,versions=v1alpha1,name=mtenantcontrolplane.kb.io,admissionReviewVersions=v1
-//+kubebuilder:webhook:path=/validate-kamaji-clastix-io-v1alpha1-tenantcontrolplane,mutating=false,failurePolicy=fail,sideEffects=None,groups=kamaji.clastix.io,resources=tenantcontrolplanes,verbs=create;update,versions=v1alpha1,name=vtenantcontrolplane.kb.io,admissionReviewVersions=v1
-
-func (in *TenantControlPlane) SetupWebhookWithManager(mgr ctrl.Manager, datastore string) error {
-	validator := &tenantControlPlaneValidator{
-		client:           mgr.GetClient(),
-		defaultDatastore: datastore,
-		log:              mgr.GetLogger().WithName("tenantcontrolplane-webhook"),
-	}
-
-	return ctrl.NewWebhookManagedBy(mgr).
-		For(in).
-		WithValidator(validator).
-		WithDefaulter(validator).
-		Complete()
-}
-
-type tenantControlPlaneValidator struct {
-	client           client.Client
-	defaultDatastore string
-	log              logr.Logger
-}
-
-func (t *tenantControlPlaneValidator) Default(_ context.Context, obj runtime.Object) error {
-	tcp, ok := obj.(*TenantControlPlane)
-	if !ok {
-		return fmt.Errorf("expected *kamajiv1alpha1.TenantControlPlane")
-	}
-
-	if len(tcp.Spec.DataStore) == 0 {
-		tcp.Spec.DataStore = t.defaultDatastore
-	}
-
-	return nil
-}
-
-func (t *tenantControlPlaneValidator) ValidateCreate(_ context.Context, obj runtime.Object) error {
-	tcp, ok := obj.(*TenantControlPlane)
-	if !ok {
-		return fmt.Errorf("expected *kamajiv1alpha1.TenantControlPlane")
-	}
-
-	t.log.Info("validate create", "name", tcp.Name, "namespace", tcp.Namespace)
-
-	ver, err := semver.New(t.normalizeKubernetesVersion(tcp.Spec.Kubernetes.Version))
-	if err != nil {
-		return errors.Wrap(err, "unable to parse the desired Kubernetes version")
-	}
-
-	supportedVer, supportedErr := semver.Make(t.normalizeKubernetesVersion(upgrade.KubeadmVersion))
-	if supportedErr != nil {
-		return errors.Wrap(supportedErr, "unable to parse the Kamaji supported Kubernetes version")
-	}
-
-	if ver.GT(supportedVer) {
-		return fmt.Errorf("unable to create a TenantControlPlane with a Kubernetes version greater than the supported one, actually %s", supportedVer.String())
-	}
-
-	return nil
-}
-
-func (t *tenantControlPlaneValidator) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) error {
-	old, ok := oldObj.(*TenantControlPlane)
-	if !ok {
-		return fmt.Errorf("expected *kamajiv1alpha1.TenantControlPlane")
-	}
-
-	tcp, ok := newObj.(*TenantControlPlane)
-	if !ok {
-		return fmt.Errorf("expected *kamajiv1alpha1.TenantControlPlane")
-	}
-
-	t.log.Info("validate update", "name", tcp.Name, "namespace", tcp.Namespace)
-
-	if err := t.validateVersionUpdate(old, tcp); err != nil {
-		return err
-	}
-	if err := t.validateDataStore(ctx, old, tcp); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (t *tenantControlPlaneValidator) ValidateDelete(context.Context, runtime.Object) error {
-	return nil
-}
-
-func (t *tenantControlPlaneValidator) validateVersionUpdate(oldObj, newObj *TenantControlPlane) error {
-	oldVer, oldErr := semver.Make(t.normalizeKubernetesVersion(oldObj.Spec.Kubernetes.Version))
-	if oldErr != nil {
-		return errors.Wrap(oldErr, "unable to parse the previous Kubernetes version")
-	}
-
-	newVer, newErr := semver.New(t.normalizeKubernetesVersion(newObj.Spec.Kubernetes.Version))
-	if newErr != nil {
-		return errors.Wrap(newErr, "unable to parse the desired Kubernetes version")
-	}
-
-	supportedVer, supportedErr := semver.Make(t.normalizeKubernetesVersion(upgrade.KubeadmVersion))
-	if supportedErr != nil {
-		return errors.Wrap(supportedErr, "unable to parse the Kamaji supported Kubernetes version")
-	}
-
-	switch {
-	case newVer.GT(supportedVer):
-		return fmt.Errorf("unable to upgrade to a version greater than the supported one, actually %s", supportedVer.String())
-	case newVer.LT(oldVer):
-		return fmt.Errorf("unable to downgrade a TenantControlPlane from %s to %s", oldVer.String(), newVer.String())
-	case newVer.Minor-oldVer.Minor > 1:
-		return fmt.Errorf("unable to upgrade to a minor version in a non-sequential mode")
-	}
-
-	return nil
-}
-
-func (t *tenantControlPlaneValidator) validateDataStore(ctx context.Context, oldObj, tcp *TenantControlPlane) error {
-	if oldObj.Spec.DataStore == tcp.Spec.DataStore {
-		return nil
-	}
-
-	previousDatastore, desiredDatastore := &DataStore{}, &DataStore{}
-
-	if err := t.client.Get(ctx, types.NamespacedName{Name: oldObj.Spec.DataStore}, previousDatastore); err != nil {
-		return fmt.Errorf("unable to retrieve old DataStore for validation: %w", err)
-	}
-
-	if err := t.client.Get(ctx, types.NamespacedName{Name: tcp.Spec.DataStore}, desiredDatastore); err != nil {
-		return fmt.Errorf("unable to retrieve old DataStore for validation: %w", err)
-	}
-
-	if previousDatastore.Spec.Driver != desiredDatastore.Spec.Driver {
-		return fmt.Errorf("migration between different Datastore drivers is not supported")
-	}
-
-	return nil
-}
-
-func (t *tenantControlPlaneValidator) normalizeKubernetesVersion(input string) string {
-	if strings.HasPrefix(input, "v") {
-		return strings.Replace(input, "v", "", 1)
-	}
-
-	return input
-}

api/v1alpha1/types.go

@@ -28,9 +28,10 @@ func (c CGroupDriver) String() string {
 }
 
 const (
-	ServiceTypeLoadBalancer = (ServiceType)(corev1.ServiceTypeLoadBalancer)
-	ServiceTypeClusterIP    = (ServiceType)(corev1.ServiceTypeClusterIP)
-	ServiceTypeNodePort     = (ServiceType)(corev1.ServiceTypeNodePort)
+	ServiceTypeLoadBalancer       = (ServiceType)(corev1.ServiceTypeLoadBalancer)
+	ServiceTypeClusterIP          = (ServiceType)(corev1.ServiceTypeClusterIP)
+	ServiceTypeNodePort           = (ServiceType)(corev1.ServiceTypeNodePort)
+	KubeconfigSecretKeyAnnotation = "kamaji.clastix.io/kubeconfig-secret-key"
 )
 
 // +kubebuilder:validation:Enum=ClusterIP;NodePort;LoadBalancer

api/v1alpha1/webhook_suite_test.go

@@ -1,123 +0,0 @@
-// Copyright 2022 Clastix Labs
-// SPDX-License-Identifier: Apache-2.0
-
-package v1alpha1
-
-import (
-	"context"
-	"crypto/tls"
-	"fmt"
-	"net"
-	"path/filepath"
-	"testing"
-	"time"
-
-	. "github.com/onsi/ginkgo/v2"
-	. "github.com/onsi/gomega"
-	admissionv1beta1 "k8s.io/api/admission/v1beta1"
-	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/client-go/rest"
-	ctrl "sigs.k8s.io/controller-runtime"
-	"sigs.k8s.io/controller-runtime/pkg/client"
-	"sigs.k8s.io/controller-runtime/pkg/envtest"
-	logf "sigs.k8s.io/controller-runtime/pkg/log"
-	"sigs.k8s.io/controller-runtime/pkg/log/zap"
-	//+kubebuilder:scaffold:imports
-)
-
-// These tests use Ginkgo (BDD-style Go testing framework). Refer to
-// http://onsi.github.io/ginkgo/ to learn more about Ginkgo.
-
-var (
-	cfg       *rest.Config
-	k8sClient client.Client
-	testEnv   *envtest.Environment
-	ctx       context.Context
-	cancel    context.CancelFunc
-)
-
-func TestAPIs(t *testing.T) {
-	RegisterFailHandler(Fail)
-
-	RunSpecs(t, "Webhook Suite")
-}
-
-var _ = BeforeSuite(func() {
-	logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true)))
-
-	ctx, cancel = context.WithCancel(context.TODO())
-
-	By("bootstrapping test environment")
-	testEnv = &envtest.Environment{
-		CRDDirectoryPaths:     []string{filepath.Join("..", "..", "config", "crd", "bases")},
-		ErrorIfCRDPathMissing: false,
-		WebhookInstallOptions: envtest.WebhookInstallOptions{
-			Paths: []string{filepath.Join("..", "..", "config", "webhook")},
-		},
-	}
-
-	var err error
-	// cfg is defined in this file globally.
-	cfg, err = testEnv.Start()
-	Expect(err).NotTo(HaveOccurred())
-	Expect(cfg).NotTo(BeNil())
-
-	scheme := runtime.NewScheme()
-	err = AddToScheme(scheme)
-	Expect(err).NotTo(HaveOccurred())
-
-	err = admissionv1beta1.AddToScheme(scheme)
-	Expect(err).NotTo(HaveOccurred())
-
-	//+kubebuilder:scaffold:scheme
-
-	k8sClient, err = client.New(cfg, client.Options{Scheme: scheme})
-	Expect(err).NotTo(HaveOccurred())
-	Expect(k8sClient).NotTo(BeNil())
-
-	// start webhook server using Manager
-	webhookInstallOptions := &testEnv.WebhookInstallOptions
-	mgr, err := ctrl.NewManager(cfg, ctrl.Options{
-		Scheme:             scheme,
-		Host:               webhookInstallOptions.LocalServingHost,
-		Port:               webhookInstallOptions.LocalServingPort,
-		CertDir:            webhookInstallOptions.LocalServingCertDir,
-		LeaderElection:     false,
-		MetricsBindAddress: "0",
-	})
-	Expect(err).NotTo(HaveOccurred())
-
-	err = (&TenantControlPlane{}).SetupWebhookWithManager(mgr, "")
-	Expect(err).NotTo(HaveOccurred())
-
-	err = (&DataStore{}).SetupWebhookWithManager(mgr)
-	Expect(err).NotTo(HaveOccurred())
-
-	//+kubebuilder:scaffold:webhook
-
-	go func() {
-		defer GinkgoRecover()
-		err = mgr.Start(ctx)
-		Expect(err).NotTo(HaveOccurred())
-	}()
-
-	// wait for the webhook server to get ready
-	dialer := &net.Dialer{Timeout: time.Second}
-	addrPort := fmt.Sprintf("%s:%d", webhookInstallOptions.LocalServingHost, webhookInstallOptions.LocalServingPort)
-	Eventually(func() error {
-		conn, err := tls.DialWithDialer(dialer, "tcp", addrPort, &tls.Config{InsecureSkipVerify: true})
-		if err != nil {
-			return err
-		}
-		conn.Close()
-
-		return nil
-	}).Should(Succeed())
-})
-
-var _ = AfterSuite(func() {
-	cancel()
-	By("tearing down the test environment")
-	err := testEnv.Stop()
-	Expect(err).NotTo(HaveOccurred())
-})

api/v1alpha1/zz_generated.deepcopy.go

@@ -1,5 +1,4 @@
 //go:build !ignore_autogenerated
-// +build !ignore_autogenerated
 
 // Copyright 2022 Clastix Labs
 // SPDX-License-Identifier: Apache-2.0
@@ -10,7 +9,7 @@ package v1alpha1
 
 import (
 	"k8s.io/api/core/v1"
-	"k8s.io/apimachinery/pkg/runtime"
+	runtime "k8s.io/apimachinery/pkg/runtime"
 )
 
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
@@ -58,6 +57,42 @@ func (in *AdditionalMetadata) DeepCopy() *AdditionalMetadata {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *AdditionalVolumeMounts) DeepCopyInto(out *AdditionalVolumeMounts) {
+	*out = *in
+	if in.APIServer != nil {
+		in, out := &in.APIServer, &out.APIServer
+		*out = make([]v1.VolumeMount, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+	if in.ControllerManager != nil {
+		in, out := &in.ControllerManager, &out.ControllerManager
+		*out = make([]v1.VolumeMount, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+	if in.Scheduler != nil {
+		in, out := &in.Scheduler, &out.Scheduler
+		*out = make([]v1.VolumeMount, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AdditionalVolumeMounts.
+func (in *AdditionalVolumeMounts) DeepCopy() *AdditionalVolumeMounts {
+	if in == nil {
+		return nil
+	}
+	out := new(AdditionalVolumeMounts)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *AddonSpec) DeepCopyInto(out *AddonSpec) {
 	*out = *in
@@ -254,35 +289,6 @@ func (in *ClientCertificate) DeepCopy() *ClientCertificate {
 	return out
 }
 
-// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *ComponentResourceRequirements) DeepCopyInto(out *ComponentResourceRequirements) {
-	*out = *in
-	if in.Limits != nil {
-		in, out := &in.Limits, &out.Limits
-		*out = make(v1.ResourceList, len(*in))
-		for key, val := range *in {
-			(*out)[key] = val.DeepCopy()
-		}
-	}
-	if in.Requests != nil {
-		in, out := &in.Requests, &out.Requests
-		*out = make(v1.ResourceList, len(*in))
-		for key, val := range *in {
-			(*out)[key] = val.DeepCopy()
-		}
-	}
-}
-
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ComponentResourceRequirements.
-func (in *ComponentResourceRequirements) DeepCopy() *ComponentResourceRequirements {
-	if in == nil {
-		return nil
-	}
-	out := new(ComponentResourceRequirements)
-	in.DeepCopyInto(out)
-	return out
-}
-
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ContentRef) DeepCopyInto(out *ContentRef) {
 	*out = *in
@@ -335,17 +341,22 @@ func (in *ControlPlaneComponentsResources) DeepCopyInto(out *ControlPlaneCompone
 	*out = *in
 	if in.APIServer != nil {
 		in, out := &in.APIServer, &out.APIServer
-		*out = new(ComponentResourceRequirements)
+		*out = new(v1.ResourceRequirements)
 		(*in).DeepCopyInto(*out)
 	}
 	if in.ControllerManager != nil {
 		in, out := &in.ControllerManager, &out.ControllerManager
-		*out = new(ComponentResourceRequirements)
+		*out = new(v1.ResourceRequirements)
 		(*in).DeepCopyInto(*out)
 	}
 	if in.Scheduler != nil {
 		in, out := &in.Scheduler, &out.Scheduler
-		*out = new(ComponentResourceRequirements)
+		*out = new(v1.ResourceRequirements)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.Kine != nil {
+		in, out := &in.Kine, &out.Kine
+		*out = new(v1.ResourceRequirements)
 		(*in).DeepCopyInto(*out)
 	}
 }
@@ -514,7 +525,11 @@ func (in *DataStoreSpec) DeepCopyInto(out *DataStoreSpec) {
 		*out = new(BasicAuth)
 		(*in).DeepCopyInto(*out)
 	}
-	in.TLSConfig.DeepCopyInto(&out.TLSConfig)
+	if in.TLSConfig != nil {
+		in, out := &in.TLSConfig, &out.TLSConfig
+		*out = new(TLSConfig)
+		(*in).DeepCopyInto(*out)
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataStoreSpec.
@@ -547,9 +562,30 @@ func (in *DataStoreStatus) DeepCopy() *DataStoreStatus {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *DatastoreUsedSecret) DeepCopyInto(out *DatastoreUsedSecret) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DatastoreUsedSecret.
+func (in *DatastoreUsedSecret) DeepCopy() *DatastoreUsedSecret {
+	if in == nil {
+		return nil
+	}
+	out := new(DatastoreUsedSecret)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *DeploymentSpec) DeepCopyInto(out *DeploymentSpec) {
 	*out = *in
+	out.RegistrySettings = in.RegistrySettings
+	if in.Replicas != nil {
+		in, out := &in.Replicas, &out.Replicas
+		*out = new(int32)
+		**out = **in
+	}
 	if in.NodeSelector != nil {
 		in, out := &in.NodeSelector, &out.NodeSelector
 		*out = make(map[string]string, len(*in))
@@ -557,6 +593,7 @@ func (in *DeploymentSpec) DeepCopyInto(out *DeploymentSpec) {
 			(*out)[key] = val
 		}
 	}
+	in.Strategy.DeepCopyInto(&out.Strategy)
 	if in.Tolerations != nil {
 		in, out := &in.Tolerations, &out.Tolerations
 		*out = make([]v1.Toleration, len(*in))
@@ -587,6 +624,33 @@ func (in *DeploymentSpec) DeepCopyInto(out *DeploymentSpec) {
 		(*in).DeepCopyInto(*out)
 	}
 	in.AdditionalMetadata.DeepCopyInto(&out.AdditionalMetadata)
+	in.PodAdditionalMetadata.DeepCopyInto(&out.PodAdditionalMetadata)
+	if in.AdditionalInitContainers != nil {
+		in, out := &in.AdditionalInitContainers, &out.AdditionalInitContainers
+		*out = make([]v1.Container, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+	if in.AdditionalContainers != nil {
+		in, out := &in.AdditionalContainers, &out.AdditionalContainers
+		*out = make([]v1.Container, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+	if in.AdditionalVolumes != nil {
+		in, out := &in.AdditionalVolumes, &out.AdditionalVolumes
+		*out = make([]v1.Volume, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+	if in.AdditionalVolumeMounts != nil {
+		in, out := &in.AdditionalVolumeMounts, &out.AdditionalVolumeMounts
+		*out = new(AdditionalVolumeMounts)
+		(*in).DeepCopyInto(*out)
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DeploymentSpec.
@@ -720,6 +784,13 @@ func (in *IngressSpec) DeepCopy() *IngressSpec {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *KonnectivityAgentSpec) DeepCopyInto(out *KonnectivityAgentSpec) {
 	*out = *in
+	if in.Tolerations != nil {
+		in, out := &in.Tolerations, &out.Tolerations
+		*out = make([]v1.Toleration, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
 	if in.ExtraArgs != nil {
 		in, out := &in.ExtraArgs, &out.ExtraArgs
 		*out = make(ExtraArgs, len(*in))
@@ -757,7 +828,7 @@ func (in *KonnectivityServerSpec) DeepCopyInto(out *KonnectivityServerSpec) {
 	*out = *in
 	if in.Resources != nil {
 		in, out := &in.Resources, &out.Resources
-		*out = new(ComponentResourceRequirements)
+		*out = new(v1.ResourceRequirements)
 		(*in).DeepCopyInto(*out)
 	}
 	if in.ExtraArgs != nil {
@@ -901,6 +972,11 @@ func (in *KubeconfigsStatus) DeepCopy() *KubeconfigsStatus {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *KubeletSpec) DeepCopyInto(out *KubeletSpec) {
 	*out = *in
+	if in.PreferredAddressTypes != nil {
+		in, out := &in.PreferredAddressTypes, &out.PreferredAddressTypes
+		*out = make([]KubeletPreferredAddressType, len(*in))
+		copy(*out, *in)
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeletSpec.
@@ -965,7 +1041,7 @@ func (in *KubernetesServiceStatus) DeepCopy() *KubernetesServiceStatus {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *KubernetesSpec) DeepCopyInto(out *KubernetesSpec) {
 	*out = *in
-	out.Kubelet = in.Kubelet
+	in.Kubelet.DeepCopyInto(&out.Kubelet)
 	if in.AdmissionControllers != nil {
 		in, out := &in.AdmissionControllers, &out.AdmissionControllers
 		*out = make(AdmissionControllers, len(*in))
@@ -1067,6 +1143,21 @@ func (in *PublicKeyPrivateKeyPairStatus) DeepCopy() *PublicKeyPrivateKeyPairStat
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *RegistrySettings) DeepCopyInto(out *RegistrySettings) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RegistrySettings.
+func (in *RegistrySettings) DeepCopy() *RegistrySettings {
+	if in == nil {
+		return nil
+	}
+	out := new(RegistrySettings)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *SecretReference) DeepCopyInto(out *SecretReference) {
 	*out = *in
@@ -1121,7 +1212,11 @@ func (in *StorageStatus) DeepCopy() *StorageStatus {
 func (in *TLSConfig) DeepCopyInto(out *TLSConfig) {
 	*out = *in
 	in.CertificateAuthority.DeepCopyInto(&out.CertificateAuthority)
-	in.ClientCertificate.DeepCopyInto(&out.ClientCertificate)
+	if in.ClientCertificate != nil {
+		in, out := &in.ClientCertificate, &out.ClientCertificate
+		*out = new(ClientCertificate)
+		(*in).DeepCopyInto(*out)
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TLSConfig.
@@ -1233,3 +1328,18 @@ func (in *TenantControlPlaneStatus) DeepCopy() *TenantControlPlaneStatus {
 	in.DeepCopyInto(out)
 	return out
 }
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *TenantControlPlaneStatusDataStore) DeepCopyInto(out *TenantControlPlaneStatusDataStore) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TenantControlPlaneStatusDataStore.
+func (in *TenantControlPlaneStatusDataStore) DeepCopy() *TenantControlPlaneStatusDataStore {
+	if in == nil {
+		return nil
+	}
+	out := new(TenantControlPlaneStatusDataStore)
+	in.DeepCopyInto(out)
+	return out
+}

assets/kamaji-logo.svg

@@ -1 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="11.85 8.10 202.80 187.55"><title>Kamaji</title><path d="M32.1 13.7c-2.4.9-6.3 3.5-8.6 5.8-7.7 7.7-7.5 5-7.5 82.5 0 77.4-.2 74.8 7.5 82.5 7.7 7.8 4.2 7.5 90 7.5s82.3.3 90-7.5c7.7-7.7 7.5-5.1 7.5-82.5s.2-74.8-7.5-82.5c-7.8-7.8-4.1-7.5-90.4-7.4-66.7 0-77.2.3-81 1.6zm160.5 9.9c1.9.9 4.4 3.1 5.7 4.8l2.2 3.1v141l-2.2 3.1c-4.8 6.7-1.1 6.4-84.8 6.4s-80 .3-84.8-6.4l-2.2-3.1v-141l2.2-3.1c4.8-6.6.8-6.4 84.6-6.4 68 0 76.3.2 79.3 1.6z"/><path d="M90.1 33.7c-5.1 2.5-7.3 6.7-6.8 13.1.3 4.1 1 5.9 3.3 8.4s2.5 3 .9 2.3c-2-.7-25.1-4.6-29-4.9-1.1 0-2 .5-2 1.4 0 1.1-1.2 1.5-4.9 1.5-6.7 0-6.8 1.9-.4 4 8.2 2.7 9 3.4 3.3 3.5-5.3 0-8.2 1.1-7.1 2.8.7 1.2-2.7 2.2-8.1 2.2-7 0-6.5 2.4 1.1 5.1l3.9 1.4-2.9.5c-4.3.8-3.2 2.3 2.8 4.1l5.3 1.5-5.2 2.7c-8.2 4.2-8.3 5.8-.4 6.1 5.6.2 7.3 1.1 4.2 2.1-2.3.7-2.8 3.1-.9 3.7.7.3-.5 2-2.8 4-5.6 5.3-4 6.4 6.2 4.5 4.4-.8 8.1-1.3 8.3-1.2.2.2-1.3 2.4-3.3 4.8-2 2.4-3.6 4.7-3.6 5.2 0 .4 1.4.5 3 .3 2.9-.4 4 .5 2 1.7-.5.3-1 1.3-1 2.2 0 1.6 2.2 1.5 6.5-.3 1.7-.7 1.6-.2-.9 3-5.4 7.2.7 6.5 13.6-1.4 2.7-1.7 5.1-3 5.4-3 .3 0-.9 2.1-2.7 4.6-4.5 6.6-2.5 7.9 3.7 2.3 4.6-4.3 4.7-4.3 3-1.2-1.9 3.8-2.1 5.6-.4 5.1.6-.2 7.1-7.1 14.3-15.4 7.2-8.2 13.7-14.9 14.5-14.9.8 0 7.3 6.7 14.6 15 7.2 8.2 13.7 15.1 14.3 15.3 1.6.5 1.4-1.4-.5-5-1.6-3.2-1.6-3.2 3.2 1 6 5.1 7.8 4 3.5-2.2-1.8-2.5-3-4.6-2.7-4.6.3 0 2.7 1.3 5.4 3 12.9 7.9 19 8.6 13.6 1.4-2.5-3.2-2.6-3.7-.9-3 5.9 2.5 7.7 1.7 5.6-2.3-.9-1.5-.6-1.7 2-1.3 3.8.6 3.7-.5-.7-5.7-2-2.3-3.5-4.4-3.2-4.6.2-.2 2.1 0 4.3.4 13.9 3 16.4 1.8 9.8-4.3-2.1-1.9-3.2-3.6-2.5-3.6 2 0 1.4-2.8-.9-3.5-3.2-1-1.3-2 4.2-2.1 7.9-.2 7.8-1.9-.4-6.1l-5.2-2.7 5.4-1.6c6.4-1.8 7.9-4 2.9-4.1h-3.3l3.9-1.5c7.3-2.6 8.4-5.4 2.2-5.4-5.1 0-9.6-1.1-9-2.2 1.1-1.7-1.8-2.8-7.1-2.8-5.7-.1-4.9-.8 3.3-3.5 6.4-2.1 6.3-4-.4-4-3.7 0-4.9-.4-4.9-1.5 0-.9-.9-1.4-2-1.4-3.9.3-27 4.2-29 4.9-1.6.7-1.4.2.9-2.3 3.7-4 4.7-11.3 2.2-16.1-4.8-9.2-18.8-9.3-23.8 0-4.4 8.3.2 18.4 9.5 20.5 3 .6 2.8.8-5.5 4l-8.8 3.3-8.7-3.3c-8.1-3.2-8.4-3.4-5.5-4.1 1.7-.3 4.3-1.5 5.7-2.7 13.1-10.3.6-30.4-14.4-23.1zm77.6 98.4c-3.6 2.1-.8 7.7 3.2 6.4 2.1-.6 3.5-3.1 2.5-4.6-1.1-1.8-4-2.7-5.7-1.8zm8.3 3.9c0 1.9.5 2.1 6.3 1.8 4.7-.2 6.2-.7 6.2-1.8s-1.5-1.6-6.2-1.8c-5.8-.3-6.3-.1-6.3 1.8zm-135.6.3c-.2.7-.3 7.4-.2 14.8l.3 13.4 3.3.3c3.1.3 3.2.2 3.2-3.4 0-2.5.7-4.6 2.1-6l2.1-2.3 5 6c3.9 4.7 5.6 5.9 7.8 5.9 1.6 0 3.1-.3 3.3-.8.3-.4-2.1-4-5.4-8.1-3.2-4-5.9-7.6-5.9-8 0-.4 2.5-3.1 5.5-6.1 3-3 5.5-5.8 5.5-6.2 0-.4-1.5-.8-3.3-.8-2.8 0-4.4 1-9.6 6.5-3.5 3.6-6.5 6.5-6.7 6.5-.2 0-.4-2.9-.4-6.5V135h-3c-1.7 0-3.3.6-3.6 1.3zm31.2 7c-1.1.8-1.5 1.9-1 3 .5 1.4 1.3 1.6 4 1.1 4.2-.8 8.4.2 8.4 2 0 .8-1.8 1.5-5.1 1.9-6 .7-8.9 2.9-8.9 6.6 0 3.2.8 4.4 3.7 6 2.9 1.5 5.2 1.4 8.6-.3 2.3-1.3 2.7-1.3 2.7 0 0 .9 1.1 1.4 3 1.4h3v-8.6c0-8.1-.1-8.7-2.9-11.5-2.5-2.5-3.7-2.9-8.3-2.9-3 0-6.2.6-7.2 1.3zm11.2 13.9c-.2 1.7-1.1 2.4-3.2 2.6-3.3.4-5.1-1-4.3-3.2.4-1.1 1.9-1.6 4.2-1.6 3.2 0 3.6.3 3.3 2.2zm13.4-4l.3 11.3h6l.5-7.8c.5-7.6 1.5-9.6 4.7-9.7 3 0 4.3 3.2 4.3 10.6v7.4h3c3 0 3 0 3-5.9 0-7.3 1.2-10.7 4.1-11.6 3.8-1.3 5.9 2.5 5.9 10.6v6.9h6v-9c0-8.3-.2-9.3-2.5-11.5-2.9-3-9.8-3.5-12.7-.8-1.7 1.5-1.9 1.5-3.6 0-2.2-2-9.2-2.3-11.1-.5-1.1 1-1.4 1-1.8 0-.3-.6-1.8-1.2-3.4-1.2h-3l.3 11.2zm45.4-9.9c-1.1.8-1.5 1.9-1 3 .5 1.4 1.3 1.6 4 1.1 4.2-.8 8.4.2 8.4 2 0 .8-1.8 1.5-5.1 1.9-6 .7-8.9 2.9-8.9 6.6 0 3.2.8 4.4 3.7 6 2.9 1.5 5.2 1.4 8.6-.3 2.3-1.3 2.7-1.3 2.7 0 0 .9 1.1 1.4 3 1.4h3v-8.6c0-8.1-.1-8.7-2.9-11.5-2.5-2.5-3.7-2.9-8.3-2.9-3 0-6.2.6-7.2 1.3zm11.2 13.9c-.2 1.7-1.1 2.4-3.2 2.6-3.3.4-5.1-1-4.3-3.2.4-1.1 1.9-1.6 4.2-1.6 3.2 0 3.6.3 3.3 2.2zm13-2.5c-.3 12.8-.3 12.8-2.7 12.8-1.5 0-2.7.8-3.1 2-2 5.4 9.4 4.3 11.9-1.2.6-1.3 1.1-7.7 1.1-14.3v-12h-6.9l-.3 12.7zm13.4-1.5l.3 11.3h6v-22l-3.3-.3-3.3-.3.3 11.3z"/></svg>

charts/kamaji/Chart.yaml

@@ -1,26 +1,24 @@
 apiVersion: v2
-appVersion: v0.1.1
-description: Kamaji is a tool aimed to build and operate a Managed Kubernetes Service
-  with a fraction of the operational burden. With Kamaji, you can deploy and operate
-  hundreds of Kubernetes clusters as a hyper-scaler.
+appVersion: v1.0.0
+description: Kamaji is the Hosted Control Plane Manager for Kubernetes.
 home: https://github.com/clastix/kamaji
-icon: https://github.com/clastix/kamaji/raw/master/assets/kamaji-logo.png
+icon: https://github.com/clastix/kamaji/raw/master/assets/logo-colored.png
 kubeVersion: ">=1.21.0-0"
 maintainers:
 - email: dario@tranchitella.eu
   name: Dario Tranchitella
+  url: https://clastix.io
 - email: me@maxgio.it
   name: Massimiliano Giovagnoli
 - email: me@bsctl.io
   name: Adriano Pezzuto
-- email: iam@mendrugory.com
-  name: Gonzalo Gabriel Jiménez Fuentes
+  url: https://clastix.io
 name: kamaji
 sources:
 - https://github.com/clastix/kamaji
 type: application
-version: 0.10.3
+version: 1.0.0
 annotations:
   catalog.cattle.io/certified: partner
   catalog.cattle.io/release-name: kamaji
-  catalog.cattle.io/display-name: Kamaji - Managed Kubernetes Service
+  catalog.cattle.io/display-name: Kamaji

charts/kamaji/README.md

@@ -1,17 +1,16 @@
 # kamaji
 
-![Version: 0.10.3](https://img.shields.io/badge/Version-0.10.3-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v0.1.1](https://img.shields.io/badge/AppVersion-v0.1.1-informational?style=flat-square)
+![Version: 1.0.0](https://img.shields.io/badge/Version-1.0.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v1.0.0](https://img.shields.io/badge/AppVersion-v1.0.0-informational?style=flat-square)
 
-Kamaji is a tool aimed to build and operate a Managed Kubernetes Service with a fraction of the operational burden. With Kamaji, you can deploy and operate hundreds of Kubernetes clusters as a hyper-scaler.
+Kamaji is the Hosted Control Plane Manager for Kubernetes.
 
 ## Maintainers
 
 | Name | Email | Url |
 | ---- | ------ | --- |
-| Dario Tranchitella | <dario@tranchitella.eu> |  |
+| Dario Tranchitella | <dario@tranchitella.eu> | <https://clastix.io> |
 | Massimiliano Giovagnoli | <me@maxgio.it> |  |
-| Adriano Pezzuto | <me@bsctl.io> |  |
-| Gonzalo Gabriel Jiménez Fuentes | <iam@mendrugory.com> |  |
+| Adriano Pezzuto | <me@bsctl.io> | <https://clastix.io> |
 
 ## Source Code
 
@@ -67,6 +66,8 @@ Here the values you can override:
 | Key | Type | Default | Description |
 |-----|------|---------|-------------|
 | affinity | object | `{}` | Kubernetes affinity rules to apply to Kamaji controller pods |
+| cfssl.image.repository | string | `"cfssl/cfssl"` |  |
+| cfssl.image.tag | string | `"latest"` |  |
 | datastore.basicAuth.passwordSecret.keyPath | string | `nil` | The Secret key where the data is stored. |
 | datastore.basicAuth.passwordSecret.name | string | `nil` | The name of the Secret containing the password used to connect to the relational database. |
 | datastore.basicAuth.passwordSecret.namespace | string | `nil` | The namespace of the Secret containing the password used to connect to the relational database. |
@@ -74,8 +75,9 @@ Here the values you can override:
 | datastore.basicAuth.usernameSecret.name | string | `nil` | The name of the Secret containing the username used to connect to the relational database. |
 | datastore.basicAuth.usernameSecret.namespace | string | `nil` | The namespace of the Secret containing the username used to connect to the relational database. |
 | datastore.driver | string | `"etcd"` | (string) The Kamaji Datastore driver, supported: etcd, MySQL, PostgreSQL (defaults=etcd). |
+| datastore.enabled | bool | `true` | (bool) Enable the Kamaji Datastore creation (default=true) |
 | datastore.endpoints | list | `[]` | (array) List of endpoints of the selected Datastore. When letting the Chart install the etcd datastore, this field is populated automatically. |
-| datastore.nameOverride | string | `nil` | The Datastore name override, if empty defaults to `default` |
+| datastore.nameOverride | string | `nil` | The Datastore name override, if empty and enabled=true defaults to `default`, if enabled=false, this is the name of the Datastore to connect to. |
 | datastore.tlsConfig.certificateAuthority.certificate.keyPath | string | `nil` | Key of the Secret which contains the content of the certificate. |
 | datastore.tlsConfig.certificateAuthority.certificate.name | string | `nil` | Name of the Secret containing the CA required to establish the mandatory SSL/TLS connection to the datastore. |
 | datastore.tlsConfig.certificateAuthority.certificate.namespace | string | `nil` | Namespace of the Secret containing the CA required to establish the mandatory SSL/TLS connection to the datastore. |
@@ -88,6 +90,7 @@ Here the values you can override:
 | datastore.tlsConfig.clientCertificate.privateKey.keyPath | string | `nil` | Key of the Secret which contains the content of the private key. |
 | datastore.tlsConfig.clientCertificate.privateKey.name | string | `nil` | Name of the Secret containing the client certificate private key required to establish the mandatory SSL/TLS connection to the datastore. |
 | datastore.tlsConfig.clientCertificate.privateKey.namespace | string | `nil` | Namespace of the Secret containing the client certificate private key required to establish the mandatory SSL/TLS connection to the datastore. |
+| datastore.tlsConfig.enabled | bool | `true` |  |
 | etcd.compactionInterval | int | `0` | ETCD Compaction interval (e.g. "5m0s"). (default: "0" (disabled)) |
 | etcd.deploy | bool | `true` | Install an etcd with enabled multi-tenancy along with Kamaji |
 | etcd.image | object | `{"pullPolicy":"IfNotPresent","repository":"quay.io/coreos/etcd","tag":"v3.5.6"}` | Install specific etcd image |
@@ -99,11 +102,13 @@ Here the values you can override:
 | etcd.overrides.endpoints | object | `{"etcd-0":"etcd-0.etcd.kamaji-system.svc.cluster.local","etcd-1":"etcd-1.etcd.kamaji-system.svc.cluster.local","etcd-2":"etcd-2.etcd.kamaji-system.svc.cluster.local"}` | (map) Dictionary of the endpoints for the etcd cluster's members, key is the name of the etcd server. Don't define the protocol (TLS is automatically inflected), or any port, inflected from .etcd.peerApiPort value. |
 | etcd.peerApiPort | int | `2380` | The peer API port which servers are listening to. |
 | etcd.persistence.accessModes[0] | string | `"ReadWriteOnce"` |  |
+| etcd.persistence.customAnnotations | object | `{}` | The custom annotations to add to the PVC |
 | etcd.persistence.size | string | `"10Gi"` |  |
-| etcd.persistence.storageClass | string | `""` |  |
+| etcd.persistence.storageClassName | string | `""` |  |
 | etcd.port | int | `2379` | The client request port. |
 | etcd.serviceAccount.create | bool | `true` | Create a ServiceAccount, required to install and provision the etcd backing storage (default: true) |
 | etcd.serviceAccount.name | string | `""` | Define the ServiceAccount name to use during the setup and provision of the etcd backing storage (default: "") |
+| etcd.tolerations | list | `[]` | (array) Kubernetes affinity rules to apply to Kamaji etcd pods |
 | extraArgs | list | `[]` | A list of extra arguments to add to the kamaji controller default ones |
 | fullnameOverride | string | `""` |  |
 | healthProbeBindAddress | string | `":8081"` | The address the probe endpoint binds to. (default ":8081") |
@@ -128,6 +133,8 @@ Here the values you can override:
 | serviceAccount.annotations | object | `{}` |  |
 | serviceAccount.create | bool | `true` |  |
 | serviceAccount.name | string | `"kamaji-controller-manager"` |  |
+| serviceMonitor.enabled | bool | `false` | Toggle the ServiceMonitor true if you have Prometheus Operator installed and configured |
+| telemetry | object | `{"disabled":false}` | Disable the analytics traces collection |
 | temporaryDirectoryPath | string | `"/tmp/kamaji"` | Directory which will be used to work with temporary files. (default "/tmp/kamaji") |
 | tolerations | list | `[]` | Kubernetes node taints that the Kamaji controller pods would tolerate |
 

charts/kamaji/app-readme.md

@@ -1,30 +1,12 @@
-# Kamaji - Managed Kubernetes Service
+# Kamaji
 
-Kamaji is a tool aimed to build and operate a Managed Kubernetes Service with a fraction of the operational burden.
+Kamaji deploys and operates Kubernetes at scale with a fraction of the operational burden.
 
 Useful links:
 - [Kamaji Github repository](https://github.com/clastix/kamaji)
-- [Kamaji Documentation](https://github.com/clastix/kamaji/docs/)
+- [Kamaji Documentation](https://kamaji.clastix.io)
 
 ## Requirements
 
 * Kubernetes v1.22+
-* Helm v3
-
-# Installation
-
-To install the Chart with the release name `kamaji`:
-
-        helm upgrade --install --namespace kamaji-system --create-namespace clastix/kamaji
-
-Show the status:
-
-        helm status kamaji -n kamaji-system
-
-Upgrade the Chart
-
-        helm upgrade kamaji -n kamaji-system clastix/kamaji
-
-Uninstall the Chart
-
-        helm uninstall kamaji -n kamaji-system
+* Helm v3

charts/kamaji/crds/datastore.yaml

@@ -4,7 +4,7 @@ kind: CustomResourceDefinition
 metadata:
   annotations:
     cert-manager.io/inject-ca-from: kamaji-system/kamaji-serving-cert
-    controller-gen.kubebuilder.io/version: v0.9.2
+    controller-gen.kubebuilder.io/version: v0.11.4
   name: datastores.kamaji.clastix.io
 spec:
   group: kamaji.clastix.io
@@ -30,10 +30,19 @@ spec:
           description: DataStore is the Schema for the datastores API.
           properties:
             apiVersion:
-              description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              description: |-
+                APIVersion defines the versioned schema of this representation of an object.
+                Servers should convert recognized schemas to the latest internal value, and
+                may reject unrecognized values.
+                More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
               type: string
             kind:
-              description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              description: |-
+                Kind is a string value representing the REST resource this object represents.
+                Servers may infer this from the endpoint the client submits requests to.
+                Cannot be updated.
+                In CamelCase.
+                More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
               type: string
             metadata:
               type: object
@@ -41,25 +50,33 @@ spec:
               description: DataStoreSpec defines the desired state of DataStore.
               properties:
                 basicAuth:
-                  description: In case of authentication enabled for the given data store, specifies the username and password pair. This value is optional.
+                  description: |-
+                    In case of authentication enabled for the given data store, specifies the username and password pair.
+                    This value is optional.
                   properties:
                     password:
                       properties:
                         content:
-                          description: Bare content of the file, base64 encoded. It has precedence over the SecretReference value.
+                          description: |-
+                            Bare content of the file, base64 encoded.
+                            It has precedence over the SecretReference value.
                           format: byte
                           type: string
                         secretReference:
                           properties:
                             keyPath:
-                              description: Name of the key for the given Secret reference where the content is stored. This value is mandatory.
+                              description: |-
+                                Name of the key for the given Secret reference where the content is stored.
+                                This value is mandatory.
                               minLength: 1
                               type: string
                             name:
-                              description: name is unique within a namespace to reference a secret resource.
+                              description: name is unique within a namespace to reference
+                                a secret resource.
                               type: string
                             namespace:
-                              description: namespace defines the space within which the secret name must be unique.
+                              description: namespace defines the space within which
+                                the secret name must be unique.
                               type: string
                           required:
                             - keyPath
@@ -69,20 +86,26 @@ spec:
                     username:
                       properties:
                         content:
-                          description: Bare content of the file, base64 encoded. It has precedence over the SecretReference value.
+                          description: |-
+                            Bare content of the file, base64 encoded.
+                            It has precedence over the SecretReference value.
                           format: byte
                           type: string
                         secretReference:
                           properties:
                             keyPath:
-                              description: Name of the key for the given Secret reference where the content is stored. This value is mandatory.
+                              description: |-
+                                Name of the key for the given Secret reference where the content is stored.
+                                This value is mandatory.
                               minLength: 1
                               type: string
                             name:
-                              description: name is unique within a namespace to reference a secret resource.
+                              description: name is unique within a namespace to reference
+                                a secret resource.
                               type: string
                             namespace:
-                              description: namespace defines the space within which the secret name must be unique.
+                              description: namespace defines the space within which
+                                the secret name must be unique.
                               type: string
                           required:
                             - keyPath
@@ -99,36 +122,49 @@ spec:
                     - etcd
                     - MySQL
                     - PostgreSQL
+                    - NATS
                   type: string
                 endpoints:
-                  description: List of the endpoints to connect to the shared datastore. No need for protocol, just bare IP/FQDN and port.
+                  description: |-
+                    List of the endpoints to connect to the shared datastore.
+                    No need for protocol, just bare IP/FQDN and port.
                   items:
                     type: string
                   minItems: 1
                   type: array
                 tlsConfig:
-                  description: Defines the TLS/SSL configuration required to connect to the data store in a secure way.
+                  description: |-
+                    Defines the TLS/SSL configuration required to connect to the data store in a secure way.
+                    This value is optional.
                   properties:
                     certificateAuthority:
-                      description: Retrieve the Certificate Authority certificate and private key, such as bare content of the file, or a SecretReference. The key reference is required since etcd authentication is based on certificates, and Kamaji is responsible in creating this.
+                      description: |-
+                        Retrieve the Certificate Authority certificate and private key, such as bare content of the file, or a SecretReference.
+                        The key reference is required since etcd authentication is based on certificates, and Kamaji is responsible in creating this.
                       properties:
                         certificate:
                           properties:
                             content:
-                              description: Bare content of the file, base64 encoded. It has precedence over the SecretReference value.
+                              description: |-
+                                Bare content of the file, base64 encoded.
+                                It has precedence over the SecretReference value.
                               format: byte
                               type: string
                             secretReference:
                               properties:
                                 keyPath:
-                                  description: Name of the key for the given Secret reference where the content is stored. This value is mandatory.
+                                  description: |-
+                                    Name of the key for the given Secret reference where the content is stored.
+                                    This value is mandatory.
                                   minLength: 1
                                   type: string
                                 name:
-                                  description: name is unique within a namespace to reference a secret resource.
+                                  description: name is unique within a namespace to
+                                    reference a secret resource.
                                   type: string
                                 namespace:
-                                  description: namespace defines the space within which the secret name must be unique.
+                                  description: namespace defines the space within which
+                                    the secret name must be unique.
                                   type: string
                               required:
                                 - keyPath
@@ -138,20 +174,26 @@ spec:
                         privateKey:
                           properties:
                             content:
-                              description: Bare content of the file, base64 encoded. It has precedence over the SecretReference value.
+                              description: |-
+                                Bare content of the file, base64 encoded.
+                                It has precedence over the SecretReference value.
                               format: byte
                               type: string
                             secretReference:
                               properties:
                                 keyPath:
-                                  description: Name of the key for the given Secret reference where the content is stored. This value is mandatory.
+                                  description: |-
+                                    Name of the key for the given Secret reference where the content is stored.
+                                    This value is mandatory.
                                   minLength: 1
                                   type: string
                                 name:
-                                  description: name is unique within a namespace to reference a secret resource.
+                                  description: name is unique within a namespace to
+                                    reference a secret resource.
                                   type: string
                                 namespace:
-                                  description: namespace defines the space within which the secret name must be unique.
+                                  description: namespace defines the space within which
+                                    the secret name must be unique.
                                   type: string
                               required:
                                 - keyPath
@@ -162,25 +204,32 @@ spec:
                         - certificate
                       type: object
                     clientCertificate:
-                      description: Specifies the SSL/TLS key and private key pair used to connect to the data store.
+                      description: Specifies the SSL/TLS key and private key pair used
+                        to connect to the data store.
                       properties:
                         certificate:
                           properties:
                             content:
-                              description: Bare content of the file, base64 encoded. It has precedence over the SecretReference value.
+                              description: |-
+                                Bare content of the file, base64 encoded.
+                                It has precedence over the SecretReference value.
                               format: byte
                               type: string
                             secretReference:
                               properties:
                                 keyPath:
-                                  description: Name of the key for the given Secret reference where the content is stored. This value is mandatory.
+                                  description: |-
+                                    Name of the key for the given Secret reference where the content is stored.
+                                    This value is mandatory.
                                   minLength: 1
                                   type: string
                                 name:
-                                  description: name is unique within a namespace to reference a secret resource.
+                                  description: name is unique within a namespace to
+                                    reference a secret resource.
                                   type: string
                                 namespace:
-                                  description: namespace defines the space within which the secret name must be unique.
+                                  description: namespace defines the space within which
+                                    the secret name must be unique.
                                   type: string
                               required:
                                 - keyPath
@@ -190,20 +239,26 @@ spec:
                         privateKey:
                           properties:
                             content:
-                              description: Bare content of the file, base64 encoded. It has precedence over the SecretReference value.
+                              description: |-
+                                Bare content of the file, base64 encoded.
+                                It has precedence over the SecretReference value.
                               format: byte
                               type: string
                             secretReference:
                               properties:
                                 keyPath:
-                                  description: Name of the key for the given Secret reference where the content is stored. This value is mandatory.
+                                  description: |-
+                                    Name of the key for the given Secret reference where the content is stored.
+                                    This value is mandatory.
                                   minLength: 1
                                   type: string
                                 name:
-                                  description: name is unique within a namespace to reference a secret resource.
+                                  description: name is unique within a namespace to
+                                    reference a secret resource.
                                   type: string
                                 namespace:
-                                  description: namespace defines the space within which the secret name must be unique.
+                                  description: namespace defines the space within which
+                                    the secret name must be unique.
                                   type: string
                               required:
                                 - keyPath
@@ -216,18 +271,17 @@ spec:
                       type: object
                   required:
                     - certificateAuthority
-                    - clientCertificate
                   type: object
               required:
                 - driver
                 - endpoints
-                - tlsConfig
               type: object
             status:
               description: DataStoreStatus defines the observed state of DataStore.
               properties:
                 usedBy:
-                  description: List of the Tenant Control Planes, namespaced named, using this data store.
+                  description: List of the Tenant Control Planes, namespaced named,
+                    using this data store.
                   items:
                     type: string
                   type: array

charts/kamaji/templates/_helpers.tpl

@@ -46,9 +46,9 @@ app.kubernetes.io/managed-by: {{ .Release.Service }}
 Selector labels
 */}}
 {{- define "kamaji.selectorLabels" -}}
-app.kubernetes.io/name: {{ include "kamaji.name" . }}
-app.kubernetes.io/instance: {{ .Release.Name }}
-app.kubernetes.io/component: controller-manager
+app.kubernetes.io/name: {{ default (include "kamaji.name" .) .name }}
+app.kubernetes.io/instance: {{ default .Release.Name .instance }}
+app.kubernetes.io/component: {{ default "controller-manager" .component }}
 {{- end }}
 
 {{/*
@@ -69,6 +69,13 @@ Create the name of the Service to user for webhooks
 {{- printf "%s-webhook-service" (include "kamaji.fullname" .) }}
 {{- end }}
 
+{{/*
+Create the name of the Service to user for metrics
+*/}}
+{{- define "kamaji.metricsServiceName" -}}
+{{- printf "%s-metrics-service" (include "kamaji.fullname" .) }}
+{{- end }}
+
 {{/*
 Create the name of the cert-manager secret
 */}}

charts/kamaji/templates/_helpers_datastore.tpl

@@ -2,7 +2,11 @@
 Create a default fully qualified datastore name.
 */}}
 {{- define "datastore.fullname" -}}
+{{- if .Values.datastore.enabled }}
 {{- default "default" .Values.datastore.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- required "A valid .Values.datastore.nameOverride required!" .Values.datastore.nameOverride }}
+{{- end }}
 {{- end }}
 
 {{/*

charts/kamaji/templates/certmanager_certificate.yaml

@@ -2,8 +2,8 @@ apiVersion: cert-manager.io/v1
 kind: Certificate
 metadata:
   labels:
-    {{- include "kamaji.labels" . | nindent 4 }}
-    app.kubernetes.io/component: certificate
+    {{- $data := . | mustMergeOverwrite (dict "component" "certificate") -}}
+    {{- include "kamaji.labels" $data | nindent 4 }}
   name: {{ include "kamaji.certificateName" . }}
   namespace: {{ .Release.Namespace }}
 spec:

charts/kamaji/templates/certmanager_issuer.yaml

@@ -2,8 +2,8 @@ apiVersion: cert-manager.io/v1
 kind: Issuer
 metadata:
   labels:
-    {{- include "kamaji.labels" . | nindent 4 }}
-    app.kubernetes.io/component: issuer
+    {{- $data := . | mustMergeOverwrite (dict "component" "issuer") -}}
+    {{- include "kamaji.labels" $data | nindent 4 }}
   name: kamaji-selfsigned-issuer
   namespace: {{ .Release.Namespace }}
 spec:

charts/kamaji/templates/controller.yaml

@@ -34,6 +34,9 @@ spec:
         - --metrics-bind-address={{ .Values.metricsBindAddress }}
         - --tmp-directory={{ .Values.temporaryDirectoryPath }}
         - --datastore={{ include "datastore.fullname" . }}
+        {{- if .Values.telemetry.disabled }}
+        - --disable-telemetry
+        {{- end }}
         {{- if .Values.loggingDevel.enable }}
         - --zap-devel
         {{- end }}
@@ -62,6 +65,9 @@ spec:
         - containerPort: 9443
           name: webhook-server
           protocol: TCP
+        - containerPort: 8080
+          name: metrics
+          protocol: TCP
         - containerPort: 8081
           name: healthcheck
           protocol: TCP
@@ -74,11 +80,16 @@ spec:
         securityContext:
           {{- toYaml .Values.securityContext | nindent 12 }}
         volumeMounts:
+          - mountPath: /tmp
+            name: tmp
           - mountPath: /tmp/k8s-webhook-server/serving-certs
             name: cert
             readOnly: true
       terminationGracePeriodSeconds: 10
       volumes:
+        - name: tmp
+          emptyDir:
+            medium: Memory
         - name: cert
           secret:
             defaultMode: 420

charts/kamaji/templates/datastore.yaml

@@ -1,3 +1,4 @@
+{{- if .Values.datastore.enabled}}
 apiVersion: kamaji.clastix.io/v1alpha1
 kind: DataStore
 metadata:
@@ -12,10 +13,21 @@ spec:
     {{- include "datastore.endpoints" . | indent 4 }}
 {{- if (and .Values.datastore.basicAuth.usernameSecret.name .Values.datastore.basicAuth.passwordSecret.name) }}
   basicAuth:
-    {{- .Values.datastore.basicAuth | toYaml | nindent 4 }}
+    username:
+      secretReference:
+        {{- .Values.datastore.basicAuth.usernameSecret | toYaml | nindent 8 }}
+    password:
+      secretReference:
+        {{- .Values.datastore.basicAuth.passwordSecret | toYaml | nindent 8 }}
 {{- end }}
+{{- if .Values.datastore.tlsConfig.enabled }}
   tlsConfig:
     certificateAuthority:
       {{- include "datastore.certificateAuthority" . | indent 6 }}
+
+    {{- if .Values.datastore.tlsConfig.clientCertificate }}
     clientCertificate:
       {{- include "datastore.clientCertificate" . | indent 6 }}
+    {{- end }}
+{{- end}}
+{{- end}}

charts/kamaji/templates/etcd_job_postdelete.yaml

@@ -28,4 +28,8 @@ spec:
             - --ignore-not-found=true
             - {{ include "etcd.caSecretName" . }}
             - {{ include "etcd.clientSecretName" . }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
 {{- end }}

charts/kamaji/templates/etcd_job_postinstall.yaml

@@ -30,11 +30,15 @@ spec:
             - bash
             - -c
             - |-
-              etcdctl member list -w table &&
-              etcdctl user add --no-password=true root &&
-              etcdctl role add root &&
-              etcdctl user grant-role root root &&
-              etcdctl auth enable
+              etcdctl member list -w table
+              if etcdctl user get root &>/dev/null; then
+                echo "User already exists, nothing to do"
+              else
+                etcdctl user add --no-password=true root &&
+                etcdctl role add root &&
+                etcdctl user grant-role root root &&
+                etcdctl auth enable
+              fi
           env:
             - name: ETCDCTL_ENDPOINTS
               value: https://etcd-0.{{ include "etcd.serviceName" . }}.{{ .Release.Namespace }}.svc.cluster.local:2379
@@ -63,4 +67,8 @@ spec:
         - name: certs
           secret:
             secretName: {{ include "etcd.caSecretName" . }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
 {{- end }}

charts/kamaji/templates/etcd_job_preinstall.yaml

@@ -19,7 +19,7 @@ spec:
       restartPolicy: Never
       initContainers:
         - name: cfssl
-          image: cfssl/cfssl:latest
+          image: "{{ .Values.cfssl.image.repository }}:{{ .Values.cfssl.image.tag }}"
           command:
             - bash
             - -c
@@ -37,13 +37,21 @@ spec:
       containers:
         - name: kubectl
           image: {{ printf "clastix/kubectl:%s" (include "etcd.jobsTagKubeVersion" .) }}
-          command:
-            - sh
-            - -c
-            - |-
-              kubectl --namespace={{ .Release.Namespace }} delete secret --ignore-not-found=true {{ include "etcd.caSecretName" . }} {{ include "etcd.clientSecretName" . }} &&
-              kubectl --namespace={{ .Release.Namespace }} create secret generic {{ include "etcd.caSecretName" . }} --from-file=/certs/ca.crt --from-file=/certs/ca.key --from-file=/certs/peer-key.pem --from-file=/certs/peer.pem --from-file=/certs/server-key.pem --from-file=/certs/server.pem &&
-              kubectl --namespace={{ .Release.Namespace }} create secret tls {{ include "etcd.clientSecretName" . }} --key=/certs/root-client-key.pem --cert=/certs/root-client.pem
+          command: ["/bin/sh", "-c"]
+          args:
+            - |
+              if kubectl get secret {{ include "etcd.caSecretName" . }} --namespace={{ .Release.Namespace }} &>/dev/null; then
+                echo "Secret {{ include "etcd.caSecretName" . }} already exists"
+              else
+                echo "Creating secret {{ include "etcd.caSecretName" . }}"
+                kubectl --namespace={{ .Release.Namespace }} create secret generic {{ include "etcd.caSecretName" . }} --from-file=/certs/ca.crt --from-file=/certs/ca.key --from-file=/certs/peer-key.pem --from-file=/certs/peer.pem --from-file=/certs/server-key.pem --from-file=/certs/server.pem
+              fi
+              if kubectl get secret {{ include "etcd.clientSecretName" . }} --namespace={{ .Release.Namespace }} &>/dev/null; then
+                echo "Secret {{ include "etcd.clientSecretName" . }} already exists"
+              else
+                echo "Creating secret {{ include "etcd.clientSecretName" . }}"
+                kubectl --namespace={{ .Release.Namespace }} create secret tls {{ include "etcd.clientSecretName" . }} --key=/certs/root-client-key.pem --cert=/certs/root-client.pem
+              fi
           volumeMounts:
             - mountPath: /certs
               name: certs
@@ -57,4 +65,8 @@ spec:
             name: {{ include "etcd.csrConfigMapName" . }}
         - name: certs
           emptyDir: {}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
 {{- end }}

charts/kamaji/templates/etcd_rbac.yaml

@@ -15,6 +15,7 @@ rules:
     resources:
       - secrets
     verbs:
+      - get
       - delete
     resourceNames:
       - {{ include "etcd.caSecretName" . }}

charts/kamaji/templates/etcd_sts.yaml

@@ -22,6 +22,10 @@ spec:
         - name: certs
           secret:
             secretName: {{ include "etcd.caSecretName" . }}
+      {{- with .Values.etcd.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
       containers:
         - name: etcd
           image: {{ .Values.etcd.image.repository }}:{{ .Values.etcd.image.tag | default "v3.5.4" }}
@@ -81,6 +85,10 @@ spec:
   volumeClaimTemplates:
   - metadata:
       name: data
+      {{- with .Values.etcd.persistence.customAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
     spec:
       storageClassName: {{ .Values.etcd.persistence.storageClassName }}
       accessModes:

charts/kamaji/templates/mutatingwebhookconfiguration.yaml

@@ -4,30 +4,10 @@ metadata:
   annotations:
     cert-manager.io/inject-ca-from: {{ .Release.Namespace }}/{{ include "kamaji.certificateName" . }}
   labels:
-    {{- include "kamaji.labels" . | nindent 4 }}
-    app.kubernetes.io/instance: mutating-webhook-configuration
+    {{- $data := . | mustMergeOverwrite (dict "instance" "mutating-webhook-configuration") -}}
+    {{- include "kamaji.labels" $data | nindent 4 }}
   name: kamaji-mutating-webhook-configuration
 webhooks:
-  - admissionReviewVersions:
-      - v1
-    clientConfig:
-      service:
-        name: {{ include "kamaji.webhookServiceName" . }}
-        namespace: {{ .Release.Namespace }}
-        path: /mutate-kamaji-clastix-io-v1alpha1-datastore
-    failurePolicy: Fail
-    name: mdatastore.kb.io
-    rules:
-      - apiGroups:
-          - kamaji.clastix.io
-        apiVersions:
-          - v1alpha1
-        operations:
-          - CREATE
-          - UPDATE
-        resources:
-          - datastores
-    sideEffects: None
   - admissionReviewVersions:
       - v1
     clientConfig:

charts/kamaji/templates/service_metrics.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    {{- $data := . | mustMergeOverwrite (dict "component" "metrics") -}}
+    {{- include "kamaji.labels" $data | nindent 4 }}
+  name: {{ include "kamaji.metricsServiceName" . }}
+  namespace: {{ .Release.Namespace }}
+spec:
+  ports:
+    - port: 8080
+      name: metrics
+      protocol: TCP
+      targetPort: metrics
+  selector:
+    {{- include "kamaji.selectorLabels" . | nindent 4 }}

charts/kamaji/templates/service_webhook.yaml

@@ -2,15 +2,15 @@ apiVersion: v1
 kind: Service
 metadata:
   labels:
-    {{- include "kamaji.labels" . | nindent 4 }}
-    app.kubernetes.io/component: webhook
-    app.kubernetes.io/instance: webhook-service
+    {{- $data := . | mustMergeOverwrite (dict "component" "webhook" "instance" "webhook-service") -}}
+    {{- include "kamaji.labels" $data | nindent 4 }}
   name: {{ include "kamaji.webhookServiceName" . }}
   namespace: {{ .Release.Namespace }}
 spec:
   ports:
     - port: 443
       protocol: TCP
+      name: webhook-server
       targetPort: webhook-server
   selector:
     {{- include "kamaji.selectorLabels" . | nindent 4 }}

charts/kamaji/templates/servicemonitor.yaml

@@ -0,0 +1,21 @@
+{{- if and (.Capabilities.APIVersions.Has "monitoring.coreos.com/v1") .Values.serviceMonitor.enabled }}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  labels:
+    {{- $data := . | mustMergeOverwrite (dict "component" "servicemonitor") -}}
+    {{- include "kamaji.labels" $data | nindent 4 }}
+  name: {{ include "kamaji.fullname" . }}
+  namespace: {{ .Release.Namespace }}
+spec:
+  endpoints:
+    - path: /metrics
+      port: metrics
+      scheme: http
+  namespaceSelector:
+    matchNames:
+      - {{ .Release.Namespace }}
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: {{ include "kamaji.name" . }}
+{{- end }}

charts/kamaji/templates/validatingwebhookconfiguration.yaml

@@ -4,10 +4,31 @@ metadata:
   annotations:
     cert-manager.io/inject-ca-from: {{ .Release.Namespace }}/{{ include "kamaji.certificateName" . }}
   labels:
-    {{- include "kamaji.labels" . | nindent 4 }}
-    app.kubernetes.io/instance: validating-webhook-configuration
+    {{- $data := . | mustMergeOverwrite (dict "instance" "validating-webhook-configuration") -}}
+    {{- include "kamaji.labels" $data | nindent 4 }}
   name: kamaji-validating-webhook-configuration
 webhooks:
+  - admissionReviewVersions:
+      - v1
+    clientConfig:
+      service:
+        name: {{ include "kamaji.webhookServiceName" . }}
+        namespace: {{ .Release.Namespace }}
+        path: /telemetry
+    failurePolicy: Ignore
+    name: telemetry.kamaji.clastix.io
+    rules:
+      - apiGroups:
+          - kamaji.clastix.io
+        apiVersions:
+          - v1alpha1
+        operations:
+          - CREATE
+          - UPDATE
+          - DELETE
+        resources:
+          - tenantcontrolplanes
+    sideEffects: None
   - admissionReviewVersions:
       - v1
     clientConfig:

charts/kamaji/values.yaml

@@ -15,6 +15,11 @@ image:
 # -- A list of extra arguments to add to the kamaji controller default ones
 extraArgs: []
 
+
+serviceMonitor:
+  # -- Toggle the ServiceMonitor true if you have Prometheus Operator installed and configured
+  enabled: false
+
 etcd:
   # -- Install an etcd with enabled multi-tenancy along with Kamaji
   deploy: true
@@ -49,9 +54,15 @@ etcd:
     name: ""
   persistence:
     size: 10Gi
-    storageClass: ""
+    storageClassName: ""
     accessModes:
     - ReadWriteOnce
+    # -- The custom annotations to add to the PVC
+    customAnnotations: {}
+    #  volumeType: local
+
+  # -- (array) Kubernetes affinity rules to apply to Kamaji etcd pods
+  tolerations: []
 
   overrides:
     caSecret:
@@ -149,7 +160,9 @@ loggingDevel:
   enable: false
 
 datastore:
-  # -- (string) The Datastore name override, if empty defaults to `default`
+  # -- (bool) Enable the Kamaji Datastore creation (default=true)
+  enabled: true
+  # -- (string) The Datastore name override, if empty and enabled=true defaults to `default`, if enabled=false, this is the name of the Datastore to connect to.
   nameOverride:
   # -- (string) The Kamaji Datastore driver, supported: etcd, MySQL, PostgreSQL (defaults=etcd).
   driver: etcd
@@ -171,6 +184,7 @@ datastore:
       # -- The Secret key where the data is stored.
       keyPath:
   tlsConfig:
+    enabled: true
     certificateAuthority:
       certificate:
         # -- Name of the Secret containing the CA required to establish the mandatory SSL/TLS connection to the datastore.
@@ -201,3 +215,13 @@ datastore:
         namespace:
         # -- Key of the Secret which contains the content of the private key.
         keyPath:
+
+cfssl:
+  image:
+    repository: cfssl/cfssl
+    tag: latest
+
+# -- Disable the analytics traces collection
+telemetry:
+  disabled: false 
+  

cmd/manager/cmd.go

@@ -7,40 +7,55 @@ import (
 	"flag"
 	"fmt"
 	"io"
+	"net/http"
 	"os"
 	goRuntime "runtime"
+	"time"
 
+	telemetryclient "github.com/clastix/kamaji-telemetry/pkg/client"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/rest"
 	"k8s.io/klog/v2"
 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/cache"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
+	ctrlwebhook "sigs.k8s.io/controller-runtime/pkg/webhook"
 
 	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
 	cmdutils "github.com/clastix/kamaji/cmd/utils"
 	"github.com/clastix/kamaji/controllers"
 	"github.com/clastix/kamaji/controllers/soot"
 	"github.com/clastix/kamaji/internal"
+	"github.com/clastix/kamaji/internal/builders/controlplane"
 	datastoreutils "github.com/clastix/kamaji/internal/datastore/utils"
 	"github.com/clastix/kamaji/internal/webhook"
+	"github.com/clastix/kamaji/internal/webhook/handlers"
+	"github.com/clastix/kamaji/internal/webhook/routes"
 )
 
+//nolint:maintidx
 func NewCmd(scheme *runtime.Scheme) *cobra.Command {
 	// CLI flags
 	var (
-		metricsBindAddress        string
-		healthProbeBindAddress    string
-		leaderElect               bool
-		tmpDirectory              string
-		kineImage                 string
-		datastore                 string
-		managerNamespace          string
-		managerServiceAccountName string
-		managerServiceName        string
-		webhookCABundle           []byte
-		migrateJobImage           string
+		metricsBindAddress         string
+		healthProbeBindAddress     string
+		leaderElect                bool
+		tmpDirectory               string
+		kineImage                  string
+		controllerReconcileTimeout time.Duration
+		cacheResyncPeriod          time.Duration
+		datastore                  string
+		managerNamespace           string
+		managerServiceAccountName  string
+		managerServiceName         string
+		webhookCABundle            []byte
+		migrateJobImage            string
+		maxConcurrentReconciles    int
+		disableTelemetry           bool
 
 		webhookCAPath string
 	)
@@ -69,6 +84,10 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command {
 				return err
 			}
 
+			if controllerReconcileTimeout.Seconds() == 0 {
+				return fmt.Errorf("the controller reconcile timeout must be greater than zero")
+			}
+
 			return nil
 		},
 		RunE: func(cmd *cobra.Command, args []string) error {
@@ -79,25 +98,42 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command {
 			setupLog.Info(fmt.Sprintf("Build date: %s", internal.BuildTime))
 			setupLog.Info(fmt.Sprintf("Go Version: %s", goRuntime.Version()))
 			setupLog.Info(fmt.Sprintf("Go OS/Arch: %s/%s", goRuntime.GOOS, goRuntime.GOARCH))
+			setupLog.Info(fmt.Sprintf("Telemetry enabled: %t", !disableTelemetry))
 
-			mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
-				Scheme:                  scheme,
-				MetricsBindAddress:      metricsBindAddress,
-				Port:                    9443,
+			telemetryClient := telemetryclient.New(http.Client{Timeout: 5 * time.Second}, "https://telemetry.clastix.io")
+			if disableTelemetry {
+				telemetryClient = telemetryclient.NewNewOp()
+			}
+
+			ctrlOpts := ctrl.Options{
+				Scheme: scheme,
+				Metrics: metricsserver.Options{
+					BindAddress: metricsBindAddress,
+				},
+				WebhookServer: ctrlwebhook.NewServer(ctrlwebhook.Options{
+					Port: 9443,
+				}),
 				HealthProbeBindAddress:  healthProbeBindAddress,
 				LeaderElection:          leaderElect,
 				LeaderElectionNamespace: managerNamespace,
-				LeaderElectionID:        "799b98bc.clastix.io",
-			})
+				LeaderElectionID:        "kamaji.clastix.io",
+				NewCache: func(config *rest.Config, opts cache.Options) (cache.Cache, error) {
+					opts.SyncPeriod = &cacheResyncPeriod
+
+					return cache.New(config, opts)
+				},
+			}
+
+			mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrlOpts)
 			if err != nil {
 				setupLog.Error(err, "unable to start manager")
 
 				return err
 			}
 
-			tcpChannel := make(controllers.TenantControlPlaneChannel)
+			tcpChannel, certChannel := make(controllers.TenantControlPlaneChannel), make(controllers.CertificateChannel)
 
-			if err = (&controllers.DataStore{TenantControlPlaneTrigger: tcpChannel}).SetupWithManager(mgr); err != nil {
+			if err = (&controllers.DataStore{Client: mgr.GetClient(), TenantControlPlaneTrigger: tcpChannel}).SetupWithManager(mgr); err != nil {
 				setupLog.Error(err, "unable to create controller", "controller", "DataStore")
 
 				return err
@@ -107,15 +143,18 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command {
 				Client:    mgr.GetClient(),
 				APIReader: mgr.GetAPIReader(),
 				Config: controllers.TenantControlPlaneReconcilerConfig{
+					ReconcileTimeout:     controllerReconcileTimeout,
 					DefaultDataStoreName: datastore,
 					KineContainerImage:   kineImage,
 					TmpBaseDirectory:     tmpDirectory,
 				},
-				TriggerChan:          tcpChannel,
-				KamajiNamespace:      managerNamespace,
-				KamajiServiceAccount: managerServiceAccountName,
-				KamajiService:        managerServiceName,
-				KamajiMigrateImage:   migrateJobImage,
+				CertificateChan:         certChannel,
+				TriggerChan:             tcpChannel,
+				KamajiNamespace:         managerNamespace,
+				KamajiServiceAccount:    managerServiceAccountName,
+				KamajiService:           managerServiceName,
+				KamajiMigrateImage:      migrateJobImage,
+				MaxConcurrentReconciles: maxConcurrentReconciles,
 			}
 
 			if err = reconciler.SetupWithManager(mgr); err != nil {
@@ -124,8 +163,31 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command {
 				return err
 			}
 
-			if err = (&webhook.Freeze{}).SetupWithManager(mgr); err != nil {
-				setupLog.Error(err, "unable to register webhook", "webhook", "Freeze")
+			k8sVersion, versionErr := cmdutils.KubernetesVersion(mgr.GetConfig())
+			if versionErr != nil {
+				setupLog.Error(err, "unable to get kubernetes version")
+
+				k8sVersion = "Unknown"
+			}
+
+			if !disableTelemetry {
+				err = mgr.Add(&controllers.TelemetryController{
+					Client:                  mgr.GetClient(),
+					KubernetesVersion:       k8sVersion,
+					KamajiVersion:           internal.GitTag,
+					TelemetryClient:         telemetryClient,
+					LeaderElectionNamespace: ctrlOpts.LeaderElectionNamespace,
+					LeaderElectionID:        ctrlOpts.LeaderElectionID,
+				})
+				if err != nil {
+					setupLog.Error(err, "unable to create controller", "controller", "TelemetryController")
+
+					return err
+				}
+			}
+
+			if err = (&controllers.CertificateLifecycle{Channel: certChannel}).SetupWithManager(mgr); err != nil {
+				setupLog.Error(err, "unable to create controller", "controller", "CertificateLifecycle")
 
 				return err
 			}
@@ -142,13 +204,49 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command {
 				return err
 			}
 
-			if err = (&kamajiv1alpha1.TenantControlPlane{}).SetupWebhookWithManager(mgr, datastore); err != nil {
-				setupLog.Error(err, "unable to create webhook", "webhook", "TenantControlPlane")
-
-				return err
-			}
-			if err = (&kamajiv1alpha1.DataStore{}).SetupWebhookWithManager(mgr); err != nil {
-				setupLog.Error(err, "unable to create webhook", "webhook", "DataStore")
+			err = webhook.Register(mgr, map[routes.Route][]handlers.Handler{
+				routes.TenantControlPlaneMigrate{}: {
+					handlers.Freeze{},
+				},
+				routes.TenantControlPlaneDefaults{}: {
+					handlers.TenantControlPlaneDefaults{
+						DefaultDatastore: datastore,
+					},
+				},
+				routes.TenantControlPlaneValidate{}: {
+					handlers.TenantControlPlaneName{},
+					handlers.TenantControlPlaneVersion{},
+					handlers.TenantControlPlaneKubeletAddresses{},
+					handlers.TenantControlPlaneDataStore{Client: mgr.GetClient()},
+					handlers.TenantControlPlaneDeployment{
+						Client: mgr.GetClient(),
+						DeploymentBuilder: controlplane.Deployment{
+							Client:             mgr.GetClient(),
+							KineContainerImage: kineImage,
+						},
+						KonnectivityBuilder: controlplane.Konnectivity{
+							Scheme: *mgr.GetScheme(),
+						},
+					},
+					handlers.TenantControlPlaneServiceCIDR{},
+				},
+				routes.TenantControlPlaneTelemetry{}: {
+					handlers.TenantControlPlaneTelemetry{
+						Enabled:           !disableTelemetry,
+						TelemetryClient:   telemetryClient,
+						KamajiVersion:     internal.GitTag,
+						KubernetesVersion: k8sVersion,
+					},
+				},
+				routes.DataStoreValidate{}: {
+					handlers.DataStoreValidation{Client: mgr.GetClient()},
+				},
+				routes.DataStoreSecrets{}: {
+					handlers.DataStoreSecretValidation{Client: mgr.GetClient()},
+				},
+			})
+			if err != nil {
+				setupLog.Error(err, "unable to create webhook")
 
 				return err
 			}
@@ -185,6 +283,7 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command {
 			return nil
 		},
 	}
+
 	// Setting zap logger
 	zapfs := flag.NewFlagSet("zap", flag.ExitOnError)
 	opts := zap.Options{
@@ -200,11 +299,15 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command {
 	cmd.Flags().StringVar(&tmpDirectory, "tmp-directory", "/tmp/kamaji", "Directory which will be used to work with temporary files.")
 	cmd.Flags().StringVar(&kineImage, "kine-image", "rancher/kine:v0.9.2-amd64", "Container image along with tag to use for the Kine sidecar container (used only if etcd-storage-type is set to one of kine strategies).")
 	cmd.Flags().StringVar(&datastore, "datastore", "etcd", "The default DataStore that should be used by Kamaji to setup the required storage.")
-	cmd.Flags().StringVar(&migrateJobImage, "migrate-image", fmt.Sprintf("clastix/kamaji:v%s", internal.GitTag), "Specify the container image to launch when a TenantControlPlane is migrated to a new datastore.")
+	cmd.Flags().StringVar(&migrateJobImage, "migrate-image", fmt.Sprintf("clastix/kamaji:%s", internal.GitTag), "Specify the container image to launch when a TenantControlPlane is migrated to a new datastore.")
+	cmd.Flags().IntVar(&maxConcurrentReconciles, "max-concurrent-tcp-reconciles", 1, "Specify the number of workers for the Tenant Control Plane controller (beware of CPU consumption)")
 	cmd.Flags().StringVar(&managerNamespace, "pod-namespace", os.Getenv("POD_NAMESPACE"), "The Kubernetes Namespace on which the Operator is running in, required for the TenantControlPlane migration jobs.")
 	cmd.Flags().StringVar(&managerServiceName, "webhook-service-name", "kamaji-webhook-service", "The Kamaji webhook server Service name which is used to get validation webhooks, required for the TenantControlPlane migration jobs.")
 	cmd.Flags().StringVar(&managerServiceAccountName, "serviceaccount-name", os.Getenv("SERVICE_ACCOUNT"), "The Kubernetes Namespace on which the Operator is running in, required for the TenantControlPlane migration jobs.")
 	cmd.Flags().StringVar(&webhookCAPath, "webhook-ca-path", "/tmp/k8s-webhook-server/serving-certs/ca.crt", "Path to the Manager webhook server CA, required for the TenantControlPlane migration jobs.")
+	cmd.Flags().DurationVar(&controllerReconcileTimeout, "controller-reconcile-timeout", 30*time.Second, "The reconciliation request timeout before the controller withdraw the external resource calls, such as dealing with the Datastore, or the Tenant Control Plane API endpoint.")
+	cmd.Flags().DurationVar(&cacheResyncPeriod, "cache-resync-period", 10*time.Hour, "The controller-runtime.Manager cache resync period.")
+	cmd.Flags().BoolVar(&disableTelemetry, "disable-telemetry", false, "Disable the analytics traces collection.")
 
 	cobra.OnInitialize(func() {
 		viper.AutomaticEnv()

cmd/root.go

@@ -4,13 +4,12 @@
 package cmd
 
 import (
-	"math/rand"
-	"time"
-
 	"github.com/spf13/cobra"
+	_ "go.uber.org/automaxprocs" // Automatically set `GOMAXPROCS` to match Linux container CPU quota.
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	appsv1 "k8s.io/kubernetes/pkg/apis/apps/v1"
 
 	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
 )
@@ -20,11 +19,9 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command {
 		Use:   "kamaji",
 		Short: "Build and operate Kubernetes at scale with a fraction of operational burden.",
 		PersistentPreRun: func(cmd *cobra.Command, args []string) {
-			// Seed is required to ensure non reproducibility for the certificates generate by Kamaji.
-			rand.Seed(time.Now().UnixNano())
-
 			utilruntime.Must(clientgoscheme.AddToScheme(scheme))
 			utilruntime.Must(kamajiv1alpha1.AddToScheme(scheme))
+			utilruntime.Must(appsv1.RegisterDefaults(scheme))
 		},
 	}
 }

cmd/utils/k8s_version.go

@@ -0,0 +1,24 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package utils
+
+import (
+	"github.com/pkg/errors"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+)
+
+func KubernetesVersion(config *rest.Config) (string, error) {
+	cs, csErr := kubernetes.NewForConfig(config)
+	if csErr != nil {
+		return "", errors.Wrap(csErr, "cannot create kubernetes clientset")
+	}
+
+	sv, svErr := cs.ServerVersion()
+	if svErr != nil {
+		return "", errors.Wrap(svErr, "cannot get Kubernetes version")
+	}
+
+	return sv.GitVersion, nil
+}

config/crd/bases/kamaji.clastix.io_datastores.yaml

@@ -3,8 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.9.2
-  creationTimestamp: null
+    controller-gen.kubebuilder.io/version: v0.14.0
   name: datastores.kamaji.clastix.io
 spec:
   group: kamaji.clastix.io
@@ -30,14 +29,19 @@ spec:
         description: DataStore is the Schema for the datastores API.
         properties:
           apiVersion:
-            description: 'APIVersion defines the versioned schema of this representation
-              of an object. Servers should convert recognized schemas to the latest
-              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
             type: string
           kind:
-            description: 'Kind is a string value representing the REST resource this
-              object represents. Servers may infer this from the endpoint the client
-              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
             type: string
           metadata:
             type: object
@@ -45,21 +49,24 @@ spec:
             description: DataStoreSpec defines the desired state of DataStore.
             properties:
               basicAuth:
-                description: In case of authentication enabled for the given data
-                  store, specifies the username and password pair. This value is optional.
+                description: |-
+                  In case of authentication enabled for the given data store, specifies the username and password pair.
+                  This value is optional.
                 properties:
                   password:
                     properties:
                       content:
-                        description: Bare content of the file, base64 encoded. It
-                          has precedence over the SecretReference value.
+                        description: |-
+                          Bare content of the file, base64 encoded.
+                          It has precedence over the SecretReference value.
                         format: byte
                         type: string
                       secretReference:
                         properties:
                           keyPath:
-                            description: Name of the key for the given Secret reference
-                              where the content is stored. This value is mandatory.
+                            description: |-
+                              Name of the key for the given Secret reference where the content is stored.
+                              This value is mandatory.
                             minLength: 1
                             type: string
                           name:
@@ -78,15 +85,17 @@ spec:
                   username:
                     properties:
                       content:
-                        description: Bare content of the file, base64 encoded. It
-                          has precedence over the SecretReference value.
+                        description: |-
+                          Bare content of the file, base64 encoded.
+                          It has precedence over the SecretReference value.
                         format: byte
                         type: string
                       secretReference:
                         properties:
                           keyPath:
-                            description: Name of the key for the given Secret reference
-                              where the content is stored. This value is mandatory.
+                            description: |-
+                              Name of the key for the given Secret reference where the content is stored.
+                              This value is mandatory.
                             minLength: 1
                             type: string
                           name:
@@ -112,37 +121,40 @@ spec:
                 - etcd
                 - MySQL
                 - PostgreSQL
+                - NATS
                 type: string
               endpoints:
-                description: List of the endpoints to connect to the shared datastore.
+                description: |-
+                  List of the endpoints to connect to the shared datastore.
                   No need for protocol, just bare IP/FQDN and port.
                 items:
                   type: string
                 minItems: 1
                 type: array
               tlsConfig:
-                description: Defines the TLS/SSL configuration required to connect
-                  to the data store in a secure way.
+                description: |-
+                  Defines the TLS/SSL configuration required to connect to the data store in a secure way.
+                  This value is optional.
                 properties:
                   certificateAuthority:
-                    description: Retrieve the Certificate Authority certificate and
-                      private key, such as bare content of the file, or a SecretReference.
-                      The key reference is required since etcd authentication is based
-                      on certificates, and Kamaji is responsible in creating this.
+                    description: |-
+                      Retrieve the Certificate Authority certificate and private key, such as bare content of the file, or a SecretReference.
+                      The key reference is required since etcd authentication is based on certificates, and Kamaji is responsible in creating this.
                     properties:
                       certificate:
                         properties:
                           content:
-                            description: Bare content of the file, base64 encoded.
+                            description: |-
+                              Bare content of the file, base64 encoded.
                               It has precedence over the SecretReference value.
                             format: byte
                             type: string
                           secretReference:
                             properties:
                               keyPath:
-                                description: Name of the key for the given Secret
-                                  reference where the content is stored. This value
-                                  is mandatory.
+                                description: |-
+                                  Name of the key for the given Secret reference where the content is stored.
+                                  This value is mandatory.
                                 minLength: 1
                                 type: string
                               name:
@@ -161,16 +173,17 @@ spec:
                       privateKey:
                         properties:
                           content:
-                            description: Bare content of the file, base64 encoded.
+                            description: |-
+                              Bare content of the file, base64 encoded.
                               It has precedence over the SecretReference value.
                             format: byte
                             type: string
                           secretReference:
                             properties:
                               keyPath:
-                                description: Name of the key for the given Secret
-                                  reference where the content is stored. This value
-                                  is mandatory.
+                                description: |-
+                                  Name of the key for the given Secret reference where the content is stored.
+                                  This value is mandatory.
                                 minLength: 1
                                 type: string
                               name:
@@ -196,16 +209,17 @@ spec:
                       certificate:
                         properties:
                           content:
-                            description: Bare content of the file, base64 encoded.
+                            description: |-
+                              Bare content of the file, base64 encoded.
                               It has precedence over the SecretReference value.
                             format: byte
                             type: string
                           secretReference:
                             properties:
                               keyPath:
-                                description: Name of the key for the given Secret
-                                  reference where the content is stored. This value
-                                  is mandatory.
+                                description: |-
+                                  Name of the key for the given Secret reference where the content is stored.
+                                  This value is mandatory.
                                 minLength: 1
                                 type: string
                               name:
@@ -224,16 +238,17 @@ spec:
                       privateKey:
                         properties:
                           content:
-                            description: Bare content of the file, base64 encoded.
+                            description: |-
+                              Bare content of the file, base64 encoded.
                               It has precedence over the SecretReference value.
                             format: byte
                             type: string
                           secretReference:
                             properties:
                               keyPath:
-                                description: Name of the key for the given Secret
-                                  reference where the content is stored. This value
-                                  is mandatory.
+                                description: |-
+                                  Name of the key for the given Secret reference where the content is stored.
+                                  This value is mandatory.
                                 minLength: 1
                                 type: string
                               name:
@@ -255,12 +270,10 @@ spec:
                     type: object
                 required:
                 - certificateAuthority
-                - clientCertificate
                 type: object
             required:
             - driver
             - endpoints
-            - tlsConfig
             type: object
           status:
             description: DataStoreStatus defines the observed state of DataStore.

config/default/kustomization.yaml

@@ -9,8 +9,8 @@ namespace: kamaji-system
 namePrefix: kamaji-
 
 # Labels to add to all resources and selectors.
-#commonLabels:
-#  someName: someValue
+commonLabels:
+  cluster.x-k8s.io/provider: "kamaji-core"
 
 bases:
 - ../crd
@@ -19,8 +19,7 @@ bases:
 - ../samples
 - ../webhook
 - ../certmanager
-# [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'.
-#- ../prometheus
+- ../prometheus
 
 patchesStrategicMerge:
 # Mount the controller config file for loading manager configurations

config/manager/kustomization.yaml

@@ -13,4 +13,4 @@ kind: Kustomization
 images:
 - name: controller
   newName: clastix/kamaji
-  newTag: v0.1.1
+  newTag: v1.0.0

config/manager/manager.yaml

@@ -30,6 +30,7 @@ spec:
         args:
         - manager
         - --leader-elect
+        - --datastore=kamaji-etcd
         env:
           - name: POD_NAMESPACE
             valueFrom:
@@ -42,6 +43,10 @@ spec:
         image: controller:latest
         imagePullPolicy: Always
         name: manager
+        ports:
+          - containerPort: 8080
+            name: metrics
+            protocol: TCP
         securityContext:
           allowPrivilegeEscalation: false
         livenessProbe:

config/metadata.yaml

@@ -0,0 +1,10 @@
+# maps release series of major.minor to cluster-api contract version
+# the contract version may change between minor or major versions, but *not*
+# between patch versions.
+#
+# update this file only when a new major or minor version is released
+apiVersion: clusterctl.cluster.x-k8s.io/v1alpha3
+releaseSeries:
+  - major: 0
+    minor: 3
+    contract: v1beta1

config/prometheus/kustomization.yaml

@@ -1,2 +1,5 @@
 resources:
 - monitor.yaml
+
+configurations:
+- kustomizeconfig.yaml

config/prometheus/kustomizeconfig.yaml

@@ -0,0 +1,4 @@
+varReference:
+  - kind: ServiceMonitor
+    group: monitoring.coreos.com
+    path: spec/namespaceSelector/matchNames

config/prometheus/monitor.yaml

@@ -1,4 +1,3 @@
-
 # Prometheus Monitor Service (Metrics)
 apiVersion: monitoring.coreos.com/v1
 kind: ServiceMonitor
@@ -10,11 +9,11 @@ metadata:
 spec:
   endpoints:
     - path: /metrics
-      port: https
-      scheme: https
-      bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
-      tlsConfig:
-        insecureSkipVerify: true
+      port: metrics
+      scheme: http
+  namespaceSelector:
+    matchNames:
+      - $(SERVICE_NAMESPACE)
   selector:
     matchLabels:
       control-plane: controller-manager

config/rbac/role.yaml

@@ -2,7 +2,6 @@
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
-  creationTimestamp: null
   name: manager-role
 rules:
 - apiGroups:

config/samples/kamaji_v1alpha1_datastore_nats_bronze.yaml

@@ -0,0 +1,34 @@
+apiVersion: kamaji.clastix.io/v1alpha1
+kind: DataStore
+metadata:
+  name: nats-bronze
+spec:
+  driver: NATS
+  endpoints:
+    - bronze-nats.nats-system.svc:4222
+  basicAuth:
+    username:
+      content: YWRtaW4=
+    password:
+      secretReference:
+        name: nats-bronze-config
+        namespace: nats-system
+        keyPath: password
+  tlsConfig:
+    certificateAuthority:
+      certificate:
+        secretReference:
+          name: nats-bronze-config
+          namespace: nats-system
+          keyPath: ca.crt
+    clientCertificate:
+      certificate:
+        secretReference:
+          name: nats-bronze-config
+          namespace: nats-system
+          keyPath: server.crt
+      privateKey:
+        secretReference:
+          name: nats-bronze-config
+          namespace: nats-system
+          keyPath: server.key

config/samples/kamaji_v1alpha1_datastore_nats_gold.yaml

@@ -0,0 +1,34 @@
+apiVersion: kamaji.clastix.io/v1alpha1
+kind: DataStore
+metadata:
+  name: nats-gold
+spec:
+  driver: NATS
+  endpoints:
+    - nats-gold.nats-system.svc:4222
+  basicAuth:
+    username:
+      content: YWRtaW4=
+    password:
+      secretReference:
+        name: nats-gold-config
+        namespace: nats-system
+        keyPath: password
+  tlsConfig:
+    certificateAuthority:
+      certificate:
+        secretReference:
+          name: nats-gold-config
+          namespace: nats-system
+          keyPath: ca.crt
+    clientCertificate:
+      certificate:
+        secretReference:
+          name: nats-gold-config
+          namespace: nats-system
+          keyPath: server.crt
+      privateKey:
+        secretReference:
+          name: nats-gold-config
+          namespace: nats-system
+          keyPath: server.key

config/samples/kamaji_v1alpha1_datastore_nats_notls.yaml

@@ -0,0 +1,17 @@
+apiVersion: kamaji.clastix.io/v1alpha1
+kind: DataStore
+metadata:
+  name: nats-notls
+spec:
+  driver: NATS
+  endpoints:
+    - notls-nats.nats-system.svc:4222
+  basicAuth:
+    username:
+      content: YWRtaW4=
+    password:
+      secretReference:
+        name: nats-notls-config
+        namespace: nats-system
+        keyPath: password
+

config/samples/kamaji_v1alpha1_datastore_nats_silver.yaml

@@ -0,0 +1,34 @@
+apiVersion: kamaji.clastix.io/v1alpha1
+kind: DataStore
+metadata:
+  name: nats-silver
+spec:
+  driver: NATS
+  endpoints:
+    - nats-silver.nats-system.svc:4222
+  basicAuth:
+    username:
+      content: YWRtaW4=
+    password:
+      secretReference:
+        name: nats-silver-config
+        namespace: nats-system
+        keyPath: password
+  tlsConfig:
+    certificateAuthority:
+      certificate:
+        secretReference:
+          name: nats-silver-config
+          namespace: nats-system
+          keyPath: ca.crt
+    clientCertificate:
+      certificate:
+        secretReference:
+          name: nats-silver-config
+          namespace: nats-system
+          keyPath: server.crt
+      privateKey:
+        secretReference:
+          name: nats-silver-config
+          namespace: nats-system
+          keyPath: server.key

config/samples/kamaji_v1alpha1_tenantcontrolplane.yaml

@@ -1,22 +1,24 @@
 apiVersion: kamaji.clastix.io/v1alpha1
 kind: TenantControlPlane
 metadata:
-  name: test
+  name: k8s-126
+  labels:
+    tenant.clastix.io: k8s-126
 spec:
   controlPlane:
     deployment:
-      replicas: 1
+      replicas: 2
     service:
       serviceType: LoadBalancer
   kubernetes:
-    version: "v1.25.4"
+    version: "v1.26.0"
     kubelet:
-      cgroupfs: cgroupfs
-    admissionControllers:
-      - ResourceQuota
-      - LimitRanger
+      cgroupfs: systemd
   networkProfile:
     port: 6443
   addons:
     coreDNS: {}
     kubeProxy: {}
+    konnectivity:
+      server:
+        port: 8132

config/samples/kamaji_v1alpha1_tenantcontrolplane_additionalcontainers.yaml

@@ -0,0 +1,32 @@
+apiVersion: kamaji.clastix.io/v1alpha1
+kind: TenantControlPlane
+metadata:
+  name: additionalcontainers
+  labels:
+    tenant.clastix.io: additionalcontainers
+spec:
+  dataStore: postgresql-bronze
+  controlPlane:
+    deployment:
+      replicas: 1
+      additionalInitContainers:
+        - name: init
+          image: registry.k8s.io/e2e-test-images/busybox:1.29-4
+          command:
+            - /bin/sh
+            - -c
+            - echo hello world
+      additionalContainers:
+        - name: nginx
+          image: registry.k8s.io/e2e-test-images/nginx:1.15-4
+    service:
+      serviceType: LoadBalancer
+  kubernetes:
+    version: "v1.26.0"
+    kubelet:
+      cgroupfs: systemd
+  networkProfile:
+    port: 6443
+  addons:
+    coreDNS: {}
+    kubeProxy: {}

config/samples/kamaji_v1alpha1_tenantcontrolplane_additionalvolumes.yaml

@@ -0,0 +1,62 @@
+apiVersion: kamaji.clastix.io/v1alpha1
+kind: TenantControlPlane
+metadata:
+  name: additional-volumes
+  labels:
+    tenant.clastix.io: additional-volumes
+spec:
+  controlPlane:
+    deployment:
+      replicas: 1
+      additionalVolumes:
+        - name: api-server-volume
+          configMap:
+            name: api-server-extra-cm
+        - name: controller-manager-volume
+          configMap:
+            name: controller-manager-extra-cm
+        - name: scheduler-volume
+          configMap:
+            name: scheduler-extra-cm
+      additionalVolumeMounts:
+        apiServer:
+          - name: api-server-volume
+            mountPath: "/tmp/api-server"
+        controllerManager:
+          - name: controller-manager-volume
+            mountPath: "/tmp/controller-manager"
+        scheduler:
+          - name: scheduler-volume
+            mountPath: "/tmp/scheduler"
+    service:
+      serviceType: LoadBalancer
+  kubernetes:
+    version: "v1.26.0"
+    kubelet:
+      cgroupfs: systemd
+  networkProfile:
+    port: 6443
+  addons:
+    coreDNS: {}
+    kubeProxy: {}
+---
+apiVersion: v1
+data:
+  api-server: "This is an API Server volume"
+kind: ConfigMap
+metadata:
+  name: api-server-extra-cm
+---
+apiVersion: v1
+data:
+  controller-manager: "This is a Controller Manager volume"
+kind: ConfigMap
+metadata:
+  name: controller-manager-extra-cm
+---
+apiVersion: v1
+data:
+  controller-manager: "This is a Scheduler volume"
+kind: ConfigMap
+metadata:
+  name: scheduler-extra-cm

config/samples/kamaji_v1alpha1_tenantcontrolplane_kine.yaml

@@ -0,0 +1,20 @@
+apiVersion: kamaji.clastix.io/v1alpha1
+kind: TenantControlPlane
+metadata:
+  name: kine
+  labels:
+    tenant.clastix.io: kine
+spec:
+  addons:
+    coreDNS: {}
+    kubeProxy: {}
+  controlPlane:
+    deployment:
+      replicas: 1
+    service:
+      serviceType: LoadBalancer
+  dataStore: postgresql-bronze
+  kubernetes:
+    kubelet:
+      cgroupfs: systemd
+    version: v1.26.0

config/samples/kamaji_v1alpha1_tenantcontrolplane_konnectivity.yaml

@@ -0,0 +1,23 @@
+apiVersion: kamaji.clastix.io/v1alpha1
+kind: TenantControlPlane
+metadata:
+  name: konnectivity-addon
+  labels:
+    tenant.clastix.io: konnectivity-addon
+spec:
+  deployment:
+    replicas: 2
+  service:
+    serviceType: LoadBalancer
+  kubernetes:
+    version: "v1.26.0"
+    kubelet:
+      cgroupfs: systemd
+  networkProfile:
+    port: 6443
+  addons:
+    coreDNS: {}
+    kubeProxy: {}
+    konnectivity:
+      server:
+        port: 8132

config/webhook/manifests.yaml

@@ -2,29 +2,8 @@
 apiVersion: admissionregistration.k8s.io/v1
 kind: MutatingWebhookConfiguration
 metadata:
-  creationTimestamp: null
   name: mutating-webhook-configuration
 webhooks:
-- admissionReviewVersions:
-  - v1
-  clientConfig:
-    service:
-      name: webhook-service
-      namespace: system
-      path: /mutate-kamaji-clastix-io-v1alpha1-datastore
-  failurePolicy: Fail
-  name: mdatastore.kb.io
-  rules:
-  - apiGroups:
-    - kamaji.clastix.io
-    apiVersions:
-    - v1alpha1
-    operations:
-    - CREATE
-    - UPDATE
-    resources:
-    - datastores
-  sideEffects: None
 - admissionReviewVersions:
   - v1
   clientConfig:
@@ -49,7 +28,6 @@ webhooks:
 apiVersion: admissionregistration.k8s.io/v1
 kind: ValidatingWebhookConfiguration
 metadata:
-  creationTimestamp: null
   name: validating-webhook-configuration
 webhooks:
 - admissionReviewVersions:
@@ -58,18 +36,20 @@ webhooks:
     service:
       name: webhook-service
       namespace: system
-      path: /validate--v1-secret
+      path: /telemetry
   failurePolicy: Ignore
-  name: vdatastoresecrets.kb.io
+  name: telemetry.kamaji.clastix.io
   rules:
   - apiGroups:
-    - ""
+    - kamaji.clastix.io
     apiVersions:
-    - v1
+    - v1alpha1
     operations:
+    - CREATE
+    - UPDATE
     - DELETE
     resources:
-    - secrets
+    - tenantcontrolplanes
   sideEffects: None
 - admissionReviewVersions:
   - v1
@@ -92,6 +72,25 @@ webhooks:
     resources:
     - datastores
   sideEffects: None
+- admissionReviewVersions:
+  - v1
+  clientConfig:
+    service:
+      name: webhook-service
+      namespace: system
+      path: /validate--v1-secret
+  failurePolicy: Ignore
+  name: vdatastoresecrets.kb.io
+  rules:
+  - apiGroups:
+    - ""
+    apiVersions:
+    - v1
+    operations:
+    - DELETE
+    resources:
+    - secrets
+  sideEffects: None
 - admissionReviewVersions:
   - v1
   clientConfig:

controllers/cert_channel.go

@@ -0,0 +1,10 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package controllers
+
+import (
+	"sigs.k8s.io/controller-runtime/pkg/event"
+)
+
+type CertificateChannel chan event.GenericEvent

controllers/certificate_lifecycle_controller.go

@@ -0,0 +1,163 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package controllers
+
+import (
+	"context"
+	"crypto/x509"
+	"fmt"
+	"time"
+
+	"github.com/pkg/errors"
+	corev1 "k8s.io/api/core/v1"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/sets"
+	clientcmdapiv1 "k8s.io/client-go/tools/clientcmd/api/v1"
+	controllerruntime "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/builder"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/event"
+	"sigs.k8s.io/controller-runtime/pkg/log"
+	"sigs.k8s.io/controller-runtime/pkg/predicate"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+	"github.com/clastix/kamaji/internal/constants"
+	"github.com/clastix/kamaji/internal/crypto"
+	"github.com/clastix/kamaji/internal/utilities"
+)
+
+type CertificateLifecycle struct {
+	Channel CertificateChannel
+	client  client.Client
+}
+
+func (s *CertificateLifecycle) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) {
+	logger := log.FromContext(ctx)
+
+	logger.Info("starting CertificateLifecycle handling")
+
+	secret := corev1.Secret{}
+	err := s.client.Get(ctx, request.NamespacedName, &secret)
+	if k8serrors.IsNotFound(err) {
+		logger.Info("resource have been deleted, skipping")
+
+		return reconcile.Result{}, nil
+	}
+	if err != nil {
+		logger.Error(err, "cannot retrieve the required resource")
+
+		return reconcile.Result{}, err
+	}
+
+	checkType, ok := secret.GetLabels()[constants.ControllerLabelResource]
+	if !ok {
+		logger.Info("missing controller label, shouldn't happen")
+
+		return reconcile.Result{}, nil
+	}
+
+	var crt *x509.Certificate
+
+	switch checkType {
+	case "x509":
+		crt, err = s.extractCertificateFromBareSecret(secret)
+	case "kubeconfig":
+		crt, err = s.extractCertificateFromKubeconfig(secret)
+	default:
+		err = fmt.Errorf("unsupported strategy, %s", checkType)
+	}
+
+	if err != nil {
+		logger.Error(err, "skipping reconciliation")
+
+		return reconcile.Result{}, nil
+	}
+
+	deadline := time.Now().AddDate(0, 0, 1)
+
+	if deadline.After(crt.NotAfter) {
+		logger.Info("certificate near expiration, must be rotated")
+
+		s.Channel <- event.GenericEvent{Object: &kamajiv1alpha1.TenantControlPlane{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      secret.GetOwnerReferences()[0].Name,
+				Namespace: secret.Namespace,
+			},
+		}}
+
+		logger.Info("certificate rotation triggered")
+
+		return reconcile.Result{}, nil
+	}
+
+	after := crt.NotAfter.Sub(deadline)
+
+	logger.Info("certificate is still valid, enqueuing back", "after", after.String())
+
+	return reconcile.Result{Requeue: true, RequeueAfter: after}, nil
+}
+
+func (s *CertificateLifecycle) extractCertificateFromBareSecret(secret corev1.Secret) (*x509.Certificate, error) {
+	var crt *x509.Certificate
+	var err error
+
+	for _, v := range secret.Data {
+		if crt, err = crypto.ParseCertificateBytes(v); err == nil {
+			break
+		}
+	}
+
+	if crt == nil {
+		return nil, fmt.Errorf("none of the provided keys is containing a valid x509 certificate")
+	}
+
+	return crt, nil
+}
+
+func (s *CertificateLifecycle) extractCertificateFromKubeconfig(secret corev1.Secret) (*x509.Certificate, error) {
+	var kc *clientcmdapiv1.Config
+	var err error
+
+	for k := range secret.Data {
+		if kc, err = utilities.DecodeKubeconfig(secret, k); err == nil {
+			break
+		}
+	}
+
+	if kc == nil {
+		return nil, fmt.Errorf("none of the provided keys is containing a valid kubeconfig")
+	}
+
+	crt, err := crypto.ParseCertificateBytes(kc.AuthInfos[0].AuthInfo.ClientCertificateData)
+	if err != nil {
+		return nil, errors.Wrap(err, "cannot parse kubeconfig certificate bytes")
+	}
+
+	return crt, nil
+}
+
+func (s *CertificateLifecycle) SetupWithManager(mgr controllerruntime.Manager) error {
+	s.client = mgr.GetClient()
+
+	supportedStrategies := sets.New[string]("x509", "kubeconfig")
+
+	return controllerruntime.NewControllerManagedBy(mgr).
+		For(&corev1.Secret{}, builder.WithPredicates(predicate.NewPredicateFuncs(func(object client.Object) bool {
+			labels := object.GetLabels()
+
+			if labels == nil {
+				return false
+			}
+
+			value, ok := labels[constants.ControllerLabelResource]
+			if !ok {
+				return false
+			}
+
+			return supportedStrategies.Has(value)
+		}))).
+		Complete(s)
+}

controllers/datastore_controller.go

@@ -19,13 +19,12 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/predicate"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
-	"sigs.k8s.io/controller-runtime/pkg/source"
 
 	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
 )
 
 type DataStore struct {
-	client client.Client
+	Client client.Client
 	// TenantControlPlaneTrigger is the channel used to communicate across the controllers:
 	// if a Data Source is updated we have to be sure that the reconciliation of the certificates content
 	// for each Tenant Control Plane is put in place properly.
@@ -39,19 +38,21 @@ func (r *DataStore) Reconcile(ctx context.Context, request reconcile.Request) (r
 	log := log.FromContext(ctx)
 
 	ds := &kamajiv1alpha1.DataStore{}
-	if err := r.client.Get(ctx, request.NamespacedName, ds); err != nil {
-		if k8serrors.IsNotFound(err) {
-			return reconcile.Result{}, nil
-		}
+	err := r.Client.Get(ctx, request.NamespacedName, ds)
+	if k8serrors.IsNotFound(err) {
+		log.Info("resource have been deleted, skipping")
 
-		log.Error(err, "unable to retrieve the request")
+		return reconcile.Result{}, nil
+	}
+	if err != nil {
+		log.Error(err, "cannot retrieve the required resource")
 
 		return reconcile.Result{}, err
 	}
 
 	tcpList := kamajiv1alpha1.TenantControlPlaneList{}
 
-	if err := r.client.List(ctx, &tcpList, client.MatchingFieldsSelector{
+	if err := r.Client.List(ctx, &tcpList, client.MatchingFieldsSelector{
 		Selector: fields.OneTermEqualSelector(kamajiv1alpha1.TenantControlPlaneUsedDataStoreKey, ds.GetName()),
 	}); err != nil {
 		log.Error(err, "cannot retrieve list of the Tenant Control Plane using the following instance")
@@ -66,7 +67,7 @@ func (r *DataStore) Reconcile(ctx context.Context, request reconcile.Request) (r
 
 	ds.Status.UsedBy = tcpSets.List()
 
-	if err := r.client.Status().Update(ctx, ds); err != nil {
+	if err := r.Client.Status().Update(ctx, ds); err != nil {
 		log.Error(err, "cannot update the status for the given instance")
 
 		return reconcile.Result{}, err
@@ -81,12 +82,6 @@ func (r *DataStore) Reconcile(ctx context.Context, request reconcile.Request) (r
 	return reconcile.Result{}, nil
 }
 
-func (r *DataStore) InjectClient(client client.Client) error {
-	r.client = client
-
-	return nil
-}
-
 func (r *DataStore) SetupWithManager(mgr controllerruntime.Manager) error {
 	enqueueFn := func(tcp *kamajiv1alpha1.TenantControlPlane, limitingInterface workqueue.RateLimitingInterface) {
 		if dataStoreName := tcp.Status.Storage.DataStoreName; len(dataStoreName) > 0 {
@@ -102,15 +97,15 @@ func (r *DataStore) SetupWithManager(mgr controllerruntime.Manager) error {
 		For(&kamajiv1alpha1.DataStore{}, builder.WithPredicates(
 			predicate.ResourceVersionChangedPredicate{},
 		)).
-		Watches(&source.Kind{Type: &kamajiv1alpha1.TenantControlPlane{}}, handler.Funcs{
-			CreateFunc: func(createEvent event.CreateEvent, limitingInterface workqueue.RateLimitingInterface) {
+		Watches(&kamajiv1alpha1.TenantControlPlane{}, handler.Funcs{
+			CreateFunc: func(_ context.Context, createEvent event.CreateEvent, limitingInterface workqueue.RateLimitingInterface) {
 				enqueueFn(createEvent.Object.(*kamajiv1alpha1.TenantControlPlane), limitingInterface)
 			},
-			UpdateFunc: func(updateEvent event.UpdateEvent, limitingInterface workqueue.RateLimitingInterface) {
+			UpdateFunc: func(_ context.Context, updateEvent event.UpdateEvent, limitingInterface workqueue.RateLimitingInterface) {
 				enqueueFn(updateEvent.ObjectOld.(*kamajiv1alpha1.TenantControlPlane), limitingInterface)
 				enqueueFn(updateEvent.ObjectNew.(*kamajiv1alpha1.TenantControlPlane), limitingInterface)
 			},
-			DeleteFunc: func(deleteEvent event.DeleteEvent, limitingInterface workqueue.RateLimitingInterface) {
+			DeleteFunc: func(_ context.Context, deleteEvent event.DeleteEvent, limitingInterface workqueue.RateLimitingInterface) {
 				enqueueFn(deleteEvent.Object.(*kamajiv1alpha1.TenantControlPlane), limitingInterface)
 			},
 		}).

controllers/finalizers/tcp.go

@@ -5,6 +5,7 @@ package finalizers
 
 const (
 	// DatastoreFinalizer is using a wrong name, since it's related to the underlying datastore.
-	DatastoreFinalizer = "finalizer.kamaji.clastix.io"
-	SootFinalizer      = "finalizer.kamaji.clastix.io/soot"
+	DatastoreFinalizer       = "finalizer.kamaji.clastix.io"
+	DatastoreSecretFinalizer = "finalizer.kamaji.clastix.io/datastore-secret"
+	SootFinalizer            = "finalizer.kamaji.clastix.io/soot"
 )

controllers/resources.go

@@ -14,6 +14,7 @@ import (
 
 	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
 	"github.com/clastix/kamaji/controllers/finalizers"
+	builder "github.com/clastix/kamaji/internal/builders/controlplane"
 	"github.com/clastix/kamaji/internal/datastore"
 	"github.com/clastix/kamaji/internal/resources"
 	ds "github.com/clastix/kamaji/internal/resources/datastore"
@@ -39,6 +40,7 @@ type GroupDeletableResourceBuilderConfiguration struct {
 	tcpReconcilerConfig TenantControlPlaneReconcilerConfig
 	tenantControlPlane  kamajiv1alpha1.TenantControlPlane
 	connection          datastore.Connection
+	dataStore           kamajiv1alpha1.DataStore
 }
 
 // GetResources returns a list of resources that will be used to provide tenant control planes
@@ -59,6 +61,11 @@ func GetDeletableResources(tcp *kamajiv1alpha1.TenantControlPlane, config GroupD
 			Client:     config.client,
 			Connection: config.connection,
 		})
+		res = append(res, &ds.Config{
+			Client:     config.client,
+			ConnString: config.connection.GetConnectionString(),
+			DataStore:  config.dataStore,
+		})
 	}
 
 	return res
@@ -175,6 +182,12 @@ func getKubeconfigResources(c client.Client, tcpReconcilerConfig TenantControlPl
 			KubeConfigFileName: resources.AdminKubeConfigFileName,
 			TmpDirectory:       getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
 		},
+		&resources.KubeconfigResource{
+			Name:               "admin-kubeconfig",
+			Client:             c,
+			KubeConfigFileName: resources.SuperAdminKubeConfigFileName,
+			TmpDirectory:       getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
+		},
 		&resources.KubeconfigResource{
 			Name:               "controller-manager-kubeconfig",
 			Client:             c,
@@ -192,6 +205,9 @@ func getKubeconfigResources(c client.Client, tcpReconcilerConfig TenantControlPl
 
 func getKubernetesStorageResources(c client.Client, dbConnection datastore.Connection, datastore kamajiv1alpha1.DataStore) []resources.Resource {
 	return []resources.Resource{
+		&ds.MultiTenancy{
+			DataStore: datastore,
+		},
 		&ds.Config{
 			Client:     c,
 			ConnString: dbConnection.GetConnectionString(),
@@ -245,7 +261,7 @@ func getKonnectivityServerRequirementsResources(c client.Client) []resources.Res
 
 func getKonnectivityServerPatchResources(c client.Client) []resources.Resource {
 	return []resources.Resource{
-		&konnectivity.KubernetesDeploymentResource{Client: c},
+		&konnectivity.KubernetesDeploymentResource{Builder: builder.Konnectivity{Scheme: *c.Scheme()}, Client: c},
 		&konnectivity.ServiceResource{Client: c},
 	}
 }

controllers/soot/controllers/coredns.go

@@ -35,7 +35,7 @@ type CoreDNS struct {
 	TriggerChannel            chan event.GenericEvent
 }
 
-func (c *CoreDNS) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) {
+func (c *CoreDNS) Reconcile(ctx context.Context, _ reconcile.Request) (reconcile.Result, error) {
 	tcp, err := c.GetTenantControlPlaneFunc()
 	if err != nil {
 		c.logger.Error(err, "cannot retrieve TenantControlPlane")
@@ -60,7 +60,7 @@ func (c *CoreDNS) Reconcile(ctx context.Context, request reconcile.Request) (rec
 		return reconcile.Result{}, nil
 	}
 
-	if err = utils.UpdateStatus(ctx, c.AdminClient, c.GetTenantControlPlaneFunc, resource); err != nil {
+	if err = utils.UpdateStatus(ctx, c.AdminClient, tcp, resource); err != nil {
 		c.logger.Error(err, "update status failed", "resource", resource.GetName())
 
 		return reconcile.Result{}, err
@@ -79,7 +79,7 @@ func (c *CoreDNS) SetupWithManager(mgr manager.Manager) error {
 		For(&rbacv1.ClusterRoleBinding{}, builder.WithPredicates(predicate.NewPredicateFuncs(func(object client.Object) bool {
 			return object.GetName() == kubeadm.CoreDNSClusterRoleBindingName
 		}))).
-		Watches(&source.Channel{Source: c.TriggerChannel}, &handler.EnqueueRequestForObject{}).
+		WatchesRawSource(source.Channel(c.TriggerChannel, &handler.EnqueueRequestForObject{})).
 		Owns(&rbacv1.ClusterRole{}).
 		Owns(&corev1.ServiceAccount{}).
 		Owns(&corev1.Service{}).

controllers/soot/controllers/konnectivity.go

@@ -60,7 +60,7 @@ func (k *KonnectivityAgent) Reconcile(ctx context.Context, _ reconcile.Request)
 			continue
 		}
 
-		if err = utils.UpdateStatus(ctx, k.AdminClient, k.GetTenantControlPlaneFunc, resource); err != nil {
+		if err = utils.UpdateStatus(ctx, k.AdminClient, tcp, resource); err != nil {
 			k.logger.Error(err, "update status failed", "resource", resource.GetName())
 
 			return reconcile.Result{}, err
@@ -80,7 +80,7 @@ func (k *KonnectivityAgent) SetupWithManager(mgr manager.Manager) error {
 		For(&appsv1.DaemonSet{}, builder.WithPredicates(predicate.NewPredicateFuncs(func(object client.Object) bool {
 			return object.GetName() == konnectivity.AgentName && object.GetNamespace() == konnectivity.AgentNamespace
 		}))).
-		Watches(&source.Kind{Type: &corev1.ServiceAccount{}}, handler.EnqueueRequestsFromMapFunc(func(object client.Object) []reconcile.Request {
+		Watches(&corev1.ServiceAccount{}, handler.EnqueueRequestsFromMapFunc(func(_ context.Context, object client.Object) []reconcile.Request {
 			if object.GetName() == konnectivity.AgentName && object.GetNamespace() == konnectivity.AgentNamespace {
 				return []reconcile.Request{
 					{
@@ -94,7 +94,7 @@ func (k *KonnectivityAgent) SetupWithManager(mgr manager.Manager) error {
 
 			return nil
 		})).
-		Watches(&source.Kind{Type: &v1.ClusterRoleBinding{}}, handler.EnqueueRequestsFromMapFunc(func(object client.Object) []reconcile.Request {
+		Watches(&v1.ClusterRoleBinding{}, handler.EnqueueRequestsFromMapFunc(func(_ context.Context, object client.Object) []reconcile.Request {
 			if object.GetName() == konnectivity.CertCommonName {
 				return []reconcile.Request{
 					{
@@ -107,6 +107,6 @@ func (k *KonnectivityAgent) SetupWithManager(mgr manager.Manager) error {
 
 			return nil
 		})).
-		Watches(&source.Channel{Source: k.TriggerChannel}, &handler.EnqueueRequestForObject{}).
+		WatchesRawSource(source.Channel(k.TriggerChannel, &handler.EnqueueRequestForObject{})).
 		Complete(k)
 }

controllers/soot/controllers/kubeadm_phase.go

@@ -50,7 +50,7 @@ func (k *KubeadmPhase) Reconcile(ctx context.Context, _ reconcile.Request) (reco
 		return reconcile.Result{}, nil
 	}
 
-	if err = utils.UpdateStatus(ctx, k.Phase.GetClient(), k.GetTenantControlPlaneFunc, k.Phase); err != nil {
+	if err = utils.UpdateStatus(ctx, k.Phase.GetClient(), tcp, k.Phase); err != nil {
 		k.logger.Error(err, "update status failed")
 
 		return reconcile.Result{}, err
@@ -67,6 +67,6 @@ func (k *KubeadmPhase) SetupWithManager(mgr manager.Manager) error {
 
 	return controllerruntime.NewControllerManagedBy(mgr).
 		For(k.Phase.GetWatchedObject(), builder.WithPredicates(predicate.NewPredicateFuncs(k.Phase.GetPredicateFunc()))).
-		Watches(&source.Channel{Source: k.TriggerChannel}, &handler.EnqueueRequestForObject{}).
+		WatchesRawSource(source.Channel(k.TriggerChannel, &handler.EnqueueRequestForObject{})).
 		Complete(k)
 }

controllers/soot/controllers/kubeproxy.go

@@ -60,7 +60,7 @@ func (k *KubeProxy) Reconcile(ctx context.Context, _ reconcile.Request) (reconci
 		return reconcile.Result{}, nil
 	}
 
-	if err = utils.UpdateStatus(ctx, k.AdminClient, k.GetTenantControlPlaneFunc, resource); err != nil {
+	if err = utils.UpdateStatus(ctx, k.AdminClient, tcp, resource); err != nil {
 		k.logger.Error(err, "update status failed")
 
 		return reconcile.Result{}, err
@@ -79,7 +79,7 @@ func (k *KubeProxy) SetupWithManager(mgr manager.Manager) error {
 		For(&rbacv1.ClusterRoleBinding{}, builder.WithPredicates(predicate.NewPredicateFuncs(func(object client.Object) bool {
 			return object.GetName() == kubeadm.KubeProxyClusterRoleBindingName
 		}))).
-		Watches(&source.Channel{Source: k.TriggerChannel}, &handler.EnqueueRequestForObject{}).
+		WatchesRawSource(source.Channel(k.TriggerChannel, &handler.EnqueueRequestForObject{})).
 		Owns(&corev1.ServiceAccount{}).
 		Owns(&rbacv1.Role{}).
 		Owns(&rbacv1.RoleBinding{}).

controllers/soot/controllers/migrate.go

@@ -11,7 +11,7 @@ import (
 	admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/utils/pointer"
+	pointer "k8s.io/utils/ptr"
 	controllerruntime "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/builder"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -84,7 +84,7 @@ func (m *Migrate) createOrUpdate(ctx context.Context) error {
 			{
 				Name: "leases.migrate.kamaji.clastix.io",
 				ClientConfig: admissionregistrationv1.WebhookClientConfig{
-					URL:      pointer.String(fmt.Sprintf("https://%s.%s.svc:443/migrate", m.WebhookServiceName, m.WebhookNamespace)),
+					URL:      pointer.To(fmt.Sprintf("https://%s.%s.svc:443/migrate", m.WebhookServiceName, m.WebhookNamespace)),
 					CABundle: m.WebhookCABundle,
 				},
 				Rules: []admissionregistrationv1.RuleWithOperations{
@@ -128,7 +128,7 @@ func (m *Migrate) createOrUpdate(ctx context.Context) error {
 			{
 				Name: "catchall.migrate.kamaji.clastix.io",
 				ClientConfig: admissionregistrationv1.WebhookClientConfig{
-					URL:      pointer.String(fmt.Sprintf("https://%s.%s.svc:443/migrate", m.WebhookServiceName, m.WebhookNamespace)),
+					URL:      pointer.To(fmt.Sprintf("https://%s.%s.svc:443/migrate", m.WebhookServiceName, m.WebhookNamespace)),
 					CABundle: m.WebhookCABundle,
 				},
 				Rules: []admissionregistrationv1.RuleWithOperations{
@@ -187,7 +187,7 @@ func (m *Migrate) SetupWithManager(mgr manager.Manager) error {
 
 			return object.GetName() == vwc.GetName()
 		}))).
-		Watches(&source.Channel{Source: m.TriggerChannel}, &handler.EnqueueRequestForObject{}).
+		WatchesRawSource(source.Channel(m.TriggerChannel, &handler.EnqueueRequestForObject{})).
 		Complete(m)
 }
 

controllers/soot/manager.go

@@ -12,13 +12,13 @@ import (
 	"k8s.io/client-go/util/retry"
 	controllerruntime "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/builder"
-	"sigs.k8s.io/controller-runtime/pkg/cache"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 	"sigs.k8s.io/controller-runtime/pkg/event"
 	"sigs.k8s.io/controller-runtime/pkg/handler"
 	"sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
+	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 	"sigs.k8s.io/controller-runtime/pkg/predicate"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 	"sigs.k8s.io/controller-runtime/pkg/source"
@@ -123,22 +123,27 @@ func (m *Manager) Reconcile(ctx context.Context, request reconcile.Request) (res
 	// the soot manager if this is already registered.
 	v, ok := m.sootMap[request.String()]
 	if ok {
-		// The TenantControlPlane is in non-ready mode, or marked for deletion:
-		// we don't want to pollute with messages due to broken connection.
-		// Once the TCP will be ready again, the event will be intercepted and the manager started back.
-		if tcpStatus == kamajiv1alpha1.VersionNotReady {
+		switch {
+		case tcpStatus == kamajiv1alpha1.VersionCARotating:
+			// The TenantControlPlane CA has been rotated, it means the running manager
+			// must be restarted to avoid certificate signed by unknown authority errors.
 			return reconcile.Result{}, m.cleanup(ctx, request, tcp)
-		}
-
-		for _, trigger := range v.triggers {
-			trigger <- event.GenericEvent{Object: tcp}
+		case tcpStatus == kamajiv1alpha1.VersionNotReady:
+			// The TenantControlPlane is in non-ready mode, or marked for deletion:
+			// we don't want to pollute with messages due to broken connection.
+			// Once the TCP will be ready again, the event will be intercepted and the manager started back.
+			return reconcile.Result{}, m.cleanup(ctx, request, tcp)
+		default:
+			for _, trigger := range v.triggers {
+				trigger <- event.GenericEvent{Object: tcp}
+			}
 		}
 
 		return reconcile.Result{}, nil
 	}
 	// No need to start a soot manager if the TenantControlPlane is not ready:
 	// enqueuing back is not required since we're going to get that event once ready.
-	if tcpStatus == kamajiv1alpha1.VersionNotReady {
+	if tcpStatus == kamajiv1alpha1.VersionNotReady || tcpStatus == kamajiv1alpha1.VersionCARotating {
 		log.FromContext(ctx).Info("skipping start of the soot manager for a not ready instance")
 
 		return reconcile.Result{}, nil
@@ -170,10 +175,12 @@ func (m *Manager) Reconcile(ctx context.Context, request reconcile.Request) (res
 	}()
 
 	mgr, err := controllerruntime.NewManager(tcpRest, controllerruntime.Options{
-		Logger:             log.Log.WithName(fmt.Sprintf("soot_%s_%s", tcp.GetNamespace(), tcp.GetName())),
-		Scheme:             m.client.Scheme(),
-		MetricsBindAddress: "0",
-		NewClient: func(cache cache.Cache, config *rest.Config, options client.Options, uncachedObjects ...client.Object) (client.Client, error) {
+		Logger: log.Log.WithName(fmt.Sprintf("soot_%s_%s", tcp.GetNamespace(), tcp.GetName())),
+		Scheme: m.client.Scheme(),
+		Metrics: metricsserver.Options{
+			BindAddress: "0",
+		},
+		NewClient: func(config *rest.Config, options client.Options) (client.Client, error) {
 			return client.New(config, client.Options{
 				Scheme: m.client.Scheme(),
 			})
@@ -251,6 +258,17 @@ func (m *Manager) Reconcile(ctx context.Context, request reconcile.Request) (res
 	if err = bootstrapToken.SetupWithManager(mgr); err != nil {
 		return reconcile.Result{}, err
 	}
+
+	kubeadmRbac := &controllers.KubeadmPhase{
+		GetTenantControlPlaneFunc: m.retrieveTenantControlPlane(tcpCtx, request),
+		Phase: &resources.KubeadmPhase{
+			Client: m.AdminClient,
+			Phase:  resources.PhaseClusterAdminRBAC,
+		},
+	}
+	if err = kubeadmRbac.SetupWithManager(mgr); err != nil {
+		return reconcile.Result{}, err
+	}
 	// Starting the manager
 	go func() {
 		if err = mgr.Start(tcpCtx); err != nil {
@@ -284,7 +302,7 @@ func (m *Manager) SetupWithManager(mgr manager.Manager) error {
 	m.sootMap = make(map[string]sootItem)
 
 	return controllerruntime.NewControllerManagedBy(mgr).
-		Watches(&source.Channel{Source: m.sootManagerErrChan}, &handler.EnqueueRequestForObject{}).
+		WatchesRawSource(source.Channel(m.sootManagerErrChan, &handler.EnqueueRequestForObject{})).
 		For(&kamajiv1alpha1.TenantControlPlane{}, builder.WithPredicates(predicate.NewPredicateFuncs(func(object client.Object) bool {
 			obj := object.(*kamajiv1alpha1.TenantControlPlane) //nolint:forcetypeassert
 			// status is required to understand if we have to start or stop the soot manager

controllers/telemetry_controller.go

@@ -0,0 +1,120 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package controllers
+
+import (
+	"context"
+	"time"
+
+	"github.com/clastix/kamaji-telemetry/api"
+	telemetry "github.com/clastix/kamaji-telemetry/pkg/client"
+	"github.com/pkg/errors"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/log"
+
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+)
+
+type TelemetryController struct {
+	Client                  client.Client
+	TelemetryClient         telemetry.Client
+	LeaderElectionNamespace string
+	LeaderElectionID        string
+	KamajiVersion           string
+	KubernetesVersion       string
+}
+
+func (m *TelemetryController) retrieveControllerUID(ctx context.Context) (string, error) {
+	var defaultSvc corev1.Service
+	if err := m.Client.Get(ctx, types.NamespacedName{Name: "kubernetes", Namespace: "default"}, &defaultSvc); err != nil {
+		return "", errors.Wrap(err, "cannot start the telemetry controller")
+	}
+
+	return string(defaultSvc.UID), nil
+}
+
+func (m *TelemetryController) Start(ctx context.Context) error {
+	logger := log.FromContext(ctx)
+
+	ticker := time.NewTicker(5 * time.Minute)
+	defer ticker.Stop()
+
+	uid, err := m.retrieveControllerUID(ctx)
+	if err != nil {
+		logger.Error(err, "cannot retrieve controller UID")
+
+		return err
+	}
+	// First run to avoid waiting 5 minutes
+	go m.collectStats(ctx, uid)
+
+	for {
+		select {
+		case <-ctx.Done():
+			return nil
+		case <-ticker.C:
+			go m.collectStats(ctx, uid)
+		}
+	}
+}
+
+func (m *TelemetryController) collectStats(ctx context.Context, uid string) {
+	logger := log.FromContext(ctx)
+
+	stats := api.Stats{
+		UUID:                uid,
+		KamajiVersion:       m.KamajiVersion,
+		KubernetesVersion:   m.KubernetesVersion,
+		TenantControlPlanes: api.TenantControlPlane{},
+		Datastores:          api.Datastores{},
+	}
+
+	var tcpList kamajiv1alpha1.TenantControlPlaneList
+	if err := m.Client.List(ctx, &tcpList); err != nil {
+		logger.Error(err, "cannot list TenantControlPlane")
+
+		return
+	}
+
+	for _, tcp := range tcpList.Items {
+		switch {
+		case tcp.Spec.ControlPlane.Deployment.Replicas == nil || *tcp.Spec.ControlPlane.Deployment.Replicas == 0:
+			stats.TenantControlPlanes.Sleeping++
+		case tcp.Status.Kubernetes.Version.Status != nil && *tcp.Status.Kubernetes.Version.Status == kamajiv1alpha1.VersionNotReady:
+			stats.TenantControlPlanes.NotReady++
+		case tcp.Status.Kubernetes.Version.Status != nil && *tcp.Status.Kubernetes.Version.Status == kamajiv1alpha1.VersionUpgrading:
+			stats.TenantControlPlanes.Upgrading++
+		default:
+			stats.TenantControlPlanes.Running++
+		}
+	}
+
+	var datastoreList kamajiv1alpha1.DataStoreList
+	if err := m.Client.List(ctx, &datastoreList); err != nil {
+		logger.Error(err, "cannot list DataStores")
+
+		return
+	}
+
+	for _, ds := range datastoreList.Items {
+		switch ds.Spec.Driver {
+		case kamajiv1alpha1.EtcdDriver:
+			stats.Datastores.Etcd.ShardCount++
+			stats.Datastores.Etcd.UsedByCount += len(ds.Status.UsedBy)
+		case kamajiv1alpha1.KinePostgreSQLDriver:
+			stats.Datastores.PostgreSQL.ShardCount++
+			stats.Datastores.PostgreSQL.UsedByCount += len(ds.Status.UsedBy)
+		case kamajiv1alpha1.KineMySQLDriver:
+			stats.Datastores.MySQL.ShardCount++
+			stats.Datastores.MySQL.UsedByCount += len(ds.Status.UsedBy)
+		case kamajiv1alpha1.KineNatsDriver:
+			stats.Datastores.NATS.ShardCount++
+			stats.Datastores.NATS.UsedByCount += len(ds.Status.UsedBy)
+		}
+	}
+
+	m.TelemetryClient.PushStats(ctx, stats)
+}

controllers/tenantcontrolplane_controller.go

@@ -6,18 +6,23 @@ package controllers
 import (
 	"context"
 	"fmt"
+	"strings"
+	"time"
 
+	"github.com/juju/mutex/v2"
 	"github.com/pkg/errors"
 	appsv1 "k8s.io/api/apps/v1"
 	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
 	networkingv1 "k8s.io/api/networking/v1"
-	errors2 "k8s.io/apimachinery/pkg/api/errors"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	k8stypes "k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/util/workqueue"
+	"k8s.io/utils/clock"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/builder"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 	"sigs.k8s.io/controller-runtime/pkg/event"
 	"sigs.k8s.io/controller-runtime/pkg/handler"
@@ -36,18 +41,26 @@ import (
 
 // TenantControlPlaneReconciler reconciles a TenantControlPlane object.
 type TenantControlPlaneReconciler struct {
-	Client               client.Client
-	APIReader            client.Reader
-	Config               TenantControlPlaneReconcilerConfig
-	TriggerChan          TenantControlPlaneChannel
-	KamajiNamespace      string
-	KamajiServiceAccount string
-	KamajiService        string
-	KamajiMigrateImage   string
+	Client                  client.Client
+	APIReader               client.Reader
+	Config                  TenantControlPlaneReconcilerConfig
+	TriggerChan             TenantControlPlaneChannel
+	KamajiNamespace         string
+	KamajiServiceAccount    string
+	KamajiService           string
+	KamajiMigrateImage      string
+	MaxConcurrentReconciles int
+	// CertificateChan is the channel used by the CertificateLifecycleController that is checking for
+	// certificates and kubeconfig user certs validity: a generic event for the given TCP will be triggered
+	// once the validity threshold for the given certificate is reached.
+	CertificateChan CertificateChannel
+
+	clock mutex.Clock
 }
 
 // TenantControlPlaneReconcilerConfig gives the necessary configuration for TenantControlPlaneReconciler.
 type TenantControlPlaneReconcilerConfig struct {
+	ReconcileTimeout     time.Duration
 	DefaultDataStoreName string
 	KineContainerImage   string
 	TmpBaseDirectory     string
@@ -66,18 +79,40 @@ type TenantControlPlaneReconcilerConfig struct {
 func (r *TenantControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
 	log := log.FromContext(ctx)
 
+	var cancelFn context.CancelFunc
+	ctx, cancelFn = context.WithTimeout(ctx, r.Config.ReconcileTimeout)
+	defer cancelFn()
+
 	tenantControlPlane, err := r.getTenantControlPlane(ctx, req.NamespacedName)()
-	if err != nil {
-		if errors2.IsNotFound(err) {
-			log.Info("resource may have been deleted, skipping")
+	if k8serrors.IsNotFound(err) {
+		log.Info("resource have been deleted, skipping")
 
-			return ctrl.Result{}, nil
-		}
-
-		log.Error(err, "cannot retrieve the required instance")
-
-		return ctrl.Result{}, err
+		return reconcile.Result{}, nil
 	}
+	if err != nil {
+		log.Error(err, "cannot retrieve the required resource")
+
+		return reconcile.Result{}, err
+	}
+
+	releaser, err := mutex.Acquire(r.mutexSpec(tenantControlPlane))
+	if err != nil {
+		switch {
+		case errors.As(err, &mutex.ErrTimeout):
+			log.Info("acquire timed out, current process is blocked by another reconciliation")
+
+			return ctrl.Result{Requeue: true}, nil
+		case errors.As(err, &mutex.ErrCancelled):
+			log.Info("acquire cancelled")
+
+			return ctrl.Result{Requeue: true}, nil
+		default:
+			log.Error(err, "acquire failed")
+
+			return ctrl.Result{}, err
+		}
+	}
+	defer releaser.Release()
 
 	markedToBeDeleted := tenantControlPlane.GetDeletionTimestamp() != nil
 
@@ -109,6 +144,7 @@ func (r *TenantControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R
 			tcpReconcilerConfig: r.Config,
 			tenantControlPlane:  *tenantControlPlane,
 			connection:          dsConnection,
+			dataStore:           *ds,
 		}
 
 		for _, resource := range GetDeletableResources(tenantControlPlane, groupDeletableResourceBuilderConfiguration) {
@@ -156,7 +192,7 @@ func (r *TenantControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R
 			continue
 		}
 
-		if err = utils.UpdateStatus(ctx, r.Client, r.getTenantControlPlane(ctx, req.NamespacedName), resource); err != nil {
+		if err = utils.UpdateStatus(ctx, r.Client, tenantControlPlane, resource); err != nil {
 			log.Error(err, "update of the resource failed", "resource", resource.GetName())
 
 			return ctrl.Result{}, err
@@ -164,7 +200,11 @@ func (r *TenantControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R
 
 		log.Info(fmt.Sprintf("%s has been configured", resource.GetName()))
 
-		return ctrl.Result{}, nil
+		if result == resources.OperationResultEnqueueBack {
+			log.Info("requested enqueuing back", "resources", resource.GetName())
+
+			return ctrl.Result{Requeue: true}, nil
+		}
 	}
 
 	log.Info(fmt.Sprintf("%s has been reconciled", tenantControlPlane.GetName()))
@@ -172,24 +212,44 @@ func (r *TenantControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R
 	return ctrl.Result{}, nil
 }
 
+func (r *TenantControlPlaneReconciler) mutexSpec(obj client.Object) mutex.Spec {
+	return mutex.Spec{
+		Name:    strings.ReplaceAll(fmt.Sprintf("kamaji%s", obj.GetUID()), "-", ""),
+		Clock:   r.clock,
+		Delay:   10 * time.Millisecond,
+		Timeout: time.Second,
+		Cancel:  nil,
+	}
+}
+
 // SetupWithManager sets up the controller with the Manager.
 func (r *TenantControlPlaneReconciler) SetupWithManager(mgr ctrl.Manager) error {
+	r.clock = clock.RealClock{}
+
 	return ctrl.NewControllerManagedBy(mgr).
-		Watches(&source.Channel{Source: r.TriggerChan}, handler.Funcs{GenericFunc: func(genericEvent event.GenericEvent, limitingInterface workqueue.RateLimitingInterface) {
+		WatchesRawSource(source.Channel(r.CertificateChan, handler.Funcs{GenericFunc: func(_ context.Context, genericEvent event.GenericEvent, limitingInterface workqueue.RateLimitingInterface) {
 			limitingInterface.AddRateLimited(ctrl.Request{
 				NamespacedName: k8stypes.NamespacedName{
 					Namespace: genericEvent.Object.GetNamespace(),
 					Name:      genericEvent.Object.GetName(),
 				},
 			})
-		}}).
+		}})).
+		WatchesRawSource(source.Channel(r.TriggerChan, handler.Funcs{GenericFunc: func(_ context.Context, genericEvent event.GenericEvent, limitingInterface workqueue.RateLimitingInterface) {
+			limitingInterface.AddRateLimited(ctrl.Request{
+				NamespacedName: k8stypes.NamespacedName{
+					Namespace: genericEvent.Object.GetNamespace(),
+					Name:      genericEvent.Object.GetName(),
+				},
+			})
+		}})).
 		For(&kamajiv1alpha1.TenantControlPlane{}).
 		Owns(&corev1.Secret{}).
 		Owns(&corev1.ConfigMap{}).
 		Owns(&appsv1.Deployment{}).
 		Owns(&corev1.Service{}).
 		Owns(&networkingv1.Ingress{}).
-		Watches(&source.Kind{Type: &batchv1.Job{}}, handler.EnqueueRequestsFromMapFunc(func(object client.Object) []reconcile.Request {
+		Watches(&batchv1.Job{}, handler.EnqueueRequestsFromMapFunc(func(_ context.Context, object client.Object) []reconcile.Request {
 			labels := object.GetLabels()
 
 			name, namespace := labels["tcp.kamaji.clastix.io/name"], labels["tcp.kamaji.clastix.io/namespace"]
@@ -217,6 +277,9 @@ func (r *TenantControlPlaneReconciler) SetupWithManager(mgr ctrl.Manager) error
 
 			return ok && v == "migrate"
 		}))).
+		WithOptions(controller.Options{
+			MaxConcurrentReconciles: r.MaxConcurrentReconciles,
+		}).
 		Complete(r)
 }
 

controllers/utils/update_status.go

@@ -7,24 +7,27 @@ import (
 	"context"
 	"fmt"
 
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/util/retry"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
 	"github.com/clastix/kamaji/internal/resources"
 )
 
-func UpdateStatus(ctx context.Context, client client.Client, tcpRetrieval TenantControlPlaneRetrievalFn, resource resources.Resource) error {
-	updateErr := retry.RetryOnConflict(retry.DefaultRetry, func() error {
-		tenantControlPlane, err := tcpRetrieval()
-		if err != nil {
-			return err
-		}
+func UpdateStatus(ctx context.Context, client client.Client, tcp *kamajiv1alpha1.TenantControlPlane, resource resources.Resource) error {
+	updateErr := retry.RetryOnConflict(retry.DefaultRetry, func() (err error) {
+		defer func() {
+			if err != nil {
+				_ = client.Get(ctx, types.NamespacedName{Name: tcp.Name, Namespace: tcp.Namespace}, tcp)
+			}
+		}()
 
-		if err = resource.UpdateTenantControlPlaneStatus(ctx, tenantControlPlane); err != nil {
+		if err = resource.UpdateTenantControlPlaneStatus(ctx, tcp); err != nil {
 			return fmt.Errorf("error applying TenantcontrolPlane status: %w", err)
 		}
 
-		if err = client.Status().Update(ctx, tenantControlPlane); err != nil {
+		if err = client.Status().Update(ctx, tcp); err != nil {
 			return fmt.Errorf("error updating tenantControlPlane status: %w", err)
 		}
 

deploy/kamaji-azure.env

@@ -15,7 +15,7 @@ export KAMAJI_NAMESPACE=kamaji-system
 export TENANT_NAMESPACE=default
 export TENANT_NAME=tenant-00
 export TENANT_DOMAIN=$KAMAJI_REGION.cloudapp.azure.com
-export TENANT_VERSION=v1.25.0
+export TENANT_VERSION=v1.26.0
 export TENANT_PORT=6443 # port used to expose the tenant api server
 export TENANT_PROXY_PORT=8132 # port used to expose the konnectivity server
 export TENANT_POD_CIDR=10.36.0.0/16

deploy/kamaji.env

@@ -5,7 +5,7 @@ export KAMAJI_NAMESPACE=kamaji-system
 export TENANT_NAMESPACE=default
 export TENANT_NAME=tenant-00
 export TENANT_DOMAIN=clastix.labs
-export TENANT_VERSION=v1.25.0
+export TENANT_VERSION=v1.26.0
 export TENANT_PORT=6443 # port used to expose the tenant api server
 export TENANT_PROXY_PORT=8132 # port used to expose the konnectivity server
 export TENANT_POD_CIDR=10.36.0.0/16

deploy/kind/Makefile

@@ -11,13 +11,13 @@ prometheus-stack:
 	helm repo update
 	helm install prometheus-stack --create-namespace -n monitoring prometheus-community/kube-prometheus-stack
 
-reqs: kind ingress-nginx etcd-cluster cert-manager
+reqs: kind ingress-nginx cert-manager
 
 cert-manager:
 	@kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.10.1/cert-manager.yaml
 
 kamaji: reqs
-	@kubectl apply -f $(kind_path)/../../config/install.yaml
+	helm install kamaji --create-namespace -n kamaji-system $(kind_path)/../../charts/kamaji
 
 destroy: kind/destroy etcd-certificates/cleanup
 

deploy/kind/join-node.bash

@@ -7,7 +7,7 @@ export DOCKER_IMAGE_NAME="kindest/node"
 export DOCKER_NETWORK="kind"
 
 # Variables
-export KUBERNETES_VERSION=${1:-v1.23.5}
+export KUBERNETES_VERSION=${1:-v1.23.4}
 export KUBECONFIG="${KUBECONFIG:-/tmp/kubeconfig}"
 
 if [ -z $2 ]

deploy/kind/kind-kamaji.yaml

@@ -26,7 +26,7 @@ nodes:
     ## expose port 31132 of the node to port 31132 on the host for konnectivity
   - containerPort: 31132
     hostPort: 31132
-    protocol: TCP   
+    protocol: TCP
     ## expose port 31443 of the node to port 31443 on the host
   - containerPort: 31443
     hostPort: 31443

deploy/kine/mysql/certs/bronze/server-csr.json

@@ -1,14 +0,0 @@
-{
-  "CN": "bronze.mysql-system.svc.cluster.local",
-  "key": {
-    "algo": "rsa",
-    "size": 2048
-  },
-  "hosts": [
-    "127.0.0.1",
-    "localhost",
-    "bronze",
-    "bronze.mysql-system.svc",
-    "bronze.mysql-system.svc.cluster.local"
-  ]
-}

deploy/kine/nats/Makefile

@@ -0,0 +1,40 @@
+ROOT_DIR:=$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))
+NAME:=default
+NAMESPACE:=nats-system
+
+.PHONY: helm
+HELM = $(shell pwd)/../../../bin/helm
+helm: ## Download helm locally if necessary.
+	$(call go-install-tool,$(HELM),helm.sh/helm/v3/cmd/helm@v3.9.0)
+
+nats: nats-certificates nats-secret nats-deployment
+
+nats-certificates:
+	rm -rf $(ROOT_DIR)/certs/$(NAME) && mkdir -p $(ROOT_DIR)/certs/$(NAME)
+	cfssl gencert -initca $(ROOT_DIR)/ca-csr.json | cfssljson -bare $(ROOT_DIR)/certs/$(NAME)/ca
+	@mv $(ROOT_DIR)/certs/$(NAME)/ca.pem $(ROOT_DIR)/certs/$(NAME)/ca.crt
+	@mv $(ROOT_DIR)/certs/$(NAME)/ca-key.pem $(ROOT_DIR)/certs/$(NAME)/ca.key
+	@NAME=$(NAME) NAMESPACE=$(NAMESPACE) envsubst < server-csr.json > $(ROOT_DIR)/certs/$(NAME)/server-csr.json
+	cfssl gencert -ca=$(ROOT_DIR)/certs/$(NAME)/ca.crt -ca-key=$(ROOT_DIR)/certs/$(NAME)/ca.key \
+		-config=$(ROOT_DIR)/config.json -profile=server \
+		$(ROOT_DIR)/certs/$(NAME)/server-csr.json | cfssljson -bare $(ROOT_DIR)/certs/$(NAME)/server
+	@mv $(ROOT_DIR)/certs/$(NAME)/server.pem $(ROOT_DIR)/certs/$(NAME)/server.crt
+	@mv $(ROOT_DIR)/certs/$(NAME)/server-key.pem $(ROOT_DIR)/certs/$(NAME)/server.key
+	chmod 644 $(ROOT_DIR)/certs/$(NAME)/*
+
+nats-secret:
+	@kubectl create namespace $(NAMESPACE) || true
+	@kubectl -n $(NAMESPACE) create secret generic nats-$(NAME)-config \
+		--from-file=$(ROOT_DIR)/certs/$(NAME)/ca.crt --from-file=$(ROOT_DIR)/certs/$(NAME)/ca.key \
+		--from-file=$(ROOT_DIR)/certs/$(NAME)/server.key --from-file=$(ROOT_DIR)/certs/$(NAME)/server.crt \
+		--from-literal=password=password \
+		--dry-run=client -o yaml | kubectl apply -f -
+
+nats-deployment:
+	@VALUES_FILE=$(if $(findstring notls,$(NAME)),values-notls.yaml,values.yaml); \
+	NAME=$(NAME) envsubst < $(ROOT_DIR)/$$VALUES_FILE | $(HELM) upgrade --install $(NAME) nats/nats --create-namespace -n nats-system -f -
+
+
+nats-destroy:
+	@NAME=$(NAME) envsubst < $(ROOT_DIR)/nats.yaml | kubectl -n $(NAMESPACE) delete --ignore-not-found -f -
+	@kubectl delete -n $(NAMESPACE) secret mysql-$(NAME)config --ignore-not-found

deploy/kine/nats/ca-csr.json

@@ -0,0 +1,18 @@
+{
+  "CN": "Clastix CA",
+  "key": {
+    "algo": "rsa",
+    "size": 2048
+  },
+  "names": [
+    {
+      "C": "IT",
+      "ST": "Italy",
+      "L": "Milan"
+    }
+  ],
+  "hosts": [
+    "127.0.0.1",
+    "localhost"
+  ]
+}

deploy/kine/nats/config.json

@@ -0,0 +1,18 @@
+{
+  "signing": {
+      "default": {
+          "expiry": "8760h"
+      },
+      "profiles": {
+          "server": {
+              "expiry": "8760h",
+              "usages": [
+                  "signing",
+                  "key encipherment",
+                  "server auth",
+                  "client auth"
+              ]
+          }
+      }
+  }
+}

deploy/kine/nats/server-csr.json

@@ -0,0 +1,14 @@
+{
+  "CN": "$NAME.$NAMESPACE.svc.cluster.local",
+  "key": {
+    "algo": "rsa",
+    "size": 2048
+  },
+  "hosts": [
+    "127.0.0.1",
+    "localhost",
+    "$NAME-nats",
+    "$NAME-nats.$NAMESPACE.svc",
+    "$NAME-nats.$NAMESPACE.svc.cluster.local"
+  ]
+}

deploy/kine/nats/values-notls.yaml

@@ -0,0 +1,14 @@
+config:
+  merge:
+    accounts:
+      private:
+        jetstream: enabled
+        users:
+        - {user: admin, password: "password", permissions: {subscribe: [">"], publish: [">"]}}
+  cluster:
+    enabled: no
+  jetstream:
+    enabled: true
+    fileStore:
+      pvc:
+        size: 32Mi