chore(release)!: switch to goreleaser and migrating tag format (#1094)

Signed-off-by: Dario Tranchitella <dario@tranchitella.eu>

.github/workflows/ci.yaml

@@ -7,28 +7,39 @@ on:
     branches: [ "*" ]
 
 jobs:
+  test:
+    name: integration
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v6
+      - uses: actions/setup-go@v6
+        with:
+          go-version-file: go.mod
+      - run: make test
   golangci:
     name: lint
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-go@v5
+      - uses: actions/checkout@v6
+      - uses: actions/setup-go@v6
         with:
           go-version-file: go.mod
       - name: Run golangci-lint
-        uses: golangci/golangci-lint-action@v6.5.1
-        with:
-          version: v1.62.2
-          only-new-issues: false
-          args: --config .golangci.yml
+        run: make golint
+# TODO(prometherion): enable back once golangci-lint is built from v1.24 rather than v1.23
+#        uses: golangci/golangci-lint-action@v6.5.2
+#        with:
+#          version: v1.62.2
+#          only-new-issues: false
+#          args: --config .golangci.yml
   diff:
     name: diff
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
         with:
           fetch-depth: 0
-      - uses: actions/setup-go@v5
+      - uses: actions/setup-go@v6
         with:
           go-version-file: go.mod
       - run: make manifests

.github/workflows/e2e.yaml

@@ -4,7 +4,7 @@ on:
   push:
     branches: [ "*" ]
     paths:
-      - '.github/workflows/e2e.yml'
+      - '.github/workflows/e2e.yaml'
       - 'api/**'
       - 'charts/kamaji/**'
       - 'controllers/**'
@@ -18,7 +18,7 @@ on:
   pull_request:
     branches: [ "*" ]
     paths:
-      - '.github/workflows/e2e.yml'
+      - '.github/workflows/e2e.yaml'
       - 'api/**'
       - 'charts/kamaji/**'
       - 'controllers/**'
@@ -33,18 +33,25 @@ on:
 jobs:
   kind:
     name: Kubernetes
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
         with:
           fetch-depth: 0
-      - uses: actions/setup-go@v5
+      - uses: actions/setup-go@v6
         with:
-          go-version: '1.22'
-          check-latest: true
+          go-version-file: go.mod
+      - name: reclaim disk space from runner
+        run: |
+          sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
       - run: |
           sudo apt-get update
           sudo apt-get install -y golang-cfssl
           sudo swapoff -a
+          sudo modprobe br_netfilter
+      - name: install required Go tools
+        run: make kind ko helm ginkgo
+      - name: cleaning up go mod
+        run: go clean -modcache
       - name: e2e testing
         run: make e2e

.github/workflows/helm.yaml

@@ -2,8 +2,7 @@ name: Helm Chart
 
 on:
   push:
-    branches: [ "*" ]
-    tags: [ "helm-v*" ]
+    branches: [ "master" ]
   pull_request:
     branches: [ "*" ]
 
@@ -12,16 +11,19 @@ jobs:
     name: diff
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
         with:
           fetch-depth: 0
       - run: make -C charts/kamaji docs
-      - name: Checking if Helm docs is not aligned
+      - name: Checking if Kamaji Helm Chart docs is not aligned
+        run: if [[ $(git diff | wc -l) -gt 0 ]]; then echo ">>> Untracked changes have not been committed" && git --no-pager diff && exit 1; fi
+      - run: make -C charts/kamaji-crds docs
+      - name: Checking if Kamaji CRDs Helm Chart docs is not aligned
         run: if [[ $(git diff | wc -l) -gt 0 ]]; then echo ">>> Untracked changes have not been committed" && git --no-pager diff && exit 1; fi
   lint:
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
       - uses: azure/setup-helm@v4
         with:
           version: 3.3.4
@@ -29,13 +31,16 @@ jobs:
         run: |-
           helm repo add clastix https://clastix.github.io/charts 
           helm dependency build ./charts/kamaji
-      - name: Linting Chart
+      - name: Linting Kamaji Helm Chart
         run: helm lint ./charts/kamaji
+      - name: Linting Kamaji CRDS Helm Chart
+        run: helm lint ./charts/kamaji-crds
   release:
-    if: startsWith(github.ref, 'refs/tags/helm-v')
+    if: github.event_name == 'push' && github.ref == 'refs/heads/master'
+    needs: [ "lint", "diff" ]
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
       - name: Publish Helm chart
         uses: stefanprodan/helm-gh-pages@master
         with:

.github/workflows/ko-build.yml

@@ -2,9 +2,6 @@ name: Container image build
 
 on:
   push:
-    tags:
-      - edge-*
-      - v*
     branches:
       - master
 
@@ -12,20 +9,22 @@ jobs:
   ko:
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
         with:
           fetch-depth: 0
-      - uses: actions/setup-go@v5
+
+      - uses: actions/setup-go@v6
         with:
           go-version-file: go.mod
+
       - name: "ko: install"
         run: make ko
+
       - name: "ko: login to quay.io container registry"
         run: ./bin/ko login quay.io -u ${{ secrets.QUAY_IO_USERNAME }} -p ${{ secrets.QUAY_IO_TOKEN }}
+
       - name: "ko: login to docker.io container registry"
         run: ./bin/ko login docker.io -u ${{ secrets.DOCKER_IO_USERNAME }} -p ${{ secrets.DOCKER_IO_TOKEN }}
-      - name: "ko: build and push tag"
-        run: make VERSION=${{ github.ref_name }} KO_LOCAL=false KO_PUSH=true build
-        if: startsWith(github.ref, 'refs/tags/v') || startsWith(github.ref, 'refs/tags/edge-')
+
       - name: "ko: build and push latest"
         run: make VERSION=latest KO_LOCAL=false KO_PUSH=true build

.github/workflows/pr.yaml

@@ -0,0 +1,23 @@
+name: Check PR Title
+
+on:
+  pull_request:
+    types: [opened, edited, reopened, synchronize]
+
+jobs:
+  semantic-pr-title:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: amannn/action-semantic-pull-request@v6
+        with:
+          types: |
+            feat
+            fix
+            chore
+            docs
+            style
+            refactor
+            perf
+            test
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/release.yml

@@ -0,0 +1,60 @@
+name: Weekly Edge Release
+
+on:
+  schedule:
+    - cron: '0 7 * * 1'  # Every Monday at 9 AM CET
+  workflow_dispatch:
+
+permissions:
+  contents: write
+
+jobs:
+  release:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
+
+      - name: "tag: compute"
+        id: git
+        run: |
+          CURRENT_DATE=$(date -u +'%Y-%m-%d')
+          YY=$(date -u +'%y')
+          M=$(date -u +'%_m' | sed 's/ //g')
+          FIRST_OF_MONTH=$(date -u -d "$CURRENT_DATE" +%Y-%m-01)
+          WEEK_NUM=$(( (($(date -u +%s) - $(date -u -d "$FIRST_OF_MONTH" +%s)) / 86400 + $(date -u -d "$FIRST_OF_MONTH" +%u) - 1) / 7 + 1 ))
+
+          TAG="$YY.$M.$WEEK_NUM-edge"
+
+          echo "tag=$TAG" >> $GITHUB_OUTPUT
+
+      - name: "tag: push"
+        run: |
+          git tag ${{ steps.git.outputs.tag }}
+          git push origin ${{ steps.git.outputs.tag }}
+
+      - uses: actions/setup-go@v6
+        with:
+          go-version-file: go.mod
+
+      - name: "deps: installing ko"
+        run: make ko
+
+      - name: "ko: login to quay.io container registry"
+        run: ./bin/ko login quay.io -u ${{ secrets.QUAY_IO_USERNAME }} -p ${{ secrets.QUAY_IO_TOKEN }}
+
+      - name: "ko: login to docker.io container registry"
+        run: ./bin/ko login docker.io -u ${{ secrets.DOCKER_IO_USERNAME }} -p ${{ secrets.DOCKER_IO_TOKEN }}
+
+      - name: "path: expanding with local binaries"
+        run: echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
+
+      - name: "goreleaser: release"
+        uses: goreleaser/goreleaser-action@v7
+        with:
+          distribution: goreleaser
+          version: "~> v2"
+          args: release --clean
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -38,3 +38,4 @@ bin
 !deploy/kine/mysql/server-csr.json
 !deploy/kine/nats/server-csr.json
 charts/kamaji/charts
+dist

.golangci.yml

@@ -1,53 +1,76 @@
-run:
-  timeout: 10m
-
-linters-settings:
-  revive:
-    rules:
-      - name: dot-imports
-        arguments:
-          - allowedPackages:
-              - "github.com/onsi/ginkgo/v2"
-              - "github.com/onsi/gomega"
-  gci:
-    sections:
-      - standard
-      - default
-      - prefix(github.com/clastix/kamaji/)
-  goheader:
-    template: |-
-      Copyright 2022 Clastix Labs
-      SPDX-License-Identifier: Apache-2.0
-
+version: "2"
 linters:
+  default: all
   disable:
-    - depguard
-    - wrapcheck
-    - mnd
-    - varnamelen
-    - testpackage
-    - tagliatelle
-    - paralleltest
-    - ireturn
-    - err113
-    - gochecknoglobals
-    - wsl
-    - exhaustive
-    - nosprintfhostport
-    - nonamedreturns
-    - interfacebloat
-    - exhaustruct
-    - lll
-    - gosec
-    - gomoddirectives
-    - godox
-    - gochecknoinits
-    - funlen
-    - dupl
     - cyclop
+    - depguard
+    - dupl
+    - err113
+    - exhaustive
+    - exhaustruct
+    - funlen
+    - gochecknoglobals
+    - gochecknoinits
     - gocognit
+    - godox
+    - gomoddirectives
+    - gosec
+    - interfacebloat
+    - ireturn
+    - lll
+    - mnd
     - nestif
+    - nonamedreturns
+    - nosprintfhostport
+    - paralleltest
     - perfsprint
-    # deprecated linters
-    - exportloopref
-  enable-all: true
+    - tagliatelle
+    - testpackage
+    - varnamelen
+    - wrapcheck
+    - wsl
+  settings:
+    staticcheck:
+      checks:
+        - all
+        - -QF1008
+    goheader:
+      template: |-
+        Copyright 2022 Clastix Labs
+        SPDX-License-Identifier: Apache-2.0
+    revive:
+      rules:
+        - name: dot-imports
+          arguments:
+            - allowedPackages:
+                - github.com/onsi/ginkgo/v2
+                - github.com/onsi/gomega
+  exclusions:
+    generated: lax
+    presets:
+      - comments
+      - common-false-positives
+      - legacy
+      - std-error-handling
+    paths:
+      - third_party$
+      - builtin$
+      - examples$
+formatters:
+  enable:
+    - gci
+    - gofmt
+    - gofumpt
+    - goimports
+  settings:
+    gci:
+      sections:
+        - standard
+        - default
+        - prefix(github.com/clastix/kamaji/)
+  exclusions:
+    generated: lax
+    paths:
+      - third_party$
+      - builtin$
+      - examples$

.goreleaser.yaml

@@ -0,0 +1,92 @@
+# yaml-language-server: $schema=https://goreleaser.com/static/schema.json
+version: 2
+
+project_name: kamaji
+
+builds:
+  - id: kamaji
+    main: .
+    binary: "{{ .ProjectName }}-{{ .Os }}-{{ .Arch }}"
+    env:
+      - CGO_ENABLED=0
+    flags:
+      - -trimpath
+    mod_timestamp: '{{ .CommitTimestamp }}'
+    ldflags:
+      - "-X github.com/clastix/kamaji/internal.GitCommit={{.Commit}}"
+      - "-X github.com/clastix/kamaji/internal.GitTag={{.Tag}}"
+      - "-X github.com/clastix/kamaji/internal.GitDirty={{ if eq .GitTreeState \"dirty\" }}.dev{{ end }}"
+      - "-X github.com/clastix/kamaji/internal.BuildTime={{.Date}}"
+      - "-X github.com/clastix/kamaji/internal.GitRepo={{ .GitURL }}"
+    goos:
+      - linux
+    goarch:
+      - amd64
+      - arm
+      - arm64
+
+kos:
+  - repositories:
+      - docker.io/clastix/kamaji
+      - quay.io/clastix/kamaji
+    tags:
+      - "{{ .Tag }}"
+    bare: true
+    preserve_import_paths: false
+    platforms:
+      - linux/amd64
+      - linux/arm64
+      - linux/arm
+
+release:
+  prerelease: auto
+  footer: |
+    **Container Images**
+    ```
+    docker pull clastix/{{ .ProjectName }}:{{ .Tag }}
+    ```
+    
+    > This is an **edge release** and is intended for testing and evaluation purposes only.
+    > It may include experimental features and does not provide the stability guarantees of a production-ready build.
+    >
+    > **Stable release artefacts** are available on a subscription basis from CLASTIX,
+    > the primary contributor to the Kamaji project.
+    >
+    > For production-grade releases and enterprise support,
+    > please refer to CLASTIX's [Support](https://clastix.io/support/) offerings.
+
+    **Full Changelog**: https://github.com/clastix/{{ .ProjectName }}/compare/{{ .PreviousTag }}...{{ .Tag }}
+
+changelog:
+  sort: asc
+  use: github
+  filters:
+    exclude:
+      - 'merge conflict'
+      - Merge pull request
+      - Merge remote-tracking branch
+      - Merge branch
+  groups:
+    - title: '🛠 Dependency updates'
+      regexp: '^.*?(feat|fix)\(deps\)!?:.+$'
+      order: 300
+    - title: '✨ New Features'
+      regexp: '^.*?feat(\([[:word:]]+\))??!?:.+$'
+      order: 100
+    - title: '🐛 Bug fixes'
+      regexp: '^.*?fix(\([[:word:]]+\))??!?:.+$'
+      order: 200
+    - title: '📖 Documentation updates'
+      regexp: ^.*?docs(\([[:word:]]+\))??!?:.+$
+      order: 400
+    - title: '🛡️ Security updates'
+      regexp: ^.*?(sec)(\([[:word:]]+\))??!?:.+$
+      order: 500
+    - title: '🚀 Build process updates'
+      regexp: ^.*?(build|ci)(\([[:word:]]+\))??!?:.+$
+      order: 600
+    - title: '📦 Other work'
+      order: 9999
+
+checksum:
+  name_template: "checksums.txt"

ADOPTERS.md

@@ -8,10 +8,15 @@ Feel free to open a Pull-Request to get yours listed.
 | Type | Name | Since | Website | Use-Case |
 |:-|:-|:-|:-|:-|
 | Vendor | Aknostic | 2023 | [link](https://aknostic.com) | Aknostic is a cloud-native consultancy company using Kamaji to build a Kubernetes based PaaS. |
-| R&D | Aruba | 2024 | [link](https://www.aruba.it/home.aspx) | Aruba Cloud is an Italian Cloud Service Provider evaluating Kamaji to build and offer [Managed Kubernetes Service](https://my.arubacloud.com). |
+| Vendor | Aruba | 2025 | [link](https://www.arubacloud.com/) | Aruba Cloud is an Italian Cloud Service Provider using Kamaji to build and offer [Managed Kubernetes Service](https://my.arubacloud.com). |
+| Vendor | CBWS | 2025 | [link](https://cbws.nl) | CBWS is an European Cloud Provider using Kamaji to build and offer their [Managed Kubernetes Service](https://cbws.nl/cloud/kubernetes/). |
+| Vendor | Coredge | 2025 | [link](https://coredge.io/) | Coredge uses Kamaji in its K8saaS offering to save infrastructure costs in its Sovereign Cloud & AI Infrastructure Platform for end-user organisations. |
 | Vendor | DCloud | 2024 | [link](https://dcloud.co.id) | DCloud is an Indonesian Cloud Provider using Kamaji to build and offer [Managed Kubernetes Service](https://dcloud.co.id/dkubes.html). |
 | Vendor | Dinova | 2025 | [link](https://dinova.one/) | Dinova is an Italian cloud services provider that integrates Kamaji in its datacenters to offer fully managed Kubernetes clusters. |
+| Vendor | Hikube | 2024 | [link](https://hikube.cloud/) | Hikube.cloud is a Swiss sovereign cloud platform with triple replication across three Swiss datacenters, offering enterprise-grade infrastructure with full data sovereignty. |
 | End-user | KINX | 2024 | [link](https://kinx.net/?lang=en) | KINX is an Internet infrastructure service provider and will use kamaji for its new [Managed Kubernetes Service](https://kinx.net/service/cloud/kubernetes/intro/?lang=en). |
+| End-user | Namecheap | 2025 | [link](https://www.namecheap.com/) | Namecheap is an ICANN-accredited domain registrar and web hosting company that provides a wide range of internet-related services and uses Kamaji for both internal and external services. |
+| Vendor | Netalia | 2025 | [link](https://www.netalia.it) | Netalia uses Kamaji for the Italian cloud
 | Vendor | Netsons | 2023 | [link](https://www.netsons.com) | Netsons is an Italian hosting and cloud provider and uses Kamaji in its [Managed Kubernetes](https://www.netsons.com/kubernetes) offering. |
 | Vendor | NVIDIA | 2024 | [link](https://github.com/NVIDIA/doca-platform) | DOCA Platform Framework manages provisioning and service orchestration for NVIDIA Bluefield DPUs. |
 | R&D | Orange | 2024 | [link](https://gitlab.com/Orange-OpenSource/kanod) | Orange is a French telecommunications company using Kamaji for experimental research purpose, with Kanod research solution. |
@@ -26,6 +31,8 @@ Feel free to open a Pull-Request to get yours listed.
 | End-user | Rackspace | 2024 | [link](https://spot.rackspace.com/) | Rackspace Spot uses Kamaji to manage our instances, offering fully-managed kubernetes infrastructure, auctioned in an open market. |
 | R&D | IONOS Cloud | 2024 | [link](https://cloud.ionos.com/) | IONOS Cloud is a German Cloud Provider evaluating Kamaji for its [Managed Kubernetes platform](https://cloud.ionos.com/managed/kubernetes). |
 | Vendor | OVHCloud | 2025 | [link](https://www.ovhcloud.com/) | OVHCloud is a European Cloud Provider that will use Kamaji for its Managed Kubernetes Service offer. |
+| Vendor | WOBCOM GmbH | 2024 | [link](https://www.wobcom.de/) | WOBCOM provides an [**Open Digital Platform**](https://www.wobcom.de/geschaeftskunden/odp/) solution for Smart Cities, which is provided for customers in a Managed Kubernetes provided by Kamaji. |
+| Vendor | Mistral AI | 2025 | [link](https://mistral.ai/products/mistral-compute) | Mistral provides a baremetal kubernetes service that uses Kamaji for control plane management. |
 
 ### Adopter Types
 

Makefile

@@ -47,6 +47,9 @@ GOLANGCI_LINT  ?= $(LOCALBIN)/golangci-lint
 HELM           ?= $(LOCALBIN)/helm
 KIND           ?= $(LOCALBIN)/kind
 KO             ?= $(LOCALBIN)/ko
+GORELEASER     ?= $(LOCALBIN)/goreleaser
+COSIGN         ?= $(LOCALBIN)/cosign
+SYFT           ?= $(LOCALBIN)/syft
 YQ             ?= $(LOCALBIN)/yq
 ENVTEST        ?= $(LOCALBIN)/setup-envtest
 
@@ -68,52 +71,78 @@ all: build
 help: ## Display this help.
 	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf "  \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
 
+##@ Documentation
+
+.PHONY: docs
+docs: ## Serve documentation locally with Docker.
+	docker run --rm -it \
+		-p 8000:8000 \
+		-v "$${PWD}/docs":/docs:Z \
+		-w /docs \
+		squidfunk/mkdocs-material \
+		serve -a 0.0.0.0:8000
+
 ##@ Binary
 
+.PHONY: cosign
+cosign: $(COSIGN) ## Download cosign locally if necessary.
+$(COSIGN): $(LOCALBIN)
+	test -s $(LOCALBIN)/cosign || GOBIN=$(LOCALBIN) go install github.com/sigstore/cosign/v3/cmd/cosign@v3.0.5
+
+.PHONY: syft
+syft: $(SYFT) ## Download syft locally if necessary.
+$(SYFT): $(LOCALBIN)
+	test -s $(LOCALBIN)/syft || GOBIN=$(LOCALBIN) go install github.com/anchore/syft/cmd/syft@v1.42.1
+
+.PHONY: goreleaser
+goreleaser: $(GORELEASER) ## Download goreleaser locally if necessary.
+$(GORELEASER): $(LOCALBIN)
+	test -s $(LOCALBIN)/goreleaser || GOBIN=$(LOCALBIN) go install github.com/goreleaser/goreleaser/v2@v2.14.1
+
 .PHONY: ko
 ko: $(KO) ## Download ko locally if necessary.
 $(KO): $(LOCALBIN)
-	test -s $(LOCALBIN)/ko || GOBIN=$(LOCALBIN) go install github.com/google/ko@v0.14.1
+	test -s $(LOCALBIN)/ko || GOBIN=$(LOCALBIN) CGO_ENABLED=0 go install -ldflags="-s -w" github.com/google/ko@v0.18.1
 
 .PHONY: yq
 yq: $(YQ) ## Download yq locally if necessary.
 $(YQ): $(LOCALBIN)
-	test -s $(LOCALBIN)/yq || GOBIN=$(LOCALBIN) go install github.com/mikefarah/yq/v4@v4.44.2
+	test -s $(LOCALBIN)/yq || GOBIN=$(LOCALBIN) CGO_ENABLED=0 go install -ldflags="-s -w" github.com/mikefarah/yq/v4@v4.44.2
 
 .PHONY: helm
 helm: $(HELM) ## Download helm locally if necessary.
 $(HELM): $(LOCALBIN)
-	test -s $(LOCALBIN)/helm || GOBIN=$(LOCALBIN) go install helm.sh/helm/v3/cmd/helm@v3.9.0
+	test -s $(LOCALBIN)/helm || GOBIN=$(LOCALBIN) CGO_ENABLED=0 go install -ldflags="-s -w" helm.sh/helm/v4/cmd/helm@v4.1.1
 
 .PHONY: ginkgo
 ginkgo: $(GINKGO) ## Download ginkgo locally if necessary.
 $(GINKGO): $(LOCALBIN)
-	test -s $(LOCALBIN)/ginkgo || GOBIN=$(LOCALBIN) go install github.com/onsi/ginkgo/v2/ginkgo
+	test -s $(LOCALBIN)/ginkgo || GOBIN=$(LOCALBIN) CGO_ENABLED=0 go install -ldflags="-s -w" github.com/onsi/ginkgo/v2/ginkgo
 
 .PHONY: kind
 kind: $(KIND) ## Download kind locally if necessary.
 $(KIND): $(LOCALBIN)
-	test -s $(LOCALBIN)/kind || GOBIN=$(LOCALBIN) go install sigs.k8s.io/kind/cmd/kind@v0.14.0
+	test -s $(LOCALBIN)/kind || GOBIN=$(LOCALBIN) CGO_ENABLED=0 go install -ldflags="-s -w" sigs.k8s.io/kind/cmd/kind@v0.14.0
 
 .PHONY: controller-gen
 controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessary.
 $(CONTROLLER_GEN): $(LOCALBIN)
-	test -s $(LOCALBIN)/controller-gen || GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-tools/cmd/controller-gen@v0.16.1
+	test -s $(LOCALBIN)/controller-gen || GOBIN=$(LOCALBIN) CGO_ENABLED=0 go install -ldflags="-s -w" sigs.k8s.io/controller-tools/cmd/controller-gen@v0.20.0
 
 .PHONY: golangci-lint
 golangci-lint: $(GOLANGCI_LINT) ## Download golangci-lint locally if necessary.
 $(GOLANGCI_LINT): $(LOCALBIN)
-	test -s $(LOCALBIN)/golangci-lint || GOBIN=$(LOCALBIN) go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.62.2
+	test -s $(LOCALBIN)/golangci-lint || GOBIN=$(LOCALBIN) CGO_ENABLED=0 go install -ldflags="-s -w" github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.0.2
 
 .PHONY: apidocs-gen
 apidocs-gen: $(APIDOCS_GEN)  ## Download crdoc locally if necessary.
 $(APIDOCS_GEN): $(LOCALBIN)
-	test -s $(LOCALBIN)/crdoc || GOBIN=$(LOCALBIN) go install fybrik.io/crdoc@latest
+	test -s $(LOCALBIN)/crdoc || GOBIN=$(LOCALBIN) CGO_ENABLED=0 go install -ldflags="-s -w" fybrik.io/crdoc@latest
 
 .PHONY: envtest
 envtest: $(ENVTEST) ## Download envtest-setup locally if necessary.
 $(ENVTEST): $(LOCALBIN)
-	test -s $(LOCALBIN)/setup-envtest || GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-runtime/tools/setup-envtest@$(ENVTEST_VERSION)
+	test -s $(LOCALBIN)/setup-envtest || GOBIN=$(LOCALBIN) CGO_ENABLED=0 go install -ldflags="-s -w" sigs.k8s.io/controller-runtime/tools/setup-envtest@$(ENVTEST_VERSION)
 
 ##@ Development
 
@@ -129,9 +158,18 @@ webhook: controller-gen yq
 	$(YQ) -i 'map(.clientConfig.service.namespace |= "{{ .Release.Namespace }}")' ./charts/kamaji/controller-gen/validating-webhook.yaml
 
 crds: controller-gen yq
+	# kamaji chart
 	$(CONTROLLER_GEN) crd webhook paths="./..." output:stdout | $(YQ) 'select(documentIndex == 0)' > ./charts/kamaji/crds/kamaji.clastix.io_datastores.yaml
-	$(CONTROLLER_GEN) crd webhook paths="./..." output:stdout | $(YQ) 'select(documentIndex == 1)' > ./charts/kamaji/crds/kamaji.clastix.io_tenantcontrolplanes.yaml
+	$(CONTROLLER_GEN) crd webhook paths="./..." output:stdout | $(YQ) 'select(documentIndex == 1)' > ./charts/kamaji/crds/kamaji.clastix.io_kubeconfiggenerators.yaml
+	$(CONTROLLER_GEN) crd webhook paths="./..." output:stdout | $(YQ) 'select(documentIndex == 2)' > ./charts/kamaji/crds/kamaji.clastix.io_tenantcontrolplanes.yaml
 	$(YQ) -i '. *n load("./charts/kamaji/controller-gen/crd-conversion.yaml")' ./charts/kamaji/crds/kamaji.clastix.io_tenantcontrolplanes.yaml
+	# kamaji-crds chart
+	cp ./charts/kamaji/controller-gen/crd-conversion.yaml ./charts/kamaji-crds/hack/crd-conversion.yaml
+	$(YQ) '.spec' ./charts/kamaji/crds/kamaji.clastix.io_datastores.yaml > ./charts/kamaji-crds/hack/kamaji.clastix.io_datastores_spec.yaml
+	$(YQ) '.spec' ./charts/kamaji/crds/kamaji.clastix.io_tenantcontrolplanes.yaml > ./charts/kamaji-crds/hack/kamaji.clastix.io_tenantcontrolplanes_spec.yaml
+	$(YQ) '.spec' ./charts/kamaji/crds/kamaji.clastix.io_kubeconfiggenerators.yaml > ./charts/kamaji-crds/hack/kamaji.clastix.io_kubeconfiggenerators_spec.yaml
+	$(YQ) -i '.conversion.webhook.clientConfig.service.name = "{{ .Values.kamajiService }}"' ./charts/kamaji-crds/hack/kamaji.clastix.io_tenantcontrolplanes_spec.yaml
+	$(YQ) -i '.conversion.webhook.clientConfig.service.namespace = "{{ .Values.kamajiNamespace }}"' ./charts/kamaji-crds/hack/kamaji.clastix.io_tenantcontrolplanes_spec.yaml
 
 manifests: rbac webhook crds ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects.
 
@@ -144,11 +182,10 @@ golint: golangci-lint ## Linting the code according to the styling guide.
 ## Run unit tests (all tests except E2E).
 .PHONY: test
 test: envtest ginkgo
-	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" $(GINKGO) -r -v --trace \
+	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" $(GINKGO) -r -v -coverprofile cover.out --trace \
 		./api/... \
 		./cmd/... \
 		./internal/... \
-		-coverprofile cover.out
 
 _datastore-mysql:
 	$(MAKE) NAME=$(NAME) -C deploy/kine/mysql mariadb
@@ -169,7 +206,7 @@ datastore-postgres:
 	$(MAKE) NAME=gold _datastore-postgres
 
 _datastore-etcd:
-	$(HELM) upgrade --install etcd-$(NAME) clastix/kamaji-etcd --create-namespace -n etcd-system --set datastore.enabled=true --set fullnameOverride=etcd-$(NAME)
+	$(HELM) upgrade --install etcd-$(NAME) clastix/kamaji-etcd --create-namespace -n $(NAMESPACE) --set datastore.enabled=true --set fullnameOverride=etcd-$(NAME) $(EXTRA_ARGS)
 
 _datastore-nats:
 	$(MAKE) NAME=$(NAME) NAMESPACE=nats-system -C deploy/kine/nats nats
@@ -178,9 +215,11 @@ _datastore-nats:
 datastore-etcd: helm
 	$(HELM) repo add clastix https://clastix.github.io/charts
 	$(HELM) repo update
-	$(MAKE) NAME=bronze _datastore-etcd
-	$(MAKE) NAME=silver _datastore-etcd
-	$(MAKE) NAME=gold _datastore-etcd
+	$(MAKE) NAME=bronze NAMESPACE=etcd-system _datastore-etcd
+	$(MAKE) NAME=silver NAMESPACE=etcd-system _datastore-etcd
+	$(MAKE) NAME=gold NAMESPACE=etcd-system _datastore-etcd
+	$(MAKE) NAME=primary NAMESPACE=kamaji-system EXTRA_ARGS='--set certManager.enabled=true --set certManager.issuerRef.kind=Issuer --set certManager.issuerRef.name=kamaji-selfsigned-issuer --set selfSignedCertificates.enabled=false' _datastore-etcd
+	$(MAKE) NAME=secondary NAMESPACE=kamaji-system EXTRA_ARGS='--set certManager.enabled=true --set certManager.ca.create=false --set certManager.ca.nameOverride=etcd-primary-ca --set certManager.issuerRef.kind=Issuer --set certManager.issuerRef.name=kamaji-selfsigned-issuer --set selfSignedCertificates.enabled=false' _datastore-etcd
 
 datastore-nats: helm
 	$(HELM) repo add nats https://nats-io.github.io/k8s/helm/charts/
@@ -226,23 +265,50 @@ metallb:
 	kubectl apply -f "https://raw.githubusercontent.com/metallb/metallb/$$(curl "https://api.github.com/repos/metallb/metallb/releases/latest" | jq -r ".tag_name")/config/manifests/metallb-native.yaml"
 	kubectl wait pods -n metallb-system -l app=metallb,component=controller --for=condition=Ready --timeout=10m
 	kubectl wait pods -n metallb-system -l app=metallb,component=speaker --for=condition=Ready --timeout=2m
-	cat hack/metallb.yaml | sed -E "s|172.19|$$(docker network inspect -f '{{range .IPAM.Config}}{{.Gateway}}{{end}}' kind | sed -E 's|^([0-9]+\.[0-9]+)\..*$$|\1|g')|g" | kubectl apply -f -
+	@IPV4_PREFIX=$$(docker network inspect kind \
+		-f '{{range .IPAM.Config}}{{println .Subnet " " .Gateway}}{{end}}' \
+		| grep -v ':' \
+		| awk '{print $$2}' \
+		| sed -E 's|^([0-9]+\.[0-9]+)\..*$$|\1|'); \
+	sed -E "s|172\.19|$$IPV4_PREFIX|g" hack/metallb.yaml | kubectl apply -f -
 
 cert-manager:
 	$(HELM) repo add jetstack https://charts.jetstack.io
 	$(HELM) upgrade --install cert-manager jetstack/cert-manager --namespace certmanager-system --create-namespace --set "installCRDs=true"
 
+gateway-api:
+	kubectl apply \
+		--server-side \
+		--force-conflicts \
+		--field-manager=helm \
+		-f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.4.0/standard-install.yaml
+	# Required for the TLSRoutes. Experimentals.
+	kubectl apply \
+		--server-side \
+		--force-conflicts \
+		--field-manager=helm \
+		-f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.4.0/experimental-install.yaml
+	kubectl wait --for=condition=Established crd/gateways.gateway.networking.k8s.io --timeout=60s
+
+envoy-gateway: gateway-api helm ## Install Envoy Gateway for Gateway API tests.
+	$(HELM) upgrade --install eg oci://docker.io/envoyproxy/gateway-helm --version v1.6.1 -n envoy-gateway-system --create-namespace
+	kubectl wait --timeout=5m -n envoy-gateway-system deployment/envoy-gateway --for=condition=Available
+
 load: kind
 	$(KIND) load docker-image --name kamaji ${CONTAINER_REPOSITORY}:${VERSION}
 
 ##@ e2e
 
 .PHONY: env
-env:
-	@make -C deploy/kind kind ingress-nginx
+env: kind
+	$(KIND) create cluster --name kamaji
+
+cleanup: kind
+	$(KIND) delete cluster --name kamaji
 
 .PHONY: e2e
-e2e: env build load helm ginkgo cert-manager ## Create a KinD cluster, install Kamaji on it and run the test suite.
+e2e: env build load helm ginkgo cert-manager gateway-api envoy-gateway ## Create a KinD cluster, install Kamaji on it and run the test suite.
+	$(HELM) upgrade --debug --install kamaji-crds ./charts/kamaji-crds --create-namespace --namespace kamaji-system
 	$(HELM) repo add clastix https://clastix.github.io/charts
 	$(HELM) dependency build ./charts/kamaji
 	$(HELM) upgrade --debug --install kamaji ./charts/kamaji --create-namespace --namespace kamaji-system --set "image.tag=$(VERSION)" --set "image.pullPolicy=Never" --set "telemetry.disabled=true"
@@ -251,6 +317,15 @@ e2e: env build load helm ginkgo cert-manager ## Create a KinD cluster, install K
 
 ##@ Document
 
+CAPI_URL = https://github.com/clastix/cluster-api-control-plane-provider-kamaji.git
+CAPI_DIR := $(shell mktemp -d)
+CRDS_DIR := $(shell mktemp -d)
+
 .PHONY: apidoc
 apidoc: apidocs-gen
-	$(APIDOCS_GEN) crdoc --resources charts/kamaji/crds --output docs/content/reference/api.md --template docs/templates/reference-cr.tmpl
+	@cp charts/kamaji/crds/*.yaml $(CRDS_DIR)
+	@git clone $(CAPI_URL) $(CAPI_DIR)
+	@cp $(CAPI_DIR)/config/crd/bases/*.yaml $(CRDS_DIR)
+	@rm -rf $(CAPI_DIR)
+	$(APIDOCS_GEN) crdoc --resources $(CRDS_DIR) --output docs/content/reference/api.md --template docs/templates/reference-cr.tmpl
+	@rm -rf $(CRDS_DIR)

NOTICE

@@ -0,0 +1,15 @@
+Kamaji — The Kubernetes Control Plane Manager: copyright 2022 Clastix Labs
+Licensed under the Apache License, Version 2.0: https://kamaji.clastix.io
+
+This product includes software developed by Clastix Labs and the Kamaji open-source community under the Apache License, Version 2.0.
+
+Kamaji powers Kubernetes Control Planes at scale for companies worldwide.
+
+We encourage all commercial products and services using Kamaji to acknowledge this publicly and join our growing ecosystem of adopters.
+
+You can support the Kamaji community by:
+- Listing Kamaji in your product's "Open Source Credits" or similar section
+- Adding your organization to the Adopters list on GitHub: https://github.com/clastix/kamaji/blob/master/ADOPTERS.md
+- Mentioning Kamaji on your company or product website
+
+Public acknowledgement strengthens the open-source ecosystem and helps ensure the sustainability of the project you rely on.

PROJECT

@@ -7,6 +7,15 @@ plugins:
 projectName: operator
 repo: github.com/clastix/kamaji
 resources:
+- api:
+    crdVersion: v1
+    namespaced: false
+  controller: true
+  domain: clastix.io
+  group: kamaji
+  kind: KubeconfigGenerator
+  path: github.com/clastix/kamaji/api/v1alpha1
+  version: v1alpha1
 - api:
     crdVersion: v1
     namespaced: true

README.md

@@ -123,6 +123,8 @@ Since Kamaji is just focusing on the Control Plane a [Kamaji's Cluster API Contr
 - YouTube ▶️ [Rancher & Kamaji: solving multitenancy challenges in the Kubernetes world](https://www.youtube.com/watch?v=VXHNrMmlF8U)
 - YouTube ▶️ [Enabling Self-Service Kubernetes clusters with Kamaji and Paralus](https://www.youtube.com/watch?v=JWA2LwZazM0)
 - YouTube ▶️ [Hosted Control Plane on Kubernetes (HPC) with Kamaji and K0mostron by Hervé Leclerc, ALTER WAY](https://www.youtube.com/watch?v=vmRdE2ngn78)
+- Medium 📖 [Set up Virtual Control Planes with Kamaji on Minikube, by Ben Soer](https://medium.com/@bensoer/set-up-virtual-control-planes-with-kamaji-on-minikube-a540be0275aa)
+- Hands-On tutorial 📖 [How to build your own managed Kubernetes service on Hetzner Cloud, by Hans Jörg Wieland](https://wieland.tech/blog/kamaji-cluster-api-and-etcd)
 
 ### 🏷️ Versioning
 

api/v1alpha1/datastore_types.go

@@ -9,6 +9,7 @@ import (
 )
 
 //+kubebuilder:validation:Enum=etcd;MySQL;PostgreSQL;NATS
+//+kubebuilder:validation:XValidation:rule="self == oldSelf",message="Datastore driver is immutable"
 
 type Driver string
 
@@ -24,6 +25,14 @@ var (
 type Endpoints []string
 
 // DataStoreSpec defines the desired state of DataStore.
+// +kubebuilder:validation:XValidation:rule="(self.driver == \"etcd\") ? (self.tlsConfig != null && (has(self.tlsConfig.certificateAuthority.privateKey.secretReference) || has(self.tlsConfig.certificateAuthority.privateKey.content))) : true", message="certificateAuthority privateKey must have secretReference or content when driver is etcd"
+// +kubebuilder:validation:XValidation:rule="(self.driver == \"etcd\") ? (self.tlsConfig != null && (has(self.tlsConfig.clientCertificate.certificate.secretReference) || has(self.tlsConfig.clientCertificate.certificate.content))) : true", message="clientCertificate must have secretReference or content when driver is etcd"
+// +kubebuilder:validation:XValidation:rule="(self.driver == \"etcd\") ? (self.tlsConfig != null && (has(self.tlsConfig.clientCertificate.privateKey.secretReference) || has(self.tlsConfig.clientCertificate.privateKey.content))) : true", message="clientCertificate privateKey must have secretReference or content when driver is etcd"
+// +kubebuilder:validation:XValidation:rule="(self.driver != \"etcd\" && has(self.tlsConfig) && has(self.tlsConfig.clientCertificate)) ? (((has(self.tlsConfig.clientCertificate.certificate.secretReference) || has(self.tlsConfig.clientCertificate.certificate.content)))) : true", message="When driver is not etcd and tlsConfig exists, clientCertificate must be null or contain valid content"
+// +kubebuilder:validation:XValidation:rule="(self.driver != \"etcd\" && has(self.basicAuth)) ? ((has(self.basicAuth.username.secretReference) || has(self.basicAuth.username.content))) : true", message="When driver is not etcd and basicAuth exists, username must have secretReference or content"
+// +kubebuilder:validation:XValidation:rule="(self.driver != \"etcd\" && has(self.basicAuth)) ? ((has(self.basicAuth.password.secretReference) || has(self.basicAuth.password.content))) : true", message="When driver is not etcd and basicAuth exists, password must have secretReference or content"
+// +kubebuilder:validation:XValidation:rule="(self.driver != \"etcd\") ? (has(self.tlsConfig) || has(self.basicAuth)) : true", message="When driver is not etcd, either tlsConfig or basicAuth must be provided"
+// +kubebuilder:validation:XValidation:rule="oldSelf == null || self.driver == oldSelf.driver", message="driver is immutable and cannot be changed after creation"
 type DataStoreSpec struct {
 	// The driver to use to connect to the shared datastore.
 	Driver Driver `json:"driver"`
@@ -80,16 +89,32 @@ type SecretReference struct {
 	KeyPath secretReferKeyPath `json:"keyPath"`
 }
 
+const (
+	DataStoreTCPFinalizer = "kamaji.clastix.io/TenantControlPlane"
+
+	DataStoreConditionValidType           = "kamaji.clastix.io/DataStoreValidation"
+	DataStoreConditionAllowedDeletionType = "kamaji.clastix.io/DataStoreAllowedDeletion"
+)
+
 // DataStoreStatus defines the observed state of DataStore.
 type DataStoreStatus struct {
+	// ObservedGeneration represents the .metadata.generation that was last reconciled.
+	// +optional
+	ObservedGeneration int64 `json:"observedGeneration,omitempty"`
 	// List of the Tenant Control Planes, namespaced named, using this data store.
 	UsedBy []string `json:"usedBy,omitempty"`
+	// Conditions contains the validation conditions for the given Datastore.
+	Conditions []metav1.Condition `json:"conditions,omitempty"`
+	// Ready returns if the DataStore is accepted and ready to get used:
+	// Kamaji will ensure certificates are available, or correctly referenced.
+	Ready bool `json:"ready"`
 }
 
 //+kubebuilder:object:root=true
 //+kubebuilder:subresource:status
 //+kubebuilder:resource:scope=Cluster
 //+kubebuilder:printcolumn:name="Driver",type="string",JSONPath=".spec.driver",description="Kamaji data store driver"
+//+kubebuilder:printcolumn:name="Ready",type="boolean",JSONPath=".status.ready",description="DataStore validated and ready for use"
 //+kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="Age"
 //+kubebuilder:metadata:annotations={"cert-manager.io/inject-ca-from=kamaji-system/kamaji-serving-cert"}
 

api/v1alpha1/groupversion_info.go

@@ -4,7 +4,6 @@
 // Package v1alpha1 contains API Schema definitions for the kamaji v1alpha1 API group
 // +kubebuilder:object:generate=true
 // +groupName=kamaji.clastix.io
-//nolint
 package v1alpha1
 
 import (

api/v1alpha1/indexer_datastore_usedsecret.go

@@ -52,12 +52,14 @@ func (d *DatastoreUsedSecret) ExtractValue() client.IndexerFunc {
 				res = append(res, d.namespacedName(*ds.Spec.TLSConfig.CertificateAuthority.PrivateKey.SecretRef))
 			}
 
-			if ds.Spec.TLSConfig.ClientCertificate.Certificate.SecretRef != nil {
-				res = append(res, d.namespacedName(*ds.Spec.TLSConfig.ClientCertificate.Certificate.SecretRef))
-			}
+			if ds.Spec.TLSConfig.ClientCertificate != nil {
+				if ds.Spec.TLSConfig.ClientCertificate.Certificate.SecretRef != nil {
+					res = append(res, d.namespacedName(*ds.Spec.TLSConfig.ClientCertificate.Certificate.SecretRef))
+				}
 
-			if ds.Spec.TLSConfig.ClientCertificate.PrivateKey.SecretRef != nil {
-				res = append(res, d.namespacedName(*ds.Spec.TLSConfig.ClientCertificate.PrivateKey.SecretRef))
+				if ds.Spec.TLSConfig.ClientCertificate.PrivateKey.SecretRef != nil {
+					res = append(res, d.namespacedName(*ds.Spec.TLSConfig.ClientCertificate.PrivateKey.SecretRef))
+				}
 			}
 		}
 

api/v1alpha1/indexer_gateway_listener.go

@@ -0,0 +1,47 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package v1alpha1
+
+import (
+	"context"
+	"fmt"
+
+	controllerruntime "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	gatewayv1 "sigs.k8s.io/gateway-api/apis/v1"
+)
+
+const (
+	GatewayListenerNameKey = "spec.listeners.name"
+)
+
+type GatewayListener struct{}
+
+func (g *GatewayListener) Object() client.Object {
+	return &gatewayv1.Gateway{}
+}
+
+func (g *GatewayListener) Field() string {
+	return GatewayListenerNameKey
+}
+
+func (g *GatewayListener) ExtractValue() client.IndexerFunc {
+	return func(object client.Object) []string {
+		gateway := object.(*gatewayv1.Gateway) //nolint:forcetypeassert
+
+		listenerNames := make([]string, 0, len(gateway.Spec.Listeners))
+		for _, listener := range gateway.Spec.Listeners {
+			// Create a composite key: namespace/gatewayName/listenerName
+			// This allows us to look up gateways by listener name while ensuring uniqueness
+			key := fmt.Sprintf("%s/%s/%s", gateway.Namespace, gateway.Name, listener.Name)
+			listenerNames = append(listenerNames, key)
+		}
+
+		return listenerNames
+	}
+}
+
+func (g *GatewayListener) SetupWithManager(ctx context.Context, mgr controllerruntime.Manager) error {
+	return mgr.GetFieldIndexer().IndexField(ctx, g.Object(), g.Field(), g.ExtractValue())
+}

api/v1alpha1/kubeconfiggenerator_types.go

@@ -0,0 +1,94 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package v1alpha1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+var (
+	ManagedByLabel  = "kamaji.clastix.io/managed-by"
+	ManagedForLabel = "kamaji.clastix.io/managed-for"
+)
+
+//+kubebuilder:object:root=true
+//+kubebuilder:subresource:status
+//+kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="Age"
+//+kubebuilder:metadata:annotations={"cert-manager.io/inject-ca-from=kamaji-system/kamaji-serving-cert"}
+//+kubebuilder:resource:scope=Cluster,shortName=kc,categories=kamaji
+
+// KubeconfigGenerator is the Schema for the kubeconfiggenerators API.
+type KubeconfigGenerator struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec   KubeconfigGeneratorSpec   `json:"spec,omitempty"`
+	Status KubeconfigGeneratorStatus `json:"status,omitempty"`
+}
+
+// CompoundValue allows defining a static, or a dynamic value.
+// Options are mutually exclusive, just one should be picked up.
+// +kubebuilder:validation:XValidation:rule="(has(self.stringValue) || has(self.fromDefinition)) && !(has(self.stringValue) && has(self.fromDefinition))",message="Either stringValue or fromDefinition must be set, but not both."
+type CompoundValue struct {
+	// StringValue is a static string value.
+	StringValue string `json:"stringValue,omitempty"`
+	// FromDefinition is used to generate a dynamic value,
+	// it uses the dot notation to access fields from the referenced TenantControlPlane object:
+	// e.g.: metadata.name
+	FromDefinition string `json:"fromDefinition,omitempty"`
+}
+
+type KubeconfigGeneratorSpec struct {
+	// NamespaceSelector is used to filter Namespaces from which the generator should extract TenantControlPlane objects.
+	NamespaceSelector metav1.LabelSelector `json:"namespaceSelector,omitempty"`
+	// TenantControlPlaneSelector is used to filter the TenantControlPlane objects that should be address by the generator.
+	TenantControlPlaneSelector metav1.LabelSelector `json:"tenantControlPlaneSelector,omitempty"`
+	// Groups is resolved a set of strings used to assign the x509 organisations field.
+	// It will be recognised by Kubernetes as user groups.
+	Groups []CompoundValue `json:"groups,omitempty"`
+	// User resolves to a string to identify the client, assigned to the x509 Common Name field.
+	User CompoundValue `json:"user"`
+	// ControlPlaneEndpointFrom is the key used to extract the Tenant Control Plane endpoint that must be used by the generator.
+	// The targeted Secret is the `${TCP}-admin-kubeconfig` one, default to `admin.svc`.
+	//+kubebuilder:default="admin.svc"
+	ControlPlaneEndpointFrom string `json:"controlPlaneEndpointFrom,omitempty"`
+}
+
+type KubeconfigGeneratorStatusError struct {
+	// Resource is the Namespaced name of the errored resource.
+	//+kubebuilder:validation:Required
+	Resource string `json:"resource"`
+	// Message is the error message recorded upon the last generator run.
+	//+kubebuilder:validation:Required
+	Message string `json:"message"`
+}
+
+// KubeconfigGeneratorStatus defines the observed state of KubeconfigGenerator.
+type KubeconfigGeneratorStatus struct {
+	// ObservedGeneration represents the .metadata.generation that was last reconciled.
+	// +optional
+	ObservedGeneration int64 `json:"observedGeneration,omitempty"`
+	// Resources is the sum of targeted TenantControlPlane objects.
+	//+kubebuilder:default=0
+	Resources int `json:"resources"`
+	// AvailableResources is the sum of successfully generated resources.
+	// In case of a different value compared to Resources, check the field errors.
+	//+kubebuilder:default=0
+	AvailableResources int `json:"availableResources"`
+	// Errors is the list of failed kubeconfig generations.
+	Errors []KubeconfigGeneratorStatusError `json:"errors,omitempty"`
+}
+
+//+kubebuilder:object:root=true
+
+// KubeconfigGeneratorList contains a list of TenantControlPlane.
+type KubeconfigGeneratorList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []KubeconfigGenerator `json:"items"`
+}
+
+func init() {
+	SchemeBuilder.Register(&KubeconfigGenerator{}, &KubeconfigGeneratorList{})
+}

api/v1alpha1/tenantcontrolplane_const.go

@@ -0,0 +1,10 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package v1alpha1
+
+const (
+	// PausedReconciliationAnnotation is an annotation that can be applied to
+	// Tenant Control Plane objects to prevent the controller from processing such a resource.
+	PausedReconciliationAnnotation = "kamaji.clastix.io/paused"
+)

api/v1alpha1/tenantcontrolplane_funcs.go

@@ -8,8 +8,8 @@ import (
 	"fmt"
 	"net"
 	"strconv"
+	"strings"
 
-	"github.com/pkg/errors"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -26,12 +26,12 @@ func (in *TenantControlPlane) AssignedControlPlaneAddress() (string, int32, erro
 
 	address, portString, err := net.SplitHostPort(in.Status.ControlPlaneEndpoint)
 	if err != nil {
-		return "", 0, errors.Wrap(err, "cannot split host port from Tenant Control Plane endpoint")
+		return "", 0, fmt.Errorf("cannot split host port from Tenant Control Plane endpoint: %w", err)
 	}
 
 	port, err := strconv.Atoi(portString)
 	if err != nil {
-		return "", 0, errors.Wrap(err, "cannot convert Tenant Control Plane port from endpoint")
+		return "", 0, fmt.Errorf("cannot convert Tenant Control Plane port from endpoint: %w", err)
 	}
 
 	return address, int32(port), nil
@@ -46,7 +46,7 @@ func (in *TenantControlPlane) DeclaredControlPlaneAddress(ctx context.Context, c
 	svc := &corev1.Service{}
 	err := client.Get(ctx, types.NamespacedName{Namespace: in.GetNamespace(), Name: in.GetName()}, svc)
 	if err != nil {
-		return "", errors.Wrap(err, "cannot retrieve Service for the TenantControlPlane")
+		return "", fmt.Errorf("cannot retrieve Service for the TenantControlPlane: %w", err)
 	}
 
 	switch {
@@ -95,3 +95,17 @@ func getLoadBalancerAddress(ingress []corev1.LoadBalancerIngress) (string, error
 
 	return "", kamajierrors.MissingValidIPError{}
 }
+
+func (in *TenantControlPlane) normalizeNamespaceName() string {
+	// The dash character (-) must be replaced with an underscore, PostgreSQL is complaining about it:
+	// https://github.com/clastix/kamaji/issues/328
+	return strings.ReplaceAll(fmt.Sprintf("%s_%s", in.GetNamespace(), in.GetName()), "-", "_")
+}
+
+func (in *TenantControlPlane) GetDefaultDatastoreUsername() string {
+	return in.normalizeNamespaceName()
+}
+
+func (in *TenantControlPlane) GetDefaultDatastoreSchema() string {
+	return in.normalizeNamespaceName()
+}

api/v1alpha1/tenantcontrolplane_jsonpatch.go

@@ -0,0 +1,83 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package v1alpha1
+
+import (
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+)
+
+type JSONPatches []JSONPatch
+
+type JSONPatch struct {
+	// Op is the RFC 6902 JSON Patch operation.
+	//+kubebuilder:validation:Enum=add;remove;replace;move;copy;test
+	Op string `json:"op"`
+	// Path specifies the target location in the JSON document. Use "/" to separate keys; "-" for appending to arrays.
+	Path string `json:"path"`
+	// From specifies the source location for move or copy operations.
+	From string `json:"from,omitempty"`
+	// Value is the operation value to be used when Op is add, replace, test.
+	Value *apiextensionsv1.JSON `json:"value,omitempty"`
+}
+
+func (p JSONPatches) ToJSON() ([]byte, error) {
+	if len(p) == 0 {
+		return []byte("[]"), nil
+	}
+
+	buf := make([]byte, 0, 256)
+	buf = append(buf, '[')
+
+	for i, patch := range p {
+		if i > 0 {
+			buf = append(buf, ',')
+		}
+
+		buf = append(buf, '{')
+
+		buf = append(buf, `"op":"`...)
+		buf = appendEscapedString(buf, patch.Op)
+		buf = append(buf, '"')
+
+		buf = append(buf, `,"path":"`...)
+		buf = appendEscapedString(buf, patch.Path)
+		buf = append(buf, '"')
+
+		if patch.From != "" {
+			buf = append(buf, `,"from":"`...)
+			buf = appendEscapedString(buf, patch.From)
+			buf = append(buf, '"')
+		}
+
+		if patch.Value != nil {
+			buf = append(buf, `,"value":`...)
+			buf = append(buf, patch.Value.Raw...)
+		}
+
+		buf = append(buf, '}')
+	}
+
+	buf = append(buf, ']')
+
+	return buf, nil
+}
+
+func appendEscapedString(dst []byte, s string) []byte {
+	for i := range s {
+		switch s[i] {
+		case '\\', '"':
+			dst = append(dst, '\\', s[i])
+		case '\n':
+			dst = append(dst, '\\', 'n')
+		case '\r':
+			dst = append(dst, '\\', 'r')
+		case '\t':
+			dst = append(dst, '\\', 't')
+		default:
+			dst = append(dst, s[i])
+		}
+	}
+
+	return dst
+}

api/v1alpha1/tenantcontrolplane_status.go

@@ -8,6 +8,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	networkingv1 "k8s.io/api/networking/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	gatewayv1 "sigs.k8s.io/gateway-api/apis/v1"
 )
 
 // APIServerCertificatesStatus defines the observed state of ETCD Certificate for API server.
@@ -122,6 +123,12 @@ type ExternalKubernetesObjectStatus struct {
 	LastUpdate metav1.Time `json:"lastUpdate,omitempty"`
 }
 
+type KonnectivityAgentStatus struct {
+	ExternalKubernetesObjectStatus `json:",inline"`
+
+	Mode KonnectivityAgentMode `json:"mode,omitempty"`
+}
+
 // KonnectivityStatus defines the status of Konnectivity as Addon.
 type KonnectivityStatus struct {
 	Enabled            bool                            `json:"enabled"`
@@ -130,8 +137,9 @@ type KonnectivityStatus struct {
 	Kubeconfig         KubeconfigStatus                `json:"kubeconfig,omitempty"`
 	ServiceAccount     ExternalKubernetesObjectStatus  `json:"sa,omitempty"`
 	ClusterRoleBinding ExternalKubernetesObjectStatus  `json:"clusterrolebinding,omitempty"`
-	Agent              ExternalKubernetesObjectStatus  `json:"agent,omitempty"`
+	Agent              KonnectivityAgentStatus         `json:"agent,omitempty"`
 	Service            KubernetesServiceStatus         `json:"service,omitempty"`
+	Gateway            *KubernetesGatewayStatus        `json:"gateway,omitempty"`
 }
 
 type KonnectivityConfigMap struct {
@@ -154,6 +162,9 @@ type AddonsStatus struct {
 
 // TenantControlPlaneStatus defines the observed state of TenantControlPlane.
 type TenantControlPlaneStatus struct {
+	// ObservedGeneration represents the .metadata.generation that was last reconciled.
+	// +optional
+	ObservedGeneration int64 `json:"observedGeneration,omitempty"`
 	// Storage Status contains information about Kubernetes storage system
 	Storage StorageStatus `json:"storage,omitempty"`
 	// Certificates contains information about the different certificates
@@ -181,13 +192,17 @@ type KubernetesStatus struct {
 	Deployment KubernetesDeploymentStatus `json:"deployment,omitempty"`
 	Service    KubernetesServiceStatus    `json:"service,omitempty"`
 	Ingress    *KubernetesIngressStatus   `json:"ingress,omitempty"`
+	Gateway    *KubernetesGatewayStatus   `json:"gateway,omitempty"`
 }
 
-// +kubebuilder:validation:Enum=Provisioning;CertificateAuthorityRotating;Upgrading;Migrating;Ready;NotReady
+// +kubebuilder:validation:Enum=Unknown;Provisioning;CertificateAuthorityRotating;Upgrading;Migrating;Ready;NotReady;Sleeping;WriteLimited
 type KubernetesVersionStatus string
 
 var (
+	VersionUnknown      KubernetesVersionStatus = "Unknown"
 	VersionProvisioning KubernetesVersionStatus = "Provisioning"
+	VersionSleeping     KubernetesVersionStatus = "Sleeping"
+	VersionWriteLimited KubernetesVersionStatus = "WriteLimited"
 	VersionCARotating   KubernetesVersionStatus = "CertificateAuthorityRotating"
 	VersionUpgrading    KubernetesVersionStatus = "Upgrading"
 	VersionMigrating    KubernetesVersionStatus = "Migrating"
@@ -235,3 +250,25 @@ type KubernetesIngressStatus struct {
 	// The namespace which the Ingress for the given cluster is deployed.
 	Namespace string `json:"namespace"`
 }
+
+type GatewayAccessPoint struct {
+	Type  *gatewayv1.AddressType `json:"type"`
+	Value string                 `json:"value"`
+	Port  int32                  `json:"port"`
+	URLs  []string               `json:"urls,omitempty"`
+}
+
+// +k8s:deepcopy-gen=false
+type RouteStatus = gatewayv1.RouteStatus
+
+// KubernetesGatewayStatus defines the status for the Tenant Control Plane Gateway in the management cluster.
+type KubernetesGatewayStatus struct {
+	// The TLSRoute status as resported by the gateway controllers.
+	RouteStatus `json:",inline"`
+
+	// Reference to the route created for this tenant.
+	RouteRef corev1.LocalObjectReference `json:"routeRef,omitempty"`
+
+	// A list of valid access points that the route exposes.
+	AccessPoints []GatewayAccessPoint `json:"accessPoints,omitempty"`
+}

api/v1alpha1/tenantcontrolplane_types.go

@@ -7,6 +7,8 @@ import (
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	gatewayv1 "sigs.k8s.io/gateway-api/apis/v1"
 )
 
 // NetworkProfileSpec defines the desired state of NetworkProfile.
@@ -66,13 +68,22 @@ const (
 )
 
 type KubeletSpec struct {
+	// ConfigurationJSONPatches contains the RFC 6902 JSON patches to customise the kubeadm generate configuration,
+	// useful to customise and mangling the configuration according to your needs;
+	// e.g.: configuring the cgroup driver used by Kubelet is possible via the following patch:
+	//
+	// [{"op": "replace", "path": "/cgroupDriver", "value": "systemd"}]
+	ConfigurationJSONPatches JSONPatches `json:"configurationJSONPatches,omitempty"`
 	// Ordered list of the preferred NodeAddressTypes to use for kubelet connections.
-	// Default to Hostname, InternalIP, ExternalIP.
-	//+kubebuilder:default={"Hostname","InternalIP","ExternalIP"}
+	// Default to InternalIP, ExternalIP, Hostname.
+	//+kubebuilder:default={"InternalIP","ExternalIP","Hostname"}
 	//+kubebuilder:validation:MinItems=1
+	//+listType=set
 	PreferredAddressTypes []KubeletPreferredAddressType `json:"preferredAddressTypes,omitempty"`
-	// CGroupFS defines the  cgroup driver for Kubelet
+	// CGroupFS defines the cgroup driver for Kubelet
 	// https://kubernetes.io/docs/tasks/administer-cluster/kubeadm/configure-cgroup-driver/
+	//
+	// Deprecated: use ConfigurationJSONPatches.
 	CGroupFS CGroupDriver `json:"cgroupfs,omitempty"`
 }
 
@@ -88,6 +99,32 @@ type KubernetesSpec struct {
 	AdmissionControllers AdmissionControllers `json:"admissionControllers,omitempty"`
 }
 
+type AdditionalPort struct {
+	// The name of this port within the Service created by Kamaji.
+	// This must be a DNS_LABEL, must have unique names, and cannot be `kube-apiserver`, or `konnectivity-server`.
+	Name string `json:"name"`
+	// The IP protocol for this port. Supports "TCP", "UDP", and "SCTP".
+	//+kubebuilder:validation:Enum=TCP;UDP;SCTP
+	//+kubebuilder:default=TCP
+	Protocol corev1.Protocol `json:"protocol,omitempty"`
+	// The application protocol for this port.
+	// This is used as a hint for implementations to offer richer behavior for protocols that they understand.
+	// This field follows standard Kubernetes label syntax.
+	// Valid values are either:
+	//
+	// * Un-prefixed protocol names - reserved for IANA standard service names (as per
+	// RFC-6335 and https://www.iana.org/assignments/service-names).
+	AppProtocol *string `json:"appProtocol,omitempty"`
+	// The port that will be exposed by this service.
+	Port int32 `json:"port"`
+	// Number or name of the port to access on the pods of the Tenant Control Plane.
+	// Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.
+	// If this is a string, it will be looked up as a named port in the
+	// target Pod's container ports. If this is not specified, the value
+	// of the 'port' field is used (an identity map).
+	TargetPort intstr.IntOrString `json:"targetPort"`
+}
+
 // AdditionalMetadata defines which additional metadata, such as labels and annotations, must be attached to the created resource.
 type AdditionalMetadata struct {
 	Labels      map[string]string `json:"labels,omitempty"`
@@ -96,6 +133,7 @@ type AdditionalMetadata struct {
 
 // ControlPlane defines how the Tenant Control Plane Kubernetes resources must be created in the Admin Cluster,
 // such as the number of Pod replicas, the Service resource, or the Ingress.
+// +kubebuilder:validation:XValidation:rule="!(has(self.ingress) && has(self.gateway))",message="using both ingress and gateway is not supported"
 type ControlPlane struct {
 	// Defining the options for the deployed Tenant Control Plane as Deployment resource.
 	Deployment DeploymentSpec `json:"deployment,omitempty"`
@@ -103,6 +141,8 @@ type ControlPlane struct {
 	Service ServiceSpec `json:"service"`
 	// Defining the options for an Optional Ingress which will expose API Server of the Tenant Control Plane
 	Ingress *IngressSpec `json:"ingress,omitempty"`
+	// Defining the options for an Optional Gateway which will expose API Server of the Tenant Control Plane
+	Gateway *GatewaySpec `json:"gateway,omitempty"`
 }
 
 // IngressSpec defines the options for the ingress which will expose API Server of the Tenant Control Plane.
@@ -114,6 +154,16 @@ type IngressSpec struct {
 	Hostname string `json:"hostname,omitempty"`
 }
 
+// GatewaySpec defines the options for the Gateway which will expose API Server of the Tenant Control Plane.
+type GatewaySpec struct {
+	// AdditionalMetadata to add Labels and Annotations support.
+	AdditionalMetadata AdditionalMetadata `json:"additionalMetadata,omitempty"`
+	// GatewayParentRefs is the class of the Gateway resource to use.
+	GatewayParentRefs []gatewayv1.ParentReference `json:"parentRefs,omitempty"`
+	// Hostname is an optional field which will be used as a route hostname.
+	Hostname gatewayv1.Hostname `json:"hostname,omitempty"`
+}
+
 type ControlPlaneComponentsResources struct {
 	APIServer         *corev1.ResourceRequirements `json:"apiServer,omitempty"`
 	ControllerManager *corev1.ResourceRequirements `json:"controllerManager,omitempty"`
@@ -123,6 +173,54 @@ type ControlPlaneComponentsResources struct {
 	Kine *corev1.ResourceRequirements `json:"kine,omitempty"`
 }
 
+// ProbeSpec defines configurable parameters for a Kubernetes probe.
+type ProbeSpec struct {
+	// InitialDelaySeconds is the number of seconds after the container has started before the probe is initiated.
+	//+kubebuilder:validation:Minimum=0
+	InitialDelaySeconds *int32 `json:"initialDelaySeconds,omitempty"`
+	// TimeoutSeconds is the number of seconds after which the probe times out.
+	//+kubebuilder:validation:Minimum=1
+	TimeoutSeconds *int32 `json:"timeoutSeconds,omitempty"`
+	// PeriodSeconds is how often (in seconds) to perform the probe.
+	//+kubebuilder:validation:Minimum=1
+	PeriodSeconds *int32 `json:"periodSeconds,omitempty"`
+	// SuccessThreshold is the minimum consecutive successes for the probe to be considered successful.
+	// Must be 1 for liveness and startup probes.
+	//+kubebuilder:validation:Minimum=1
+	SuccessThreshold *int32 `json:"successThreshold,omitempty"`
+	// FailureThreshold is the consecutive failure count required to consider the probe failed.
+	//+kubebuilder:validation:Minimum=1
+	FailureThreshold *int32 `json:"failureThreshold,omitempty"`
+}
+
+// ProbeSet defines per-probe-type configuration.
+type ProbeSet struct {
+	// Liveness defines parameters for the liveness probe.
+	Liveness *ProbeSpec `json:"liveness,omitempty"`
+	// Readiness defines parameters for the readiness probe.
+	Readiness *ProbeSpec `json:"readiness,omitempty"`
+	// Startup defines parameters for the startup probe.
+	Startup *ProbeSpec `json:"startup,omitempty"`
+}
+
+// ControlPlaneProbes defines probe configuration for Control Plane components.
+// Global probe settings (Liveness, Readiness, Startup) apply to all components.
+// Per-component settings (APIServer, ControllerManager, Scheduler) override global settings.
+type ControlPlaneProbes struct {
+	// Liveness defines default parameters for liveness probes of all Control Plane components.
+	Liveness *ProbeSpec `json:"liveness,omitempty"`
+	// Readiness defines default parameters for the readiness probe of kube-apiserver.
+	Readiness *ProbeSpec `json:"readiness,omitempty"`
+	// Startup defines default parameters for startup probes of all Control Plane components.
+	Startup *ProbeSpec `json:"startup,omitempty"`
+	// APIServer defines probe overrides for kube-apiserver, taking precedence over global probe settings.
+	APIServer *ProbeSet `json:"apiServer,omitempty"`
+	// ControllerManager defines probe overrides for kube-controller-manager, taking precedence over global probe settings.
+	ControllerManager *ProbeSet `json:"controllerManager,omitempty"`
+	// Scheduler defines probe overrides for kube-scheduler, taking precedence over global probe settings.
+	Scheduler *ProbeSet `json:"scheduler,omitempty"`
+}
+
 type DeploymentSpec struct {
 	// RegistrySettings allows to override the default images for the given Tenant Control Plane instance.
 	// It could be used to point to a different container registry rather than the public one.
@@ -174,6 +272,10 @@ type DeploymentSpec struct {
 	// AdditionalVolumeMounts allows to mount an additional volume into each component of the Control Plane
 	// (kube-apiserver, controller-manager, and scheduler).
 	AdditionalVolumeMounts *AdditionalVolumeMounts `json:"additionalVolumeMounts,omitempty"`
+	// Probes defines the probe configuration for the Control Plane components
+	// (kube-apiserver, controller-manager, and scheduler).
+	// Override TimeoutSeconds, PeriodSeconds, and FailureThreshold for resource-constrained environments.
+	Probes *ControlPlaneProbes `json:"probes,omitempty"`
 	//+kubebuilder:default="default"
 	// ServiceAccountName allows to specify the service account to be mounted to the pods of the Control plane deployment
 	ServiceAccountName string `json:"serviceAccountName,omitempty"`
@@ -197,6 +299,9 @@ type ControlPlaneExtraArgs struct {
 
 type ServiceSpec struct {
 	AdditionalMetadata AdditionalMetadata `json:"additionalMetadata,omitempty"`
+	// AdditionalPorts allows adding additional ports to the Service generated Kamaji
+	// which targets the Tenant Control Plane pods.
+	AdditionalPorts []AdditionalPort `json:"additionalPorts,omitempty"`
 	// ServiceType allows specifying how to expose the Tenant Control Plane.
 	ServiceType ServiceType `json:"serviceType"`
 }
@@ -225,7 +330,9 @@ type KonnectivityServerSpec struct {
 	// The port which Konnectivity server is listening to.
 	Port int32 `json:"port"`
 	// Container image version of the Konnectivity server.
-	//+kubebuilder:default=v0.28.6
+	// If left empty, Kamaji will automatically inflect the version from the deployed Tenant Control Plane.
+	//
+	// WARNING: for last cut-off releases, the container image could be not available.
 	Version string `json:"version,omitempty"`
 	// Container image used by the Konnectivity server.
 	//+kubebuilder:default=registry.k8s.io/kas-network-proxy/proxy-server
@@ -235,25 +342,50 @@ type KonnectivityServerSpec struct {
 	ExtraArgs ExtraArgs                    `json:"extraArgs,omitempty"`
 }
 
+type KonnectivityAgentMode string
+
+var (
+	KonnectivityAgentModeDaemonSet  KonnectivityAgentMode = "DaemonSet"
+	KonnectivityAgentModeDeployment KonnectivityAgentMode = "Deployment"
+)
+
+//+kubebuilder:validation:XValidation:rule="!(self.mode == 'DaemonSet' && has(self.replicas) && self.replicas != 0) && !(self.mode == 'Deployment' && has(self.replicas) && self.replicas == 0)",message="replicas must be 0 (or unset) when mode is DaemonSet, and greater than 0 (or unset) when mode is Deployment"
+
 type KonnectivityAgentSpec struct {
 	// AgentImage defines the container image for Konnectivity's agent.
 	//+kubebuilder:default=registry.k8s.io/kas-network-proxy/proxy-agent
 	Image string `json:"image,omitempty"`
 	// Version for Konnectivity agent.
-	//+kubebuilder:default=v0.28.6
+	// If left empty, Kamaji will automatically inflect the version from the deployed Tenant Control Plane.
+	//
+	// WARNING: for last cut-off releases, the container image could be not available.
 	Version string `json:"version,omitempty"`
 	// Tolerations for the deployed agent.
 	// Can be customized to start the konnectivity-agent even if the nodes are not ready or tainted.
 	//+kubebuilder:default={{key: "CriticalAddonsOnly", operator: "Exists"}}
 	Tolerations []corev1.Toleration `json:"tolerations,omitempty"`
 	ExtraArgs   ExtraArgs           `json:"extraArgs,omitempty"`
+	// HostNetwork enables the konnectivity agent to use the Host network namespace.
+	// By enabling this mode, the Agent doesn't need to wait for the CNI initialisation,
+	// enabling a sort of out-of-band access to nodes for troubleshooting scenarios,
+	// or when the agent needs direct access to the host network.
+	//+kubebuilder:default=false
+	HostNetwork bool `json:"hostNetwork,omitempty"`
+	// Mode allows specifying the Agent deployment mode: Deployment, or DaemonSet (default).
+	//+kubebuilder:default="DaemonSet"
+	//+kubebuilder:validation:Enum=DaemonSet;Deployment
+	Mode KonnectivityAgentMode `json:"mode,omitempty"`
+	// Replicas defines the number of replicas when Mode is Deployment.
+	// Must be 0 if Mode is DaemonSet.
+	//+kubebuilder:validation:Optional
+	Replicas *int32 `json:"replicas,omitempty"`
 }
 
 // KonnectivitySpec defines the spec for Konnectivity.
 type KonnectivitySpec struct {
-	//+kubebuilder:default={version:"v0.28.6",image:"registry.k8s.io/kas-network-proxy/proxy-server",port:8132}
+	//+kubebuilder:default={image:"registry.k8s.io/kas-network-proxy/proxy-server",port:8132}
 	KonnectivityServerSpec KonnectivityServerSpec `json:"server,omitempty"`
-	//+kubebuilder:default={version:"v0.28.6",image:"registry.k8s.io/kas-network-proxy/proxy-agent"}
+	//+kubebuilder:default={image:"registry.k8s.io/kas-network-proxy/proxy-agent",mode:"DaemonSet"}
 	KonnectivityAgentSpec KonnectivityAgentSpec `json:"agent,omitempty"`
 }
 
@@ -269,14 +401,44 @@ type AddonsSpec struct {
 	KubeProxy *AddonSpec `json:"kubeProxy,omitempty"`
 }
 
+type Permissions struct {
+	BlockCreate bool `json:"blockCreation,omitempty"`
+	BlockUpdate bool `json:"blockUpdate,omitempty"`
+	BlockDelete bool `json:"blockDeletion,omitempty"`
+}
+
+func (p *Permissions) HasAnyLimitation() bool {
+	if p.BlockCreate || p.BlockUpdate || p.BlockDelete {
+		return true
+	}
+
+	return false
+}
+
+// DataStoreOverride defines which kubernetes resource will be stored in a dedicated datastore.
+type DataStoreOverride struct {
+	// Resource specifies which kubernetes resource to target.
+	Resource string `json:"resource,omitempty"`
+	// DataStore specifies the DataStore that should be used to store the Kubernetes data for the given Resource.
+	DataStore string `json:"dataStore,omitempty"`
+}
+
 // TenantControlPlaneSpec defines the desired state of TenantControlPlane.
 // +kubebuilder:validation:XValidation:rule="!has(oldSelf.dataStore) || has(self.dataStore)", message="unsetting the dataStore is not supported"
 // +kubebuilder:validation:XValidation:rule="!has(oldSelf.dataStoreSchema) || has(self.dataStoreSchema)", message="unsetting the dataStoreSchema is not supported"
+// +kubebuilder:validation:XValidation:rule="!has(oldSelf.dataStoreUsername) || has(self.dataStoreUsername)", message="unsetting the dataStoreUsername is not supported"
 // +kubebuilder:validation:XValidation:rule="!has(self.networkProfile.loadBalancerSourceRanges) || (size(self.networkProfile.loadBalancerSourceRanges) == 0 || self.controlPlane.service.serviceType == 'LoadBalancer')", message="LoadBalancer source ranges are supported only with LoadBalancer service type"
 // +kubebuilder:validation:XValidation:rule="!has(self.networkProfile.loadBalancerClass) || self.controlPlane.service.serviceType == 'LoadBalancer'", message="LoadBalancerClass is supported only with LoadBalancer service type"
 // +kubebuilder:validation:XValidation:rule="self.controlPlane.service.serviceType != 'LoadBalancer' || (oldSelf.controlPlane.service.serviceType != 'LoadBalancer' && self.controlPlane.service.serviceType == 'LoadBalancer') || has(self.networkProfile.loadBalancerClass) == has(oldSelf.networkProfile.loadBalancerClass)",message="LoadBalancerClass cannot be set or unset at runtime"
 
 type TenantControlPlaneSpec struct {
+	// WritePermissions allows to select which operations (create, delete, update) must be blocked:
+	// by default, all actions are allowed, and API Server can write to its Datastore.
+	//
+	// By blocking all actions, the Tenant Control Plane can enter in a Read Only mode:
+	// this phase can be used to prevent Datastore quota exhaustion or for your own business logic
+	// (e.g.: blocking creation and update, but allowing deletion to "clean up" space).
+	WritePermissions Permissions `json:"writePermissions,omitempty"`
 	// DataStore specifies the DataStore that should be used to store the Kubernetes data for the given Tenant Control Plane.
 	// When Kamaji runs with the default DataStore flag, all empty values will inherit the default value.
 	// By leaving it empty and running Kamaji with no default DataStore flag, it is possible to achieve automatic assignment to a specific DataStore object.
@@ -289,8 +451,16 @@ type TenantControlPlaneSpec struct {
 	// to the user to avoid clashes between different TenantControlPlanes. If not set upon creation, Kamaji will default the
 	// DataStoreSchema by concatenating the namespace and name of the TenantControlPlane.
 	// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="changing the dataStoreSchema is not supported"
-	DataStoreSchema string       `json:"dataStoreSchema,omitempty"`
-	ControlPlane    ControlPlane `json:"controlPlane"`
+	DataStoreSchema string `json:"dataStoreSchema,omitempty"`
+	// DataStoreUsername allows to specify the username of the database (for relational DataStores). This
+	// value is optional and immutable. Note that Kamaji currently doesn't ensure that DataStoreUsername values are unique. It's up
+	// to the user to avoid clashes between different TenantControlPlanes. If not set upon creation, Kamaji will default the
+	// DataStoreUsername by concatenating the namespace and name of the TenantControlPlane.
+	// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="changing the dataStoreUsername is not supported"
+	DataStoreUsername string `json:"dataStoreUsername,omitempty"`
+	// DataStoreOverride defines which kubernetes resources will be stored in dedicated datastores.
+	DataStoreOverrides []DataStoreOverride `json:"dataStoreOverrides,omitempty"`
+	ControlPlane       ControlPlane        `json:"controlPlane"`
 	// Kubernetes specification for tenant control plane
 	Kubernetes KubernetesSpec `json:"kubernetes"`
 	// NetworkProfile specifies how the network is
@@ -304,6 +474,7 @@ type TenantControlPlaneSpec struct {
 //+kubebuilder:subresource:scale:specpath=.spec.controlPlane.deployment.replicas,statuspath=.status.kubernetesResources.deployment.replicas,selectorpath=.status.kubernetesResources.deployment.selector
 //+kubebuilder:resource:categories=kamaji,shortName=tcp
 //+kubebuilder:printcolumn:name="Version",type="string",JSONPath=".spec.kubernetes.version",description="Kubernetes version"
+//+kubebuilder:printcolumn:name="Installed Version",type="string",JSONPath=".status.kubernetesResources.version.version",description="The actual installed Kubernetes version from status"
 //+kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.kubernetesResources.version.status",description="Status"
 //+kubebuilder:printcolumn:name="Control-Plane endpoint",type="string",JSONPath=".status.controlPlaneEndpoint",description="Tenant Control Plane Endpoint (API server)"
 //+kubebuilder:printcolumn:name="Kubeconfig",type="string",JSONPath=".status.kubeconfig.admin.secretName",description="Secret which contains admin kubeconfig"

api/v1alpha1/tenantcontrolplane_types_test.go

@@ -19,7 +19,7 @@ var _ = Describe("Cluster controller", func() {
 	)
 
 	BeforeEach(func() {
-		ctx = context.Background() //nolint:fatcontext
+		ctx = context.Background()
 		tcp = &TenantControlPlane{
 			ObjectMeta: metav1.ObjectMeta{
 				Name:      "tcp",

api/v1alpha1/validations_test.go

@@ -0,0 +1,177 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package v1alpha1
+
+import (
+	"context"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+var _ = Describe("Datastores validation test", func() {
+	var (
+		ctx context.Context
+		ds  *DataStore
+	)
+
+	BeforeEach(func() {
+		ctx = context.Background()
+		ds = &DataStore{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "ds",
+				Namespace: "default",
+			},
+			Spec: DataStoreSpec{},
+		}
+	})
+
+	AfterEach(func() {
+		if err := k8sClient.Delete(ctx, ds); err != nil && !apierrors.IsNotFound(err) {
+			Expect(err).NotTo(HaveOccurred())
+		}
+	})
+
+	Context("DataStores fields", func() {
+		It("datastores of type ETCD must have their TLS configurations set correctly", func() {
+			ds = &DataStore{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "bad-etcd",
+				},
+				Spec: DataStoreSpec{
+					Driver:    "etcd",
+					Endpoints: []string{"etcd-server:2379"},
+					TLSConfig: &TLSConfig{
+						CertificateAuthority: CertKeyPair{},
+						ClientCertificate:    &ClientCertificate{},
+					},
+				},
+			}
+
+			err := k8sClient.Create(ctx, ds)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("certificateAuthority privateKey must have secretReference or content when driver is etcd"))
+		})
+
+		It("valid ETCD DataStore should be created", func() {
+			var (
+				cert = []byte("cert")
+				key  = []byte("privkey")
+			)
+
+			ds = &DataStore{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "good-etcd",
+				},
+				Spec: DataStoreSpec{
+					Driver:    "etcd",
+					Endpoints: []string{"etcd-server:2379"},
+					TLSConfig: &TLSConfig{
+						CertificateAuthority: CertKeyPair{
+							Certificate: ContentRef{
+								Content: cert,
+							},
+							PrivateKey: &ContentRef{
+								Content: key,
+							},
+						},
+						ClientCertificate: &ClientCertificate{
+							Certificate: ContentRef{
+								Content: cert,
+							},
+							PrivateKey: ContentRef{
+								Content: key,
+							},
+						},
+					},
+				},
+			}
+
+			err := k8sClient.Create(ctx, ds)
+			Expect(err).To(Not(HaveOccurred()))
+		})
+
+		It("datastores of type PostgreSQL must have either basicAuth or tlsConfig", func() {
+			ds = &DataStore{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "bad-pg",
+				},
+				Spec: DataStoreSpec{
+					Driver:    "PostgreSQL",
+					Endpoints: []string{"pg-server:5432"},
+				},
+			}
+
+			err := k8sClient.Create(ctx, ds)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("When driver is not etcd, either tlsConfig or basicAuth must be provided"))
+		})
+
+		It("datastores of type PostgreSQL can have basicAuth", func() {
+			ds = &DataStore{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "good-pg",
+				},
+				Spec: DataStoreSpec{
+					Driver:    "PostgreSQL",
+					Endpoints: []string{"pg-server:5432"},
+					BasicAuth: &BasicAuth{
+						Username: ContentRef{
+							Content: []byte("postgres"),
+						},
+						Password: ContentRef{
+							Content: []byte("postgres"),
+						},
+					},
+				},
+			}
+
+			err := k8sClient.Create(ctx, ds)
+			Expect(err).To(Not(HaveOccurred()))
+		})
+
+		It("datastores of type PostgreSQL must have tlsConfig with proper content", func() {
+			ds = &DataStore{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "bad-pg",
+				},
+				Spec: DataStoreSpec{
+					Driver:    "PostgreSQL",
+					Endpoints: []string{"pg-server:5432"},
+					TLSConfig: &TLSConfig{
+						ClientCertificate: &ClientCertificate{},
+					},
+				},
+			}
+
+			err := k8sClient.Create(context.Background(), ds)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("When driver is not etcd and tlsConfig exists, clientCertificate must be null or contain valid content"))
+		})
+
+		It("datastores of type PostgreSQL need a proper clientCertificate", func() {
+			ds = &DataStore{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "good-pg",
+				},
+				Spec: DataStoreSpec{
+					Driver:    "PostgreSQL",
+					Endpoints: []string{"pg-server:5432"},
+					TLSConfig: &TLSConfig{
+						ClientCertificate: &ClientCertificate{
+							Certificate: ContentRef{
+								Content: []byte("cert"),
+							},
+						},
+					},
+				},
+			}
+
+			err := k8sClient.Create(context.Background(), ds)
+			Expect(err).ToNot(HaveOccurred())
+		})
+	})
+})

api/v1alpha1/zz_generated.deepcopy.go

@@ -8,8 +8,10 @@
 package v1alpha1
 
 import (
-	"k8s.io/api/core/v1"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	runtime "k8s.io/apimachinery/pkg/runtime"
+	apisv1 "sigs.k8s.io/gateway-api/apis/v1"
 )
 
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
@@ -57,26 +59,47 @@ func (in *AdditionalMetadata) DeepCopy() *AdditionalMetadata {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *AdditionalPort) DeepCopyInto(out *AdditionalPort) {
+	*out = *in
+	if in.AppProtocol != nil {
+		in, out := &in.AppProtocol, &out.AppProtocol
+		*out = new(string)
+		**out = **in
+	}
+	out.TargetPort = in.TargetPort
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AdditionalPort.
+func (in *AdditionalPort) DeepCopy() *AdditionalPort {
+	if in == nil {
+		return nil
+	}
+	out := new(AdditionalPort)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *AdditionalVolumeMounts) DeepCopyInto(out *AdditionalVolumeMounts) {
 	*out = *in
 	if in.APIServer != nil {
 		in, out := &in.APIServer, &out.APIServer
-		*out = make([]v1.VolumeMount, len(*in))
+		*out = make([]corev1.VolumeMount, len(*in))
 		for i := range *in {
 			(*in)[i].DeepCopyInto(&(*out)[i])
 		}
 	}
 	if in.ControllerManager != nil {
 		in, out := &in.ControllerManager, &out.ControllerManager
-		*out = make([]v1.VolumeMount, len(*in))
+		*out = make([]corev1.VolumeMount, len(*in))
 		for i := range *in {
 			(*in)[i].DeepCopyInto(&(*out)[i])
 		}
 	}
 	if in.Scheduler != nil {
 		in, out := &in.Scheduler, &out.Scheduler
-		*out = make([]v1.VolumeMount, len(*in))
+		*out = make([]corev1.VolumeMount, len(*in))
 		for i := range *in {
 			(*in)[i].DeepCopyInto(&(*out)[i])
 		}
@@ -289,6 +312,21 @@ func (in *ClientCertificate) DeepCopy() *ClientCertificate {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *CompoundValue) DeepCopyInto(out *CompoundValue) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CompoundValue.
+func (in *CompoundValue) DeepCopy() *CompoundValue {
+	if in == nil {
+		return nil
+	}
+	out := new(CompoundValue)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ContentRef) DeepCopyInto(out *ContentRef) {
 	*out = *in
@@ -324,6 +362,11 @@ func (in *ControlPlane) DeepCopyInto(out *ControlPlane) {
 		*out = new(IngressSpec)
 		(*in).DeepCopyInto(*out)
 	}
+	if in.Gateway != nil {
+		in, out := &in.Gateway, &out.Gateway
+		*out = new(GatewaySpec)
+		(*in).DeepCopyInto(*out)
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ControlPlane.
@@ -341,22 +384,22 @@ func (in *ControlPlaneComponentsResources) DeepCopyInto(out *ControlPlaneCompone
 	*out = *in
 	if in.APIServer != nil {
 		in, out := &in.APIServer, &out.APIServer
-		*out = new(v1.ResourceRequirements)
+		*out = new(corev1.ResourceRequirements)
 		(*in).DeepCopyInto(*out)
 	}
 	if in.ControllerManager != nil {
 		in, out := &in.ControllerManager, &out.ControllerManager
-		*out = new(v1.ResourceRequirements)
+		*out = new(corev1.ResourceRequirements)
 		(*in).DeepCopyInto(*out)
 	}
 	if in.Scheduler != nil {
 		in, out := &in.Scheduler, &out.Scheduler
-		*out = new(v1.ResourceRequirements)
+		*out = new(corev1.ResourceRequirements)
 		(*in).DeepCopyInto(*out)
 	}
 	if in.Kine != nil {
 		in, out := &in.Kine, &out.Kine
-		*out = new(v1.ResourceRequirements)
+		*out = new(corev1.ResourceRequirements)
 		(*in).DeepCopyInto(*out)
 	}
 }
@@ -406,6 +449,51 @@ func (in *ControlPlaneExtraArgs) DeepCopy() *ControlPlaneExtraArgs {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ControlPlaneProbes) DeepCopyInto(out *ControlPlaneProbes) {
+	*out = *in
+	if in.Liveness != nil {
+		in, out := &in.Liveness, &out.Liveness
+		*out = new(ProbeSpec)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.Readiness != nil {
+		in, out := &in.Readiness, &out.Readiness
+		*out = new(ProbeSpec)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.Startup != nil {
+		in, out := &in.Startup, &out.Startup
+		*out = new(ProbeSpec)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.APIServer != nil {
+		in, out := &in.APIServer, &out.APIServer
+		*out = new(ProbeSet)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.ControllerManager != nil {
+		in, out := &in.ControllerManager, &out.ControllerManager
+		*out = new(ProbeSet)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.Scheduler != nil {
+		in, out := &in.Scheduler, &out.Scheduler
+		*out = new(ProbeSet)
+		(*in).DeepCopyInto(*out)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ControlPlaneProbes.
+func (in *ControlPlaneProbes) DeepCopy() *ControlPlaneProbes {
+	if in == nil {
+		return nil
+	}
+	out := new(ControlPlaneProbes)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *DataStore) DeepCopyInto(out *DataStore) {
 	*out = *in
@@ -496,6 +584,21 @@ func (in *DataStoreList) DeepCopyObject() runtime.Object {
 	return nil
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *DataStoreOverride) DeepCopyInto(out *DataStoreOverride) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataStoreOverride.
+func (in *DataStoreOverride) DeepCopy() *DataStoreOverride {
+	if in == nil {
+		return nil
+	}
+	out := new(DataStoreOverride)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *DataStoreSetupStatus) DeepCopyInto(out *DataStoreSetupStatus) {
 	*out = *in
@@ -596,19 +699,19 @@ func (in *DeploymentSpec) DeepCopyInto(out *DeploymentSpec) {
 	in.Strategy.DeepCopyInto(&out.Strategy)
 	if in.Tolerations != nil {
 		in, out := &in.Tolerations, &out.Tolerations
-		*out = make([]v1.Toleration, len(*in))
+		*out = make([]corev1.Toleration, len(*in))
 		for i := range *in {
 			(*in)[i].DeepCopyInto(&(*out)[i])
 		}
 	}
 	if in.Affinity != nil {
 		in, out := &in.Affinity, &out.Affinity
-		*out = new(v1.Affinity)
+		*out = new(corev1.Affinity)
 		(*in).DeepCopyInto(*out)
 	}
 	if in.TopologySpreadConstraints != nil {
 		in, out := &in.TopologySpreadConstraints, &out.TopologySpreadConstraints
-		*out = make([]v1.TopologySpreadConstraint, len(*in))
+		*out = make([]corev1.TopologySpreadConstraint, len(*in))
 		for i := range *in {
 			(*in)[i].DeepCopyInto(&(*out)[i])
 		}
@@ -627,21 +730,21 @@ func (in *DeploymentSpec) DeepCopyInto(out *DeploymentSpec) {
 	in.PodAdditionalMetadata.DeepCopyInto(&out.PodAdditionalMetadata)
 	if in.AdditionalInitContainers != nil {
 		in, out := &in.AdditionalInitContainers, &out.AdditionalInitContainers
-		*out = make([]v1.Container, len(*in))
+		*out = make([]corev1.Container, len(*in))
 		for i := range *in {
 			(*in)[i].DeepCopyInto(&(*out)[i])
 		}
 	}
 	if in.AdditionalContainers != nil {
 		in, out := &in.AdditionalContainers, &out.AdditionalContainers
-		*out = make([]v1.Container, len(*in))
+		*out = make([]corev1.Container, len(*in))
 		for i := range *in {
 			(*in)[i].DeepCopyInto(&(*out)[i])
 		}
 	}
 	if in.AdditionalVolumes != nil {
 		in, out := &in.AdditionalVolumes, &out.AdditionalVolumes
-		*out = make([]v1.Volume, len(*in))
+		*out = make([]corev1.Volume, len(*in))
 		for i := range *in {
 			(*in)[i].DeepCopyInto(&(*out)[i])
 		}
@@ -651,6 +754,11 @@ func (in *DeploymentSpec) DeepCopyInto(out *DeploymentSpec) {
 		*out = new(AdditionalVolumeMounts)
 		(*in).DeepCopyInto(*out)
 	}
+	if in.Probes != nil {
+		in, out := &in.Probes, &out.Probes
+		*out = new(ControlPlaneProbes)
+		(*in).DeepCopyInto(*out)
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DeploymentSpec.
@@ -750,6 +858,69 @@ func (in ExtraArgs) DeepCopy() ExtraArgs {
 	return *out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *GatewayAccessPoint) DeepCopyInto(out *GatewayAccessPoint) {
+	*out = *in
+	if in.Type != nil {
+		in, out := &in.Type, &out.Type
+		*out = new(apisv1.AddressType)
+		**out = **in
+	}
+	if in.URLs != nil {
+		in, out := &in.URLs, &out.URLs
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GatewayAccessPoint.
+func (in *GatewayAccessPoint) DeepCopy() *GatewayAccessPoint {
+	if in == nil {
+		return nil
+	}
+	out := new(GatewayAccessPoint)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *GatewayListener) DeepCopyInto(out *GatewayListener) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GatewayListener.
+func (in *GatewayListener) DeepCopy() *GatewayListener {
+	if in == nil {
+		return nil
+	}
+	out := new(GatewayListener)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *GatewaySpec) DeepCopyInto(out *GatewaySpec) {
+	*out = *in
+	in.AdditionalMetadata.DeepCopyInto(&out.AdditionalMetadata)
+	if in.GatewayParentRefs != nil {
+		in, out := &in.GatewayParentRefs, &out.GatewayParentRefs
+		*out = make([]apisv1.ParentReference, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GatewaySpec.
+func (in *GatewaySpec) DeepCopy() *GatewaySpec {
+	if in == nil {
+		return nil
+	}
+	out := new(GatewaySpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ImageOverrideTrait) DeepCopyInto(out *ImageOverrideTrait) {
 	*out = *in
@@ -781,12 +952,53 @@ func (in *IngressSpec) DeepCopy() *IngressSpec {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *JSONPatch) DeepCopyInto(out *JSONPatch) {
+	*out = *in
+	if in.Value != nil {
+		in, out := &in.Value, &out.Value
+		*out = new(v1.JSON)
+		(*in).DeepCopyInto(*out)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new JSONPatch.
+func (in *JSONPatch) DeepCopy() *JSONPatch {
+	if in == nil {
+		return nil
+	}
+	out := new(JSONPatch)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in JSONPatches) DeepCopyInto(out *JSONPatches) {
+	{
+		in := &in
+		*out = make(JSONPatches, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new JSONPatches.
+func (in JSONPatches) DeepCopy() JSONPatches {
+	if in == nil {
+		return nil
+	}
+	out := new(JSONPatches)
+	in.DeepCopyInto(out)
+	return *out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *KonnectivityAgentSpec) DeepCopyInto(out *KonnectivityAgentSpec) {
 	*out = *in
 	if in.Tolerations != nil {
 		in, out := &in.Tolerations, &out.Tolerations
-		*out = make([]v1.Toleration, len(*in))
+		*out = make([]corev1.Toleration, len(*in))
 		for i := range *in {
 			(*in)[i].DeepCopyInto(&(*out)[i])
 		}
@@ -796,6 +1008,11 @@ func (in *KonnectivityAgentSpec) DeepCopyInto(out *KonnectivityAgentSpec) {
 		*out = make(ExtraArgs, len(*in))
 		copy(*out, *in)
 	}
+	if in.Replicas != nil {
+		in, out := &in.Replicas, &out.Replicas
+		*out = new(int32)
+		**out = **in
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KonnectivityAgentSpec.
@@ -808,6 +1025,22 @@ func (in *KonnectivityAgentSpec) DeepCopy() *KonnectivityAgentSpec {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *KonnectivityAgentStatus) DeepCopyInto(out *KonnectivityAgentStatus) {
+	*out = *in
+	in.ExternalKubernetesObjectStatus.DeepCopyInto(&out.ExternalKubernetesObjectStatus)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KonnectivityAgentStatus.
+func (in *KonnectivityAgentStatus) DeepCopy() *KonnectivityAgentStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(KonnectivityAgentStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *KonnectivityConfigMap) DeepCopyInto(out *KonnectivityConfigMap) {
 	*out = *in
@@ -828,7 +1061,7 @@ func (in *KonnectivityServerSpec) DeepCopyInto(out *KonnectivityServerSpec) {
 	*out = *in
 	if in.Resources != nil {
 		in, out := &in.Resources, &out.Resources
-		*out = new(v1.ResourceRequirements)
+		*out = new(corev1.ResourceRequirements)
 		(*in).DeepCopyInto(*out)
 	}
 	if in.ExtraArgs != nil {
@@ -875,6 +1108,11 @@ func (in *KonnectivityStatus) DeepCopyInto(out *KonnectivityStatus) {
 	in.ClusterRoleBinding.DeepCopyInto(&out.ClusterRoleBinding)
 	in.Agent.DeepCopyInto(&out.Agent)
 	in.Service.DeepCopyInto(&out.Service)
+	if in.Gateway != nil {
+		in, out := &in.Gateway, &out.Gateway
+		*out = new(KubernetesGatewayStatus)
+		(*in).DeepCopyInto(*out)
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KonnectivityStatus.
@@ -935,6 +1173,123 @@ func (in *KubeadmPhasesStatus) DeepCopy() *KubeadmPhasesStatus {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *KubeconfigGenerator) DeepCopyInto(out *KubeconfigGenerator) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
+	in.Spec.DeepCopyInto(&out.Spec)
+	in.Status.DeepCopyInto(&out.Status)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeconfigGenerator.
+func (in *KubeconfigGenerator) DeepCopy() *KubeconfigGenerator {
+	if in == nil {
+		return nil
+	}
+	out := new(KubeconfigGenerator)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *KubeconfigGenerator) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *KubeconfigGeneratorList) DeepCopyInto(out *KubeconfigGeneratorList) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ListMeta.DeepCopyInto(&out.ListMeta)
+	if in.Items != nil {
+		in, out := &in.Items, &out.Items
+		*out = make([]KubeconfigGenerator, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeconfigGeneratorList.
+func (in *KubeconfigGeneratorList) DeepCopy() *KubeconfigGeneratorList {
+	if in == nil {
+		return nil
+	}
+	out := new(KubeconfigGeneratorList)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *KubeconfigGeneratorList) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *KubeconfigGeneratorSpec) DeepCopyInto(out *KubeconfigGeneratorSpec) {
+	*out = *in
+	in.NamespaceSelector.DeepCopyInto(&out.NamespaceSelector)
+	in.TenantControlPlaneSelector.DeepCopyInto(&out.TenantControlPlaneSelector)
+	if in.Groups != nil {
+		in, out := &in.Groups, &out.Groups
+		*out = make([]CompoundValue, len(*in))
+		copy(*out, *in)
+	}
+	out.User = in.User
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeconfigGeneratorSpec.
+func (in *KubeconfigGeneratorSpec) DeepCopy() *KubeconfigGeneratorSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(KubeconfigGeneratorSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *KubeconfigGeneratorStatus) DeepCopyInto(out *KubeconfigGeneratorStatus) {
+	*out = *in
+	if in.Errors != nil {
+		in, out := &in.Errors, &out.Errors
+		*out = make([]KubeconfigGeneratorStatusError, len(*in))
+		copy(*out, *in)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeconfigGeneratorStatus.
+func (in *KubeconfigGeneratorStatus) DeepCopy() *KubeconfigGeneratorStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(KubeconfigGeneratorStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *KubeconfigGeneratorStatusError) DeepCopyInto(out *KubeconfigGeneratorStatusError) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeconfigGeneratorStatusError.
+func (in *KubeconfigGeneratorStatusError) DeepCopy() *KubeconfigGeneratorStatusError {
+	if in == nil {
+		return nil
+	}
+	out := new(KubeconfigGeneratorStatusError)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *KubeconfigStatus) DeepCopyInto(out *KubeconfigStatus) {
 	*out = *in
@@ -972,6 +1327,13 @@ func (in *KubeconfigsStatus) DeepCopy() *KubeconfigsStatus {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *KubeletSpec) DeepCopyInto(out *KubeletSpec) {
 	*out = *in
+	if in.ConfigurationJSONPatches != nil {
+		in, out := &in.ConfigurationJSONPatches, &out.ConfigurationJSONPatches
+		*out = make(JSONPatches, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
 	if in.PreferredAddressTypes != nil {
 		in, out := &in.PreferredAddressTypes, &out.PreferredAddressTypes
 		*out = make([]KubeletPreferredAddressType, len(*in))
@@ -1006,6 +1368,30 @@ func (in *KubernetesDeploymentStatus) DeepCopy() *KubernetesDeploymentStatus {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *KubernetesGatewayStatus) DeepCopyInto(out *KubernetesGatewayStatus) {
+	*out = *in
+	in.RouteStatus.DeepCopyInto(&out.RouteStatus)
+	out.RouteRef = in.RouteRef
+	if in.AccessPoints != nil {
+		in, out := &in.AccessPoints, &out.AccessPoints
+		*out = make([]GatewayAccessPoint, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubernetesGatewayStatus.
+func (in *KubernetesGatewayStatus) DeepCopy() *KubernetesGatewayStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(KubernetesGatewayStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *KubernetesIngressStatus) DeepCopyInto(out *KubernetesIngressStatus) {
 	*out = *in
@@ -1070,6 +1456,11 @@ func (in *KubernetesStatus) DeepCopyInto(out *KubernetesStatus) {
 		*out = new(KubernetesIngressStatus)
 		(*in).DeepCopyInto(*out)
 	}
+	if in.Gateway != nil {
+		in, out := &in.Gateway, &out.Gateway
+		*out = new(KubernetesGatewayStatus)
+		(*in).DeepCopyInto(*out)
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubernetesStatus.
@@ -1137,6 +1528,91 @@ func (in *NetworkProfileSpec) DeepCopy() *NetworkProfileSpec {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *Permissions) DeepCopyInto(out *Permissions) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Permissions.
+func (in *Permissions) DeepCopy() *Permissions {
+	if in == nil {
+		return nil
+	}
+	out := new(Permissions)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ProbeSet) DeepCopyInto(out *ProbeSet) {
+	*out = *in
+	if in.Liveness != nil {
+		in, out := &in.Liveness, &out.Liveness
+		*out = new(ProbeSpec)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.Readiness != nil {
+		in, out := &in.Readiness, &out.Readiness
+		*out = new(ProbeSpec)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.Startup != nil {
+		in, out := &in.Startup, &out.Startup
+		*out = new(ProbeSpec)
+		(*in).DeepCopyInto(*out)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProbeSet.
+func (in *ProbeSet) DeepCopy() *ProbeSet {
+	if in == nil {
+		return nil
+	}
+	out := new(ProbeSet)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ProbeSpec) DeepCopyInto(out *ProbeSpec) {
+	*out = *in
+	if in.InitialDelaySeconds != nil {
+		in, out := &in.InitialDelaySeconds, &out.InitialDelaySeconds
+		*out = new(int32)
+		**out = **in
+	}
+	if in.TimeoutSeconds != nil {
+		in, out := &in.TimeoutSeconds, &out.TimeoutSeconds
+		*out = new(int32)
+		**out = **in
+	}
+	if in.PeriodSeconds != nil {
+		in, out := &in.PeriodSeconds, &out.PeriodSeconds
+		*out = new(int32)
+		**out = **in
+	}
+	if in.SuccessThreshold != nil {
+		in, out := &in.SuccessThreshold, &out.SuccessThreshold
+		*out = new(int32)
+		**out = **in
+	}
+	if in.FailureThreshold != nil {
+		in, out := &in.FailureThreshold, &out.FailureThreshold
+		*out = new(int32)
+		**out = **in
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProbeSpec.
+func (in *ProbeSpec) DeepCopy() *ProbeSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(ProbeSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *PublicKeyPrivateKeyPairStatus) DeepCopyInto(out *PublicKeyPrivateKeyPairStatus) {
 	*out = *in
@@ -1188,6 +1664,13 @@ func (in *SecretReference) DeepCopy() *SecretReference {
 func (in *ServiceSpec) DeepCopyInto(out *ServiceSpec) {
 	*out = *in
 	in.AdditionalMetadata.DeepCopyInto(&out.AdditionalMetadata)
+	if in.AdditionalPorts != nil {
+		in, out := &in.AdditionalPorts, &out.AdditionalPorts
+		*out = make([]AdditionalPort, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ServiceSpec.
@@ -1301,6 +1784,12 @@ func (in *TenantControlPlaneList) DeepCopyObject() runtime.Object {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *TenantControlPlaneSpec) DeepCopyInto(out *TenantControlPlaneSpec) {
 	*out = *in
+	out.WritePermissions = in.WritePermissions
+	if in.DataStoreOverrides != nil {
+		in, out := &in.DataStoreOverrides, &out.DataStoreOverrides
+		*out = make([]DataStoreOverride, len(*in))
+		copy(*out, *in)
+	}
 	in.ControlPlane.DeepCopyInto(&out.ControlPlane)
 	in.Kubernetes.DeepCopyInto(&out.Kubernetes)
 	in.NetworkProfile.DeepCopyInto(&out.NetworkProfile)

charts/kamaji-crds/.helmignore

@@ -0,0 +1,28 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/
+# Helm source files
+README.md.gotmpl
+.helmignore
+# Build tools
+Makefile

charts/kamaji-crds/Chart.yaml

@@ -0,0 +1,39 @@
+apiVersion: v2
+appVersion: latest
+description: Kamaji is the Hosted Control Plane Manager for Kubernetes.
+home: https://github.com/clastix/kamaji
+icon: https://github.com/clastix/kamaji/raw/master/assets/logo-colored.png
+maintainers:
+  - email: dario@tranchitella.eu
+    name: Dario Tranchitella
+    url: https://clastix.io
+  - email: me@bsctl.io
+    name: Adriano Pezzuto
+    url: https://clastix.io
+name: kamaji-crds
+sources:
+  - https://github.com/clastix/kamaji
+type: application
+version: 0.0.0+latest
+annotations:
+  artifacthub.io/crds: |
+    - kind: TenantControlPlane
+      version: v1alpha1
+      name: tenantcontrolplanes.kamaji.clastix.io
+      displayName: TenantControlPlane
+      description: TenantControlPlane defines the desired state for a Control Plane backed by Kamaji.
+    - kind: DataStore
+      version: v1alpha1
+      name: datastores.kamaji.clastix.io
+      displayName: DataStore
+      description: DataStores is holding all the required details to communicate with a Datastore, such as etcd, MySQL, PostgreSQL, and NATS.
+  artifacthub.io/links: |
+    - name: CLASTIX
+      url: https://clastix.io
+    - name: support
+      url: https://clastix.io/support
+  artifacthub.io/changes: |
+    - kind: changed
+      description: Upgrading support to Kubernetes v1.35
+    - kind: added
+      description: Supporting multiple Datastore via etcd overrides

charts/kamaji-crds/Makefile

@@ -0,0 +1,9 @@
+docs: HELMDOCS_VERSION := v1.8.1
+docs: docker
+	@docker run --rm -v "$$(pwd):/helm-docs" -u $$(id -u) jnorwood/helm-docs:$(HELMDOCS_VERSION)
+
+docker:
+	@hash docker 2>/dev/null || {\
+		echo "You need docker" &&\
+		exit 1;\
+	}

charts/kamaji-crds/NOTES.txt

@@ -0,0 +1,2 @@
+Kamaji Custom Resource Definitions have been installed properly:
+you can proceed to upgrade your Kamaji operator instance.

charts/kamaji-crds/README.md

@@ -0,0 +1,66 @@
+# kamaji-crds
+
+![Version: 0.0.0+latest](https://img.shields.io/badge/Version-0.0.0+latest-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: latest](https://img.shields.io/badge/AppVersion-latest-informational?style=flat-square)
+
+Kamaji is the Hosted Control Plane Manager for Kubernetes.
+
+## Maintainers
+
+| Name | Email | Url |
+| ---- | ------ | --- |
+| Dario Tranchitella | <dario@tranchitella.eu> | <https://clastix.io> |
+| Adriano Pezzuto | <me@bsctl.io> | <https://clastix.io> |
+
+## Source Code
+
+* <https://github.com/clastix/kamaji>
+
+[Kamaji](https://github.com/clastix/kamaji) Custom Resource Definitions packaged as Helm Charts.
+
+## How to use this chart
+
+Add `clastix` Helm repository:
+
+    helm repo add clastix https://clastix.github.io/charts
+
+Install the Chart with the release name `kamaji-crds`:
+
+    helm upgrade --install --namespace kamaji-system --create-namespace kamaji-crds clastix/kamaji-crds
+
+Show the status:
+
+    helm status kamaji-crds -n kamaji-system
+
+Upgrade the Chart
+
+    helm upgrade kamaji-crds -n kamaji-system clastix/kamaji-crds
+
+Uninstall the Chart
+
+    helm uninstall kamaji-crds -n kamaji-system
+
+## Customize the installation
+
+There are two methods for specifying overrides of values during Chart installation: `--values` and `--set`.
+
+The `--values` option is the preferred method because it allows you to keep your overrides in a YAML file, rather than specifying them all on the command line. Create a copy of the YAML file `values.yaml` and add your overrides to it.
+
+Specify your overrides file when you install the Chart:
+
+    helm upgrade kamaji-crds --install --namespace kamaji-system --create-namespace clastix/kamaji-crds --values myvalues.yaml
+
+The values in your overrides file `myvalues.yaml` will override their counterparts in the Chart's values.yaml file. Any values in `values.yaml` that weren’t overridden will keep their defaults.
+
+If you only need to make minor customizations, you can specify them on the command line by using the `--set` option. For example:
+
+    helm upgrade kamaji-crds --install --namespace kamaji-system --create-namespace clastix/kamaji-crds --set kamajiCertificateName=kamaji
+
+## Values
+
+| Key | Type | Default | Description |
+|-----|------|---------|-------------|
+| fullnameOverride | string | `""` | Overrides the full name of the resources created by the chart. |
+| kamajiCertificateName | string | `"kamaji-serving-cert"` | The cert-manager Certificate resource name, holding the Certificate Authority for webhooks. |
+| kamajiNamespace | string | `"kamaji-system"` | The namespace where Kamaji has been installed: required to inject the Certificate Authority for cert-manager. |
+| kamajiService | string | `"kamaji-webhook-service"` | The Kamaji webhook Service name. |
+| nameOverride | string | `""` | Overrides the name of the chart for resource naming purposes. |

charts/kamaji-crds/README.md.gotmpl

@@ -0,0 +1,54 @@
+{{ template "chart.header" . }}
+{{ template "chart.deprecationWarning" . }}
+
+{{ template "chart.badgesSection" . }}
+
+{{ template "chart.description" . }}
+
+{{ template "chart.maintainersSection" . }}
+
+{{ template "chart.sourcesSection" . }}
+
+{{ template "chart.requirementsSection" . }}
+
+[Kamaji](https://github.com/clastix/kamaji) Custom Resource Definitions packaged as Helm Charts.
+
+## How to use this chart
+
+Add `clastix` Helm repository:
+
+    helm repo add clastix https://clastix.github.io/charts
+
+Install the Chart with the release name `kamaji-crds`:
+
+    helm upgrade --install --namespace kamaji-system --create-namespace kamaji-crds clastix/kamaji-crds
+
+Show the status:
+
+    helm status kamaji-crds -n kamaji-system
+
+Upgrade the Chart
+
+    helm upgrade kamaji-crds -n kamaji-system clastix/kamaji-crds
+
+Uninstall the Chart
+
+    helm uninstall kamaji-crds -n kamaji-system
+
+## Customize the installation
+
+There are two methods for specifying overrides of values during Chart installation: `--values` and `--set`.
+
+The `--values` option is the preferred method because it allows you to keep your overrides in a YAML file, rather than specifying them all on the command line. Create a copy of the YAML file `values.yaml` and add your overrides to it.
+
+Specify your overrides file when you install the Chart:
+
+    helm upgrade kamaji-crds --install --namespace kamaji-system --create-namespace clastix/kamaji-crds --values myvalues.yaml
+
+The values in your overrides file `myvalues.yaml` will override their counterparts in the Chart's values.yaml file. Any values in `values.yaml` that weren’t overridden will keep their defaults.
+
+If you only need to make minor customizations, you can specify them on the command line by using the `--set` option. For example:
+
+    helm upgrade kamaji-crds --install --namespace kamaji-system --create-namespace clastix/kamaji-crds --set kamajiCertificateName=kamaji
+
+{{ template "chart.valuesSection" . }}

charts/kamaji-crds/hack/crd-conversion.yaml

@@ -0,0 +1,11 @@
+spec:
+  conversion:
+    strategy: Webhook
+    webhook:
+      clientConfig:
+        service:
+          name: kamaji-webhook-service
+          namespace: kamaji-system
+          path: /convert
+      conversionReviewVersions:
+        - v1

charts/kamaji-crds/hack/kamaji.clastix.io_datastores_spec.yaml

@@ -0,0 +1,361 @@
+group: kamaji.clastix.io
+names:
+  kind: DataStore
+  listKind: DataStoreList
+  plural: datastores
+  singular: datastore
+scope: Cluster
+versions:
+  - additionalPrinterColumns:
+      - description: Kamaji data store driver
+        jsonPath: .spec.driver
+        name: Driver
+        type: string
+      - description: DataStore validated and ready for use
+        jsonPath: .status.ready
+        name: Ready
+        type: boolean
+      - description: Age
+        jsonPath: .metadata.creationTimestamp
+        name: Age
+        type: date
+    name: v1alpha1
+    schema:
+      openAPIV3Schema:
+        description: DataStore is the Schema for the datastores API.
+        properties:
+          apiVersion:
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            type: string
+          kind:
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: DataStoreSpec defines the desired state of DataStore.
+            properties:
+              basicAuth:
+                description: |-
+                  In case of authentication enabled for the given data store, specifies the username and password pair.
+                  This value is optional.
+                properties:
+                  password:
+                    properties:
+                      content:
+                        description: |-
+                          Bare content of the file, base64 encoded.
+                          It has precedence over the SecretReference value.
+                        format: byte
+                        type: string
+                      secretReference:
+                        properties:
+                          keyPath:
+                            description: |-
+                              Name of the key for the given Secret reference where the content is stored.
+                              This value is mandatory.
+                            minLength: 1
+                            type: string
+                          name:
+                            description: name is unique within a namespace to reference a secret resource.
+                            type: string
+                          namespace:
+                            description: namespace defines the space within which the secret name must be unique.
+                            type: string
+                        required:
+                          - keyPath
+                        type: object
+                        x-kubernetes-map-type: atomic
+                    type: object
+                  username:
+                    properties:
+                      content:
+                        description: |-
+                          Bare content of the file, base64 encoded.
+                          It has precedence over the SecretReference value.
+                        format: byte
+                        type: string
+                      secretReference:
+                        properties:
+                          keyPath:
+                            description: |-
+                              Name of the key for the given Secret reference where the content is stored.
+                              This value is mandatory.
+                            minLength: 1
+                            type: string
+                          name:
+                            description: name is unique within a namespace to reference a secret resource.
+                            type: string
+                          namespace:
+                            description: namespace defines the space within which the secret name must be unique.
+                            type: string
+                        required:
+                          - keyPath
+                        type: object
+                        x-kubernetes-map-type: atomic
+                    type: object
+                required:
+                  - password
+                  - username
+                type: object
+              driver:
+                description: The driver to use to connect to the shared datastore.
+                enum:
+                  - etcd
+                  - MySQL
+                  - PostgreSQL
+                  - NATS
+                type: string
+                x-kubernetes-validations:
+                  - message: Datastore driver is immutable
+                    rule: self == oldSelf
+              endpoints:
+                description: |-
+                  List of the endpoints to connect to the shared datastore.
+                  No need for protocol, just bare IP/FQDN and port.
+                items:
+                  type: string
+                minItems: 1
+                type: array
+              tlsConfig:
+                description: |-
+                  Defines the TLS/SSL configuration required to connect to the data store in a secure way.
+                  This value is optional.
+                properties:
+                  certificateAuthority:
+                    description: |-
+                      Retrieve the Certificate Authority certificate and private key, such as bare content of the file, or a SecretReference.
+                      The key reference is required since etcd authentication is based on certificates, and Kamaji is responsible in creating this.
+                    properties:
+                      certificate:
+                        properties:
+                          content:
+                            description: |-
+                              Bare content of the file, base64 encoded.
+                              It has precedence over the SecretReference value.
+                            format: byte
+                            type: string
+                          secretReference:
+                            properties:
+                              keyPath:
+                                description: |-
+                                  Name of the key for the given Secret reference where the content is stored.
+                                  This value is mandatory.
+                                minLength: 1
+                                type: string
+                              name:
+                                description: name is unique within a namespace to reference a secret resource.
+                                type: string
+                              namespace:
+                                description: namespace defines the space within which the secret name must be unique.
+                                type: string
+                            required:
+                              - keyPath
+                            type: object
+                            x-kubernetes-map-type: atomic
+                        type: object
+                      privateKey:
+                        properties:
+                          content:
+                            description: |-
+                              Bare content of the file, base64 encoded.
+                              It has precedence over the SecretReference value.
+                            format: byte
+                            type: string
+                          secretReference:
+                            properties:
+                              keyPath:
+                                description: |-
+                                  Name of the key for the given Secret reference where the content is stored.
+                                  This value is mandatory.
+                                minLength: 1
+                                type: string
+                              name:
+                                description: name is unique within a namespace to reference a secret resource.
+                                type: string
+                              namespace:
+                                description: namespace defines the space within which the secret name must be unique.
+                                type: string
+                            required:
+                              - keyPath
+                            type: object
+                            x-kubernetes-map-type: atomic
+                        type: object
+                    required:
+                      - certificate
+                    type: object
+                  clientCertificate:
+                    description: Specifies the SSL/TLS key and private key pair used to connect to the data store.
+                    properties:
+                      certificate:
+                        properties:
+                          content:
+                            description: |-
+                              Bare content of the file, base64 encoded.
+                              It has precedence over the SecretReference value.
+                            format: byte
+                            type: string
+                          secretReference:
+                            properties:
+                              keyPath:
+                                description: |-
+                                  Name of the key for the given Secret reference where the content is stored.
+                                  This value is mandatory.
+                                minLength: 1
+                                type: string
+                              name:
+                                description: name is unique within a namespace to reference a secret resource.
+                                type: string
+                              namespace:
+                                description: namespace defines the space within which the secret name must be unique.
+                                type: string
+                            required:
+                              - keyPath
+                            type: object
+                            x-kubernetes-map-type: atomic
+                        type: object
+                      privateKey:
+                        properties:
+                          content:
+                            description: |-
+                              Bare content of the file, base64 encoded.
+                              It has precedence over the SecretReference value.
+                            format: byte
+                            type: string
+                          secretReference:
+                            properties:
+                              keyPath:
+                                description: |-
+                                  Name of the key for the given Secret reference where the content is stored.
+                                  This value is mandatory.
+                                minLength: 1
+                                type: string
+                              name:
+                                description: name is unique within a namespace to reference a secret resource.
+                                type: string
+                              namespace:
+                                description: namespace defines the space within which the secret name must be unique.
+                                type: string
+                            required:
+                              - keyPath
+                            type: object
+                            x-kubernetes-map-type: atomic
+                        type: object
+                    required:
+                      - certificate
+                      - privateKey
+                    type: object
+                required:
+                  - certificateAuthority
+                type: object
+            required:
+              - driver
+              - endpoints
+            type: object
+            x-kubernetes-validations:
+              - message: certificateAuthority privateKey must have secretReference or content when driver is etcd
+                rule: '(self.driver == "etcd") ? (self.tlsConfig != null && (has(self.tlsConfig.certificateAuthority.privateKey.secretReference) || has(self.tlsConfig.certificateAuthority.privateKey.content))) : true'
+              - message: clientCertificate must have secretReference or content when driver is etcd
+                rule: '(self.driver == "etcd") ? (self.tlsConfig != null && (has(self.tlsConfig.clientCertificate.certificate.secretReference) || has(self.tlsConfig.clientCertificate.certificate.content))) : true'
+              - message: clientCertificate privateKey must have secretReference or content when driver is etcd
+                rule: '(self.driver == "etcd") ? (self.tlsConfig != null && (has(self.tlsConfig.clientCertificate.privateKey.secretReference) || has(self.tlsConfig.clientCertificate.privateKey.content))) : true'
+              - message: When driver is not etcd and tlsConfig exists, clientCertificate must be null or contain valid content
+                rule: '(self.driver != "etcd" && has(self.tlsConfig) && has(self.tlsConfig.clientCertificate)) ? (((has(self.tlsConfig.clientCertificate.certificate.secretReference) || has(self.tlsConfig.clientCertificate.certificate.content)))) : true'
+              - message: When driver is not etcd and basicAuth exists, username must have secretReference or content
+                rule: '(self.driver != "etcd" && has(self.basicAuth)) ? ((has(self.basicAuth.username.secretReference) || has(self.basicAuth.username.content))) : true'
+              - message: When driver is not etcd and basicAuth exists, password must have secretReference or content
+                rule: '(self.driver != "etcd" && has(self.basicAuth)) ? ((has(self.basicAuth.password.secretReference) || has(self.basicAuth.password.content))) : true'
+              - message: When driver is not etcd, either tlsConfig or basicAuth must be provided
+                rule: '(self.driver != "etcd") ? (has(self.tlsConfig) || has(self.basicAuth)) : true'
+              - message: driver is immutable and cannot be changed after creation
+                rule: oldSelf == null || self.driver == oldSelf.driver
+          status:
+            description: DataStoreStatus defines the observed state of DataStore.
+            properties:
+              conditions:
+                description: Conditions contains the validation conditions for the given Datastore.
+                items:
+                  description: Condition contains details for one aspect of the current state of this API Resource.
+                  properties:
+                    lastTransitionTime:
+                      description: |-
+                        lastTransitionTime is the last time the condition transitioned from one status to another.
+                        This should be when the underlying condition changed.  If that is not known, then using the time when the API field changed is acceptable.
+                      format: date-time
+                      type: string
+                    message:
+                      description: |-
+                        message is a human readable message indicating details about the transition.
+                        This may be an empty string.
+                      maxLength: 32768
+                      type: string
+                    observedGeneration:
+                      description: |-
+                        observedGeneration represents the .metadata.generation that the condition was set based upon.
+                        For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date
+                        with respect to the current state of the instance.
+                      format: int64
+                      minimum: 0
+                      type: integer
+                    reason:
+                      description: |-
+                        reason contains a programmatic identifier indicating the reason for the condition's last transition.
+                        Producers of specific condition types may define expected values and meanings for this field,
+                        and whether the values are considered a guaranteed API.
+                        The value should be a CamelCase string.
+                        This field may not be empty.
+                      maxLength: 1024
+                      minLength: 1
+                      pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
+                      type: string
+                    status:
+                      description: status of the condition, one of True, False, Unknown.
+                      enum:
+                        - "True"
+                        - "False"
+                        - Unknown
+                      type: string
+                    type:
+                      description: type of condition in CamelCase or in foo.example.com/CamelCase.
+                      maxLength: 316
+                      pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
+                      type: string
+                  required:
+                    - lastTransitionTime
+                    - message
+                    - reason
+                    - status
+                    - type
+                  type: object
+                type: array
+              observedGeneration:
+                description: ObservedGeneration represents the .metadata.generation that was last reconciled.
+                format: int64
+                type: integer
+              ready:
+                description: |-
+                  Ready returns if the DataStore is accepted and ready to get used:
+                  Kamaji will ensure certificates are available, or correctly referenced.
+                type: boolean
+              usedBy:
+                description: List of the Tenant Control Planes, namespaced named, using this data store.
+                items:
+                  type: string
+                type: array
+            required:
+              - ready
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}

charts/kamaji-crds/hack/kamaji.clastix.io_kubeconfiggenerators_spec.yaml

@@ -0,0 +1,218 @@
+group: kamaji.clastix.io
+names:
+  categories:
+    - kamaji
+  kind: KubeconfigGenerator
+  listKind: KubeconfigGeneratorList
+  plural: kubeconfiggenerators
+  shortNames:
+    - kc
+  singular: kubeconfiggenerator
+scope: Cluster
+versions:
+  - additionalPrinterColumns:
+      - description: Age
+        jsonPath: .metadata.creationTimestamp
+        name: Age
+        type: date
+    name: v1alpha1
+    schema:
+      openAPIV3Schema:
+        description: KubeconfigGenerator is the Schema for the kubeconfiggenerators API.
+        properties:
+          apiVersion:
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            type: string
+          kind:
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            type: string
+          metadata:
+            type: object
+          spec:
+            properties:
+              controlPlaneEndpointFrom:
+                default: admin.svc
+                description: |-
+                  ControlPlaneEndpointFrom is the key used to extract the Tenant Control Plane endpoint that must be used by the generator.
+                  The targeted Secret is the `${TCP}-admin-kubeconfig` one, default to `admin.svc`.
+                type: string
+              groups:
+                description: |-
+                  Groups is resolved a set of strings used to assign the x509 organisations field.
+                  It will be recognised by Kubernetes as user groups.
+                items:
+                  description: |-
+                    CompoundValue allows defining a static, or a dynamic value.
+                    Options are mutually exclusive, just one should be picked up.
+                  properties:
+                    fromDefinition:
+                      description: |-
+                        FromDefinition is used to generate a dynamic value,
+                        it uses the dot notation to access fields from the referenced TenantControlPlane object:
+                        e.g.: metadata.name
+                      type: string
+                    stringValue:
+                      description: StringValue is a static string value.
+                      type: string
+                  type: object
+                  x-kubernetes-validations:
+                    - message: Either stringValue or fromDefinition must be set, but not both.
+                      rule: (has(self.stringValue) || has(self.fromDefinition)) && !(has(self.stringValue) && has(self.fromDefinition))
+                type: array
+              namespaceSelector:
+                description: NamespaceSelector is used to filter Namespaces from which the generator should extract TenantControlPlane objects.
+                properties:
+                  matchExpressions:
+                    description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
+                    items:
+                      description: |-
+                        A label selector requirement is a selector that contains values, a key, and an operator that
+                        relates the key and values.
+                      properties:
+                        key:
+                          description: key is the label key that the selector applies to.
+                          type: string
+                        operator:
+                          description: |-
+                            operator represents a key's relationship to a set of values.
+                            Valid operators are In, NotIn, Exists and DoesNotExist.
+                          type: string
+                        values:
+                          description: |-
+                            values is an array of string values. If the operator is In or NotIn,
+                            the values array must be non-empty. If the operator is Exists or DoesNotExist,
+                            the values array must be empty. This array is replaced during a strategic
+                            merge patch.
+                          items:
+                            type: string
+                          type: array
+                          x-kubernetes-list-type: atomic
+                      required:
+                        - key
+                        - operator
+                      type: object
+                    type: array
+                    x-kubernetes-list-type: atomic
+                  matchLabels:
+                    additionalProperties:
+                      type: string
+                    description: |-
+                      matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels
+                      map is equivalent to an element of matchExpressions, whose key field is "key", the
+                      operator is "In", and the values array contains only "value". The requirements are ANDed.
+                    type: object
+                type: object
+                x-kubernetes-map-type: atomic
+              tenantControlPlaneSelector:
+                description: TenantControlPlaneSelector is used to filter the TenantControlPlane objects that should be address by the generator.
+                properties:
+                  matchExpressions:
+                    description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
+                    items:
+                      description: |-
+                        A label selector requirement is a selector that contains values, a key, and an operator that
+                        relates the key and values.
+                      properties:
+                        key:
+                          description: key is the label key that the selector applies to.
+                          type: string
+                        operator:
+                          description: |-
+                            operator represents a key's relationship to a set of values.
+                            Valid operators are In, NotIn, Exists and DoesNotExist.
+                          type: string
+                        values:
+                          description: |-
+                            values is an array of string values. If the operator is In or NotIn,
+                            the values array must be non-empty. If the operator is Exists or DoesNotExist,
+                            the values array must be empty. This array is replaced during a strategic
+                            merge patch.
+                          items:
+                            type: string
+                          type: array
+                          x-kubernetes-list-type: atomic
+                      required:
+                        - key
+                        - operator
+                      type: object
+                    type: array
+                    x-kubernetes-list-type: atomic
+                  matchLabels:
+                    additionalProperties:
+                      type: string
+                    description: |-
+                      matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels
+                      map is equivalent to an element of matchExpressions, whose key field is "key", the
+                      operator is "In", and the values array contains only "value". The requirements are ANDed.
+                    type: object
+                type: object
+                x-kubernetes-map-type: atomic
+              user:
+                description: User resolves to a string to identify the client, assigned to the x509 Common Name field.
+                properties:
+                  fromDefinition:
+                    description: |-
+                      FromDefinition is used to generate a dynamic value,
+                      it uses the dot notation to access fields from the referenced TenantControlPlane object:
+                      e.g.: metadata.name
+                    type: string
+                  stringValue:
+                    description: StringValue is a static string value.
+                    type: string
+                type: object
+                x-kubernetes-validations:
+                  - message: Either stringValue or fromDefinition must be set, but not both.
+                    rule: (has(self.stringValue) || has(self.fromDefinition)) && !(has(self.stringValue) && has(self.fromDefinition))
+            required:
+              - user
+            type: object
+          status:
+            description: KubeconfigGeneratorStatus defines the observed state of KubeconfigGenerator.
+            properties:
+              availableResources:
+                default: 0
+                description: |-
+                  AvailableResources is the sum of successfully generated resources.
+                  In case of a different value compared to Resources, check the field errors.
+                type: integer
+              errors:
+                description: Errors is the list of failed kubeconfig generations.
+                items:
+                  properties:
+                    message:
+                      description: Message is the error message recorded upon the last generator run.
+                      type: string
+                    resource:
+                      description: Resource is the Namespaced name of the errored resource.
+                      type: string
+                  required:
+                    - message
+                    - resource
+                  type: object
+                type: array
+              observedGeneration:
+                description: ObservedGeneration represents the .metadata.generation that was last reconciled.
+                format: int64
+                type: integer
+              resources:
+                default: 0
+                description: Resources is the sum of targeted TenantControlPlane objects.
+                type: integer
+            required:
+              - availableResources
+              - resources
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}

charts/kamaji-crds/templates/_helpers.tpl

@@ -0,0 +1,49 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "kamaji-crds.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "kamaji.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "kamaji-crds.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Create the cert-manager annotation to inject Certificate CA.
+*/}}
+{{- define "kamaji-crds.certManagerAnnotation" -}}
+{{- printf "%s/%s" (required "A valid .Values.kamajiNamespace is required" .Values.kamajiNamespace) (required "A valid .Values.kamajiCertificateName is required" .Values.kamajiCertificateName) }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "kamaji-crds.labels" -}}
+helm.sh/chart: {{ include "kamaji-crds.chart" . }}
+app.kubernetes.io/name: {{ include "kamaji-crds.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+app.kubernetes.io/component: "crds"
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}

charts/kamaji-crds/templates/kamaji.clastix.io_datastores.yaml

@@ -0,0 +1,10 @@
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    cert-manager.io/inject-ca-from: {{ include "kamaji-crds.certManagerAnnotation" . }}
+  labels:
+    {{- include "kamaji-crds.labels" . | nindent 4 }}
+  name: datastores.kamaji.clastix.io
+spec:
+  {{ tpl (.Files.Get "hack/kamaji.clastix.io_datastores_spec.yaml") . | nindent 2}}

charts/kamaji-crds/templates/kamaji.clastix.io_kubeconfiggenerators.yaml

@@ -0,0 +1,10 @@
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    cert-manager.io/inject-ca-from: {{ include "kamaji-crds.certManagerAnnotation" . }}
+  labels:
+    {{- include "kamaji-crds.labels" . | nindent 4 }}
+  name: kubeconfiggenerators.kamaji.clastix.io
+spec:
+  {{ tpl (.Files.Get "hack/kamaji.clastix.io_kubeconfiggenerators_spec.yaml") . | nindent 2 }}

charts/kamaji-crds/templates/kamaji.clastix.io_tenantcontrolplanes.yaml

@@ -0,0 +1,10 @@
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    cert-manager.io/inject-ca-from: {{ include "kamaji-crds.certManagerAnnotation" . }}
+  labels:
+    {{- include "kamaji-crds.labels" . | nindent 4 }}
+  name: tenantcontrolplanes.kamaji.clastix.io
+spec:
+  {{ tpl (.Files.Get "hack/kamaji.clastix.io_tenantcontrolplanes_spec.yaml") . | nindent 2 }}

charts/kamaji-crds/values.yaml

@@ -0,0 +1,15 @@
+# Default values for kamaji-crds.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+# -- Overrides the name of the chart for resource naming purposes.
+nameOverride: ""
+# -- Overrides the full name of the resources created by the chart.
+fullnameOverride: ""
+
+# -- The namespace where Kamaji has been installed: required to inject the Certificate Authority for cert-manager.
+kamajiNamespace: kamaji-system
+# -- The Kamaji webhook Service name.
+kamajiService: kamaji-webhook-service
+# -- The cert-manager Certificate resource name, holding the Certificate Authority for webhooks.
+kamajiCertificateName: kamaji-serving-cert

charts/kamaji/.helmignore

@@ -21,3 +21,8 @@
 .idea/
 *.tmproj
 .vscode/
+# Helm source files
+README.md.gotmpl
+.helmignore
+# Build tools
+Makefile

charts/kamaji/Chart.lock

@@ -1,6 +1,6 @@
 dependencies:
 - name: kamaji-etcd
   repository: https://clastix.github.io/charts
-  version: 0.9.2
-digest: sha256:ba76d3a30e5e20dbbbbcc36a0e7465d4b1adacc956061e7f6ea47b99fc8f08a6
-generated: "2025-03-14T21:23:30.421915+09:00"
+  version: 0.11.0
+digest: sha256:96b4115b8c02f771f809ec1bed3be3a3903e7e8315d6966aa54b0f73230ea421
+generated: "2025-07-03T09:19:19.835421461+02:00"

charts/kamaji/Chart.yaml

@@ -1,5 +1,5 @@
 apiVersion: v2
-appVersion: v0.0.0
+appVersion: latest
 description: Kamaji is the Hosted Control Plane Manager for Kubernetes.
 home: https://github.com/clastix/kamaji
 icon: https://github.com/clastix/kamaji/raw/master/assets/logo-colored.png
@@ -17,11 +17,11 @@ name: kamaji
 sources:
 - https://github.com/clastix/kamaji
 type: application
-version: 0.0.0
+version: 0.0.0+latest
 dependencies:
 - name: kamaji-etcd
   repository: https://clastix.github.io/charts
-  version: ">=0.9.2"
+  version: ">=0.11.0"
   condition: kamaji-etcd.deploy
 annotations:
   catalog.cattle.io/certified: partner
@@ -46,4 +46,5 @@ annotations:
   artifacthub.io/operator: "true"
   artifacthub.io/operatorCapabilities: "full lifecycle"
   artifacthub.io/changes: |
-    - Using dependency chart `kamaji-etcd` as a default DataStore.
+    - kind: added
+      description: Releasing latest chart at every push

charts/kamaji/README.md

@@ -1,6 +1,6 @@
 # kamaji
 
-![Version: 0.0.0](https://img.shields.io/badge/Version-0.0.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v0.0.0](https://img.shields.io/badge/AppVersion-v0.0.0-informational?style=flat-square)
+![Version: 0.0.0+latest](https://img.shields.io/badge/Version-0.0.0+latest-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: latest](https://img.shields.io/badge/AppVersion-latest-informational?style=flat-square)
 
 Kamaji is the Hosted Control Plane Manager for Kubernetes.
 
@@ -22,7 +22,7 @@ Kubernetes: `>=1.21.0-0`
 
 | Repository | Name | Version |
 |------------|------|---------|
-| https://clastix.github.io/charts | kamaji-etcd | >=0.9.2 |
+| https://clastix.github.io/charts | kamaji-etcd | >=0.11.0 |
 
 [Kamaji](https://github.com/clastix/kamaji) requires a [multi-tenant `etcd`](https://github.com/clastix/kamaji-internal/blob/master/deploy/getting-started-with-kamaji.md#setup-internal-multi-tenant-etcd) cluster.
 This Helm Chart starting from v0.1.1 provides the installation of an internal `etcd` in order to streamline the local test. If you'd like to use an externally managed etcd instance, you can specify the overrides and by setting the value `etcd.deploy=false`.
@@ -82,10 +82,25 @@ Here the values you can override:
 | image.repository | string | `"clastix/kamaji"` | The container image of the Kamaji controller. |
 | image.tag | string | `nil` | Overrides the image tag whose default is the chart appVersion. |
 | imagePullSecrets | list | `[]` |  |
-| kamaji-etcd.datastore.enabled | bool | `true` |  |
-| kamaji-etcd.datastore.name | string | `"default"` |  |
-| kamaji-etcd.deploy | bool | `true` |  |
-| kamaji-etcd.fullnameOverride | string | `"kamaji-etcd"` |  |
+| kamaji-etcd | object | `{"clusterDomain":"cluster.local","datastore":{"enabled":true,"name":"default"},"deploy":true,"fullnameOverride":"kamaji-etcd"}` | Subchart: See https://github.com/clastix/kamaji-etcd/blob/master/charts/kamaji-etcd/values.yaml |
+| kubeconfigGenerator.affinity | object | `{}` | Kubernetes affinity rules to apply to Kubeconfig Generator controller pods |
+| kubeconfigGenerator.enableLeaderElect | bool | `true` | Enables the leader election. |
+| kubeconfigGenerator.enabled | bool | `false` | Toggle to deploy the Kubeconfig Generator Deployment. |
+| kubeconfigGenerator.extraArgs | list | `[]` | A list of extra arguments to add to the Kubeconfig Generator controller default ones. |
+| kubeconfigGenerator.fullnameOverride | string | `""` |  |
+| kubeconfigGenerator.healthProbeBindAddress | string | `":8081"` | The address the probe endpoint binds to. |
+| kubeconfigGenerator.loggingDevel.enable | bool | `false` | Development Mode defaults(encoder=consoleEncoder,logLevel=Debug,stackTraceLevel=Warn). Production Mode defaults(encoder=jsonEncoder,logLevel=Info,stackTraceLevel=Error) |
+| kubeconfigGenerator.nodeSelector | object | `{}` | Kubernetes node selector rules to schedule Kubeconfig Generator controller |
+| kubeconfigGenerator.podAnnotations | object | `{}` | The annotations to apply to the Kubeconfig Generator controller pods. |
+| kubeconfigGenerator.podSecurityContext | object | `{"runAsNonRoot":true}` | The securityContext to apply to the Kubeconfig Generator controller pods. |
+| kubeconfigGenerator.replicaCount | int | `2` | The number of the pod replicas for the Kubeconfig Generator controller. |
+| kubeconfigGenerator.resources.limits.cpu | string | `"200m"` |  |
+| kubeconfigGenerator.resources.limits.memory | string | `"512Mi"` |  |
+| kubeconfigGenerator.resources.requests.cpu | string | `"200m"` |  |
+| kubeconfigGenerator.resources.requests.memory | string | `"512Mi"` |  |
+| kubeconfigGenerator.securityContext | object | `{"allowPrivilegeEscalation":false}` | The securityContext to apply to the Kubeconfig Generator controller container only. |
+| kubeconfigGenerator.serviceAccountOverride | string | `""` | The name of the service account to use. If not set, the root Kamaji one will be used. |
+| kubeconfigGenerator.tolerations | list | `[]` | Kubernetes node taints that the Kubeconfig Generator controller pods would tolerate |
 | livenessProbe | object | `{"httpGet":{"path":"/healthz","port":"healthcheck"},"initialDelaySeconds":15,"periodSeconds":20}` | The livenessProbe for the controller container |
 | loggingDevel.enable | bool | `false` | Development Mode defaults(encoder=consoleEncoder,logLevel=Debug,stackTraceLevel=Warn). Production Mode defaults(encoder=jsonEncoder,logLevel=Info,stackTraceLevel=Error) (default false) |
 | metricsBindAddress | string | `":8080"` | The address the metric endpoint binds to. (default ":8080") |

charts/kamaji/app-readme.md

@@ -1,12 +0,0 @@
-# Kamaji
-
-Kamaji deploys and operates Kubernetes at scale with a fraction of the operational burden.
-
-Useful links:
-- [Kamaji Github repository](https://github.com/clastix/kamaji)
-- [Kamaji Documentation](https://kamaji.clastix.io)
-
-## Requirements
-
-* Kubernetes v1.22+
-* Helm v3

charts/kamaji/controller-gen/clusterrole.yaml

@@ -1,3 +1,25 @@
+- apiGroups:
+    - ""
+  resources:
+    - configmaps
+    - secrets
+    - services
+  verbs:
+    - create
+    - delete
+    - get
+    - list
+    - patch
+    - update
+    - watch
+- apiGroups:
+    - ""
+  resources:
+    - namespaces
+  verbs:
+    - get
+    - list
+    - watch
 - apiGroups:
     - apps
   resources:
@@ -21,11 +43,19 @@
     - list
     - watch
 - apiGroups:
-    - ""
+    - gateway.networking.k8s.io
   resources:
-    - configmaps
-    - secrets
-    - services
+    - gateways
+  verbs:
+    - get
+    - list
+    - watch
+- apiGroups:
+    - gateway.networking.k8s.io
+  resources:
+    - grpcroutes
+    - httproutes
+    - tlsroutes
   verbs:
     - create
     - delete
@@ -51,6 +81,7 @@
     - kamaji.clastix.io
   resources:
     - datastores/status
+    - kubeconfiggenerators/status
     - tenantcontrolplanes/status
   verbs:
     - get
@@ -59,6 +90,18 @@
 - apiGroups:
     - kamaji.clastix.io
   resources:
+    - kubeconfiggenerators
+  verbs:
+    - create
+    - get
+    - list
+    - patch
+    - update
+    - watch
+- apiGroups:
+    - kamaji.clastix.io
+  resources:
+    - kubeconfiggenerators/finalizers
     - tenantcontrolplanes/finalizers
   verbs:
     - update

charts/kamaji/controller-gen/validating-webhook.yaml

@@ -19,46 +19,6 @@
       resources:
         - tenantcontrolplanes
   sideEffects: None
-- admissionReviewVersions:
-    - v1
-  clientConfig:
-    service:
-      name: '{{ include "kamaji.webhookServiceName" . }}'
-      namespace: '{{ .Release.Namespace }}'
-      path: /validate-kamaji-clastix-io-v1alpha1-datastore
-  failurePolicy: Fail
-  name: vdatastore.kb.io
-  rules:
-    - apiGroups:
-        - kamaji.clastix.io
-      apiVersions:
-        - v1alpha1
-      operations:
-        - CREATE
-        - UPDATE
-        - DELETE
-      resources:
-        - datastores
-  sideEffects: None
-- admissionReviewVersions:
-    - v1
-  clientConfig:
-    service:
-      name: '{{ include "kamaji.webhookServiceName" . }}'
-      namespace: '{{ .Release.Namespace }}'
-      path: /validate--v1-secret
-  failurePolicy: Ignore
-  name: vdatastoresecrets.kb.io
-  rules:
-    - apiGroups:
-        - ""
-      apiVersions:
-        - v1
-      operations:
-        - DELETE
-      resources:
-        - secrets
-  sideEffects: None
 - admissionReviewVersions:
     - v1
   clientConfig:

charts/kamaji/crds/kamaji.clastix.io_datastores.yaml

@@ -4,7 +4,7 @@ kind: CustomResourceDefinition
 metadata:
   annotations:
     cert-manager.io/inject-ca-from: kamaji-system/kamaji-serving-cert
-    controller-gen.kubebuilder.io/version: v0.16.1
+    controller-gen.kubebuilder.io/version: v0.20.0
   name: datastores.kamaji.clastix.io
 spec:
   group: kamaji.clastix.io
@@ -20,6 +20,10 @@ spec:
           jsonPath: .spec.driver
           name: Driver
           type: string
+        - description: DataStore validated and ready for use
+          jsonPath: .status.ready
+          name: Ready
+          type: boolean
         - description: Age
           jsonPath: .metadata.creationTimestamp
           name: Age
@@ -120,6 +124,9 @@ spec:
                     - PostgreSQL
                     - NATS
                   type: string
+                  x-kubernetes-validations:
+                    - message: Datastore driver is immutable
+                      rule: self == oldSelf
                 endpoints:
                   description: |-
                     List of the endpoints to connect to the shared datastore.
@@ -263,14 +270,98 @@ spec:
                 - driver
                 - endpoints
               type: object
+              x-kubernetes-validations:
+                - message: certificateAuthority privateKey must have secretReference or content when driver is etcd
+                  rule: '(self.driver == "etcd") ? (self.tlsConfig != null && (has(self.tlsConfig.certificateAuthority.privateKey.secretReference) || has(self.tlsConfig.certificateAuthority.privateKey.content))) : true'
+                - message: clientCertificate must have secretReference or content when driver is etcd
+                  rule: '(self.driver == "etcd") ? (self.tlsConfig != null && (has(self.tlsConfig.clientCertificate.certificate.secretReference) || has(self.tlsConfig.clientCertificate.certificate.content))) : true'
+                - message: clientCertificate privateKey must have secretReference or content when driver is etcd
+                  rule: '(self.driver == "etcd") ? (self.tlsConfig != null && (has(self.tlsConfig.clientCertificate.privateKey.secretReference) || has(self.tlsConfig.clientCertificate.privateKey.content))) : true'
+                - message: When driver is not etcd and tlsConfig exists, clientCertificate must be null or contain valid content
+                  rule: '(self.driver != "etcd" && has(self.tlsConfig) && has(self.tlsConfig.clientCertificate)) ? (((has(self.tlsConfig.clientCertificate.certificate.secretReference) || has(self.tlsConfig.clientCertificate.certificate.content)))) : true'
+                - message: When driver is not etcd and basicAuth exists, username must have secretReference or content
+                  rule: '(self.driver != "etcd" && has(self.basicAuth)) ? ((has(self.basicAuth.username.secretReference) || has(self.basicAuth.username.content))) : true'
+                - message: When driver is not etcd and basicAuth exists, password must have secretReference or content
+                  rule: '(self.driver != "etcd" && has(self.basicAuth)) ? ((has(self.basicAuth.password.secretReference) || has(self.basicAuth.password.content))) : true'
+                - message: When driver is not etcd, either tlsConfig or basicAuth must be provided
+                  rule: '(self.driver != "etcd") ? (has(self.tlsConfig) || has(self.basicAuth)) : true'
+                - message: driver is immutable and cannot be changed after creation
+                  rule: oldSelf == null || self.driver == oldSelf.driver
             status:
               description: DataStoreStatus defines the observed state of DataStore.
               properties:
+                conditions:
+                  description: Conditions contains the validation conditions for the given Datastore.
+                  items:
+                    description: Condition contains details for one aspect of the current state of this API Resource.
+                    properties:
+                      lastTransitionTime:
+                        description: |-
+                          lastTransitionTime is the last time the condition transitioned from one status to another.
+                          This should be when the underlying condition changed.  If that is not known, then using the time when the API field changed is acceptable.
+                        format: date-time
+                        type: string
+                      message:
+                        description: |-
+                          message is a human readable message indicating details about the transition.
+                          This may be an empty string.
+                        maxLength: 32768
+                        type: string
+                      observedGeneration:
+                        description: |-
+                          observedGeneration represents the .metadata.generation that the condition was set based upon.
+                          For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date
+                          with respect to the current state of the instance.
+                        format: int64
+                        minimum: 0
+                        type: integer
+                      reason:
+                        description: |-
+                          reason contains a programmatic identifier indicating the reason for the condition's last transition.
+                          Producers of specific condition types may define expected values and meanings for this field,
+                          and whether the values are considered a guaranteed API.
+                          The value should be a CamelCase string.
+                          This field may not be empty.
+                        maxLength: 1024
+                        minLength: 1
+                        pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
+                        type: string
+                      status:
+                        description: status of the condition, one of True, False, Unknown.
+                        enum:
+                          - "True"
+                          - "False"
+                          - Unknown
+                        type: string
+                      type:
+                        description: type of condition in CamelCase or in foo.example.com/CamelCase.
+                        maxLength: 316
+                        pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
+                        type: string
+                    required:
+                      - lastTransitionTime
+                      - message
+                      - reason
+                      - status
+                      - type
+                    type: object
+                  type: array
+                observedGeneration:
+                  description: ObservedGeneration represents the .metadata.generation that was last reconciled.
+                  format: int64
+                  type: integer
+                ready:
+                  description: |-
+                    Ready returns if the DataStore is accepted and ready to get used:
+                    Kamaji will ensure certificates are available, or correctly referenced.
+                  type: boolean
                 usedBy:
                   description: List of the Tenant Control Planes, namespaced named, using this data store.
                   items:
                     type: string
                   type: array
+              required:
+                - ready
               type: object
           type: object
       served: true

charts/kamaji/crds/kamaji.clastix.io_kubeconfiggenerators.yaml

@@ -0,0 +1,226 @@
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    cert-manager.io/inject-ca-from: kamaji-system/kamaji-serving-cert
+    controller-gen.kubebuilder.io/version: v0.20.0
+  name: kubeconfiggenerators.kamaji.clastix.io
+spec:
+  group: kamaji.clastix.io
+  names:
+    categories:
+      - kamaji
+    kind: KubeconfigGenerator
+    listKind: KubeconfigGeneratorList
+    plural: kubeconfiggenerators
+    shortNames:
+      - kc
+    singular: kubeconfiggenerator
+  scope: Cluster
+  versions:
+    - additionalPrinterColumns:
+        - description: Age
+          jsonPath: .metadata.creationTimestamp
+          name: Age
+          type: date
+      name: v1alpha1
+      schema:
+        openAPIV3Schema:
+          description: KubeconfigGenerator is the Schema for the kubeconfiggenerators API.
+          properties:
+            apiVersion:
+              description: |-
+                APIVersion defines the versioned schema of this representation of an object.
+                Servers should convert recognized schemas to the latest internal value, and
+                may reject unrecognized values.
+                More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+              type: string
+            kind:
+              description: |-
+                Kind is a string value representing the REST resource this object represents.
+                Servers may infer this from the endpoint the client submits requests to.
+                Cannot be updated.
+                In CamelCase.
+                More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+              type: string
+            metadata:
+              type: object
+            spec:
+              properties:
+                controlPlaneEndpointFrom:
+                  default: admin.svc
+                  description: |-
+                    ControlPlaneEndpointFrom is the key used to extract the Tenant Control Plane endpoint that must be used by the generator.
+                    The targeted Secret is the `${TCP}-admin-kubeconfig` one, default to `admin.svc`.
+                  type: string
+                groups:
+                  description: |-
+                    Groups is resolved a set of strings used to assign the x509 organisations field.
+                    It will be recognised by Kubernetes as user groups.
+                  items:
+                    description: |-
+                      CompoundValue allows defining a static, or a dynamic value.
+                      Options are mutually exclusive, just one should be picked up.
+                    properties:
+                      fromDefinition:
+                        description: |-
+                          FromDefinition is used to generate a dynamic value,
+                          it uses the dot notation to access fields from the referenced TenantControlPlane object:
+                          e.g.: metadata.name
+                        type: string
+                      stringValue:
+                        description: StringValue is a static string value.
+                        type: string
+                    type: object
+                    x-kubernetes-validations:
+                      - message: Either stringValue or fromDefinition must be set, but not both.
+                        rule: (has(self.stringValue) || has(self.fromDefinition)) && !(has(self.stringValue) && has(self.fromDefinition))
+                  type: array
+                namespaceSelector:
+                  description: NamespaceSelector is used to filter Namespaces from which the generator should extract TenantControlPlane objects.
+                  properties:
+                    matchExpressions:
+                      description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
+                      items:
+                        description: |-
+                          A label selector requirement is a selector that contains values, a key, and an operator that
+                          relates the key and values.
+                        properties:
+                          key:
+                            description: key is the label key that the selector applies to.
+                            type: string
+                          operator:
+                            description: |-
+                              operator represents a key's relationship to a set of values.
+                              Valid operators are In, NotIn, Exists and DoesNotExist.
+                            type: string
+                          values:
+                            description: |-
+                              values is an array of string values. If the operator is In or NotIn,
+                              the values array must be non-empty. If the operator is Exists or DoesNotExist,
+                              the values array must be empty. This array is replaced during a strategic
+                              merge patch.
+                            items:
+                              type: string
+                            type: array
+                            x-kubernetes-list-type: atomic
+                        required:
+                          - key
+                          - operator
+                        type: object
+                      type: array
+                      x-kubernetes-list-type: atomic
+                    matchLabels:
+                      additionalProperties:
+                        type: string
+                      description: |-
+                        matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels
+                        map is equivalent to an element of matchExpressions, whose key field is "key", the
+                        operator is "In", and the values array contains only "value". The requirements are ANDed.
+                      type: object
+                  type: object
+                  x-kubernetes-map-type: atomic
+                tenantControlPlaneSelector:
+                  description: TenantControlPlaneSelector is used to filter the TenantControlPlane objects that should be address by the generator.
+                  properties:
+                    matchExpressions:
+                      description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
+                      items:
+                        description: |-
+                          A label selector requirement is a selector that contains values, a key, and an operator that
+                          relates the key and values.
+                        properties:
+                          key:
+                            description: key is the label key that the selector applies to.
+                            type: string
+                          operator:
+                            description: |-
+                              operator represents a key's relationship to a set of values.
+                              Valid operators are In, NotIn, Exists and DoesNotExist.
+                            type: string
+                          values:
+                            description: |-
+                              values is an array of string values. If the operator is In or NotIn,
+                              the values array must be non-empty. If the operator is Exists or DoesNotExist,
+                              the values array must be empty. This array is replaced during a strategic
+                              merge patch.
+                            items:
+                              type: string
+                            type: array
+                            x-kubernetes-list-type: atomic
+                        required:
+                          - key
+                          - operator
+                        type: object
+                      type: array
+                      x-kubernetes-list-type: atomic
+                    matchLabels:
+                      additionalProperties:
+                        type: string
+                      description: |-
+                        matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels
+                        map is equivalent to an element of matchExpressions, whose key field is "key", the
+                        operator is "In", and the values array contains only "value". The requirements are ANDed.
+                      type: object
+                  type: object
+                  x-kubernetes-map-type: atomic
+                user:
+                  description: User resolves to a string to identify the client, assigned to the x509 Common Name field.
+                  properties:
+                    fromDefinition:
+                      description: |-
+                        FromDefinition is used to generate a dynamic value,
+                        it uses the dot notation to access fields from the referenced TenantControlPlane object:
+                        e.g.: metadata.name
+                      type: string
+                    stringValue:
+                      description: StringValue is a static string value.
+                      type: string
+                  type: object
+                  x-kubernetes-validations:
+                    - message: Either stringValue or fromDefinition must be set, but not both.
+                      rule: (has(self.stringValue) || has(self.fromDefinition)) && !(has(self.stringValue) && has(self.fromDefinition))
+              required:
+                - user
+              type: object
+            status:
+              description: KubeconfigGeneratorStatus defines the observed state of KubeconfigGenerator.
+              properties:
+                availableResources:
+                  default: 0
+                  description: |-
+                    AvailableResources is the sum of successfully generated resources.
+                    In case of a different value compared to Resources, check the field errors.
+                  type: integer
+                errors:
+                  description: Errors is the list of failed kubeconfig generations.
+                  items:
+                    properties:
+                      message:
+                        description: Message is the error message recorded upon the last generator run.
+                        type: string
+                      resource:
+                        description: Resource is the Namespaced name of the errored resource.
+                        type: string
+                    required:
+                      - message
+                      - resource
+                    type: object
+                  type: array
+                observedGeneration:
+                  description: ObservedGeneration represents the .metadata.generation that was last reconciled.
+                  format: int64
+                  type: integer
+                resources:
+                  default: 0
+                  description: Resources is the sum of targeted TenantControlPlane objects.
+                  type: integer
+              required:
+                - availableResources
+                - resources
+              type: object
+          type: object
+      served: true
+      storage: true
+      subresources:
+        status: {}

charts/kamaji/templates/_helpers.tpl

@@ -89,3 +89,15 @@ Create the name of the cert-manager Certificate
 {{- define "kamaji.certificateName" -}}
 {{- printf "%s-serving-cert" (include "kamaji.fullname" .) }}
 {{- end }}
+
+
+{{/*
+Kubeconfig Generator Deployment name.
+*/}}
+{{- define "kamaji.kubeconfigGeneratorName" -}}
+{{- if .Values.kubeconfigGenerator.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name "kubeconfig-generator" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}

charts/kamaji/templates/controller.yaml

@@ -19,10 +19,6 @@ spec:
       labels:
         {{- include "kamaji.selectorLabels" . | nindent 8 }}
     spec:
-      {{- with .Values.imagePullSecrets }}
-      imagePullSecrets:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
       securityContext:
         {{- toYaml .Values.podSecurityContext | nindent 8 }}
       serviceAccountName: {{ include "kamaji.serviceAccountName" . }}

charts/kamaji/templates/kubeconfiggenerator-deployment.yaml

@@ -0,0 +1,54 @@
+{{- if .Values.kubeconfigGenerator.enabled }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    {{- include "kamaji.labels" . | nindent 4 }}
+  name: {{ include "kamaji.kubeconfigGeneratorName" . }}
+  namespace: {{ .Release.Namespace }}
+spec:
+  replicas: {{ .Values.kubeconfigGenerator.replicaCount }}
+  selector:
+    matchLabels:
+      {{- include "kamaji.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      {{- with .Values.kubeconfigGenerator.podAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      labels:
+        {{- include "kamaji.selectorLabels" . | nindent 8 }}
+    spec:
+      securityContext:
+        {{- toYaml .Values.kubeconfigGenerator.podSecurityContext | nindent 8 }}
+      serviceAccountName: {{ default .Values.kubeconfigGenerator.serviceAccountOverride (include "kamaji.serviceAccountName" .) }}
+      containers:
+        - args:
+          - kubeconfig-generator
+          - --health-probe-bind-address={{ .Values.kubeconfigGenerator.healthProbeBindAddress }}
+          - --leader-elect={{ .Values.kubeconfigGenerator.enableLeaderElect }}
+          {{- if .Values.kubeconfigGenerator.loggingDevel.enable }}- --zap-devel{{- end }}
+          {{- with .Values.kubeconfigGenerator.extraArgs }}
+          {{- toYaml . | nindent 10 }}
+          {{- end }}
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          name: controller
+          resources:
+            {{- toYaml .Values.kubeconfigGenerator.resources | nindent 12 }}
+          securityContext:
+            {{- toYaml .Values.kubeconfigGenerator.securityContext | nindent 12 }}
+      {{- with .Values.kubeconfigGenerator.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.kubeconfigGenerator.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.kubeconfigGenerator.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+{{- end }}

charts/kamaji/templates/rbac.yaml

@@ -9,6 +9,10 @@ metadata:
     {{- toYaml . | nindent 4 }}
   {{- end }}
   namespace: {{ .Release.Namespace }}
+{{- with .Values.imagePullSecrets }}
+imagePullSecrets:
+  {{- toYaml . | nindent 2 }}
+{{- end }}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role

charts/kamaji/values.yaml

@@ -98,9 +98,12 @@ loggingDevel:
 # -- If specified, all the Kamaji instances with an unassigned DataStore will inherit this default value.
 defaultDatastoreName: default
 
+# -- Subchart: See https://github.com/clastix/kamaji-etcd/blob/master/charts/kamaji-etcd/values.yaml
 kamaji-etcd:
   deploy: true
   fullnameOverride: kamaji-etcd
+  ## -- Important, this must match your management cluster's clusterDomain, otherwise the init jobs will fail
+  clusterDomain: "cluster.local"
   datastore:
     enabled: true
     name: default
@@ -108,4 +111,48 @@ kamaji-etcd:
 # -- Disable the analytics traces collection
 telemetry:
   disabled: false
-  
+
+kubeconfigGenerator:
+  # -- Toggle to deploy the Kubeconfig Generator Deployment.
+  enabled: false
+  fullnameOverride: ""
+  # -- The number of the pod replicas for the Kubeconfig Generator controller.
+  replicaCount: 2
+  # -- The annotations to apply to the Kubeconfig Generator controller pods.
+  podAnnotations: {}
+  # -- The securityContext to apply to the Kubeconfig Generator controller pods.
+  podSecurityContext:
+    runAsNonRoot: true
+  # -- The name of the service account to use. If not set, the root Kamaji one will be used.
+  serviceAccountOverride: ""
+  # -- The address the probe endpoint binds to.
+  healthProbeBindAddress: ":8081"
+  # -- Enables the leader election.
+  enableLeaderElect: true
+  loggingDevel:
+    # -- Development Mode defaults(encoder=consoleEncoder,logLevel=Debug,stackTraceLevel=Warn). Production Mode defaults(encoder=jsonEncoder,logLevel=Info,stackTraceLevel=Error)
+    enable: false
+  # -- A list of extra arguments to add to the Kubeconfig Generator controller default ones.
+  extraArgs: []
+  resources:
+    limits:
+      cpu: 200m
+      memory: 512Mi
+    requests:
+      cpu: 200m
+      memory: 512Mi
+  # -- The securityContext to apply to the Kubeconfig Generator controller container only.
+  securityContext:
+    allowPrivilegeEscalation: false
+    # capabilities:
+    #   drop:
+    #   - ALL
+    # readOnlyRootFilesystem: true
+    # runAsNonRoot: true
+    # runAsUser: 1000
+  # -- Kubernetes node selector rules to schedule Kubeconfig Generator controller
+  nodeSelector: {}
+  # -- Kubernetes node taints that the Kubeconfig Generator controller pods would tolerate
+  tolerations: []
+  # -- Kubernetes affinity rules to apply to Kubeconfig Generator controller pods
+  affinity: {}

cmd/kubeconfig-generator/cmd.go

@@ -0,0 +1,167 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package kubeconfiggenerator
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	goRuntime "runtime"
+	"time"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/rest"
+	"k8s.io/klog/v2"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/cache"
+	"sigs.k8s.io/controller-runtime/pkg/event"
+	"sigs.k8s.io/controller-runtime/pkg/healthz"
+	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
+
+	"github.com/clastix/kamaji/controllers"
+	"github.com/clastix/kamaji/internal"
+)
+
+func NewCmd(scheme *runtime.Scheme) *cobra.Command {
+	// CLI flags
+	var (
+		metricsBindAddress            string
+		healthProbeBindAddress        string
+		leaderElect                   bool
+		controllerReconcileTimeout    time.Duration
+		cacheResyncPeriod             time.Duration
+		managerNamespace              string
+		certificateExpirationDeadline time.Duration
+	)
+
+	cmd := &cobra.Command{
+		Use:           "kubeconfig-generator",
+		Short:         "Start the Kubeconfig Generator manager",
+		SilenceErrors: false,
+		SilenceUsage:  true,
+		PreRunE: func(*cobra.Command, []string) error {
+			// Avoid polluting stdout with useless details by the underlying klog implementations
+			klog.SetOutput(io.Discard)
+			klog.LogToStderr(false)
+
+			if certificateExpirationDeadline < 24*time.Hour {
+				return fmt.Errorf("certificate expiration deadline must be at least 24 hours")
+			}
+
+			return nil
+		},
+		RunE: func(*cobra.Command, []string) error {
+			ctx := ctrl.SetupSignalHandler()
+
+			setupLog := ctrl.Log.WithName("kubeconfig-generator")
+
+			setupLog.Info(fmt.Sprintf("Kamaji version %s %s%s", internal.GitTag, internal.GitCommit, internal.GitDirty))
+			setupLog.Info(fmt.Sprintf("Build from: %s", internal.GitRepo))
+			setupLog.Info(fmt.Sprintf("Build date: %s", internal.BuildTime))
+			setupLog.Info(fmt.Sprintf("Go Version: %s", goRuntime.Version()))
+			setupLog.Info(fmt.Sprintf("Go OS/Arch: %s/%s", goRuntime.GOOS, goRuntime.GOARCH))
+
+			ctrlOpts := ctrl.Options{
+				Scheme: scheme,
+				Metrics: metricsserver.Options{
+					BindAddress: metricsBindAddress,
+				},
+				HealthProbeBindAddress:  healthProbeBindAddress,
+				LeaderElection:          leaderElect,
+				LeaderElectionNamespace: managerNamespace,
+				LeaderElectionID:        "kubeconfiggenerator.kamaji.clastix.io",
+				NewCache: func(config *rest.Config, opts cache.Options) (cache.Cache, error) {
+					opts.SyncPeriod = &cacheResyncPeriod
+
+					return cache.New(config, opts)
+				},
+			}
+
+			triggerChan := make(chan event.GenericEvent)
+
+			mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrlOpts)
+			if err != nil {
+				setupLog.Error(err, "unable to start manager")
+
+				return err
+			}
+
+			setupLog.Info("setting probes")
+			{
+				if err = mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
+					setupLog.Error(err, "unable to set up health check")
+
+					return err
+				}
+				if err = mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil {
+					setupLog.Error(err, "unable to set up ready check")
+
+					return err
+				}
+			}
+
+			certController := &controllers.CertificateLifecycle{Channel: triggerChan, Deadline: certificateExpirationDeadline}
+			certController.EnqueueFn = certController.EnqueueForKubeconfigGenerator
+			if err = certController.SetupWithManager(mgr); err != nil {
+				setupLog.Error(err, "unable to create controller", "controller", "CertificateLifecycle")
+
+				return err
+			}
+
+			if err = (&controllers.KubeconfigGeneratorWatcher{
+				Client:        mgr.GetClient(),
+				GeneratorChan: triggerChan,
+			}).SetupWithManager(mgr); err != nil {
+				setupLog.Error(err, "unable to create controller", "controller", "KubeconfigGeneratorWatcher")
+
+				return err
+			}
+
+			if err = (&controllers.KubeconfigGeneratorReconciler{
+				Client:            mgr.GetClient(),
+				NotValidThreshold: certificateExpirationDeadline,
+				CertificateChan:   triggerChan,
+			}).SetupWithManager(mgr); err != nil {
+				setupLog.Error(err, "unable to create controller", "controller", "KubeconfigGenerator")
+
+				return err
+			}
+
+			setupLog.Info("starting manager")
+			if err = mgr.Start(ctx); err != nil {
+				setupLog.Error(err, "problem running manager")
+
+				return err
+			}
+
+			return nil
+		},
+	}
+	// Setting zap logger
+	zapfs := flag.NewFlagSet("zap", flag.ExitOnError)
+	opts := zap.Options{
+		Development: true,
+	}
+	opts.BindFlags(zapfs)
+	cmd.Flags().AddGoFlagSet(zapfs)
+	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
+	// Setting CLI flags
+	cmd.Flags().StringVar(&metricsBindAddress, "metrics-bind-address", ":8090", "The address the metric endpoint binds to.")
+	cmd.Flags().StringVar(&healthProbeBindAddress, "health-probe-bind-address", ":8091", "The address the probe endpoint binds to.")
+	cmd.Flags().BoolVar(&leaderElect, "leader-elect", true, "Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.")
+	cmd.Flags().DurationVar(&controllerReconcileTimeout, "controller-reconcile-timeout", 30*time.Second, "The reconciliation request timeout before the controller withdraw the external resource calls, such as dealing with the Datastore, or the Tenant Control Plane API endpoint.")
+	cmd.Flags().DurationVar(&cacheResyncPeriod, "cache-resync-period", 10*time.Hour, "The controller-runtime.Manager cache resync period.")
+	cmd.Flags().StringVar(&managerNamespace, "pod-namespace", os.Getenv("POD_NAMESPACE"), "The Kubernetes Namespace on which the Operator is running in, required for the TenantControlPlane migration jobs.")
+	cmd.Flags().DurationVar(&certificateExpirationDeadline, "certificate-expiration-deadline", 24*time.Hour, "Define the deadline upon certificate expiration to start the renewal process, cannot be less than a 24 hours.")
+
+	cobra.OnInitialize(func() {
+		viper.AutomaticEnv()
+	})
+
+	return cmd
+}

cmd/manager/cmd.go

@@ -16,10 +16,12 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/discovery"
 	"k8s.io/client-go/rest"
 	"k8s.io/klog/v2"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/cache"
+	"sigs.k8s.io/controller-runtime/pkg/event"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
@@ -31,7 +33,7 @@ import (
 	"github.com/clastix/kamaji/controllers/soot"
 	"github.com/clastix/kamaji/internal"
 	"github.com/clastix/kamaji/internal/builders/controlplane"
-	datastoreutils "github.com/clastix/kamaji/internal/datastore/utils"
+	"github.com/clastix/kamaji/internal/utilities"
 	"github.com/clastix/kamaji/internal/webhook"
 	"github.com/clastix/kamaji/internal/webhook/handlers"
 	"github.com/clastix/kamaji/internal/webhook/routes"
@@ -61,8 +63,6 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command {
 		webhookCAPath string
 	)
 
-	ctx := ctrl.SetupSignalHandler()
-
 	cmd := &cobra.Command{
 		Use:           "manager",
 		Short:         "Start the Kamaji Kubernetes Operator",
@@ -85,10 +85,6 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command {
 				return fmt.Errorf("unable to read webhook CA: %w", err)
 			}
 
-			if err = datastoreutils.CheckExists(ctx, scheme, datastore); err != nil {
-				return err
-			}
-
 			if controllerReconcileTimeout.Seconds() == 0 {
 				return fmt.Errorf("the controller reconcile timeout must be greater than zero")
 			}
@@ -96,6 +92,8 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command {
 			return nil
 		},
 		RunE: func(*cobra.Command, []string) error {
+			ctx := ctrl.SetupSignalHandler()
+
 			setupLog := ctrl.Log.WithName("setup")
 
 			setupLog.Info(fmt.Sprintf("Kamaji version %s %s%s", internal.GitTag, internal.GitCommit, internal.GitDirty))
@@ -136,7 +134,7 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command {
 				return err
 			}
 
-			tcpChannel, certChannel := make(controllers.TenantControlPlaneChannel), make(controllers.CertificateChannel)
+			tcpChannel, certChannel := make(chan event.GenericEvent), make(chan event.GenericEvent)
 
 			if err = (&controllers.DataStore{Client: mgr.GetClient(), TenantControlPlaneTrigger: tcpChannel}).SetupWithManager(mgr); err != nil {
 				setupLog.Error(err, "unable to create controller", "controller", "DataStore")
@@ -144,15 +142,23 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command {
 				return err
 			}
 
+			discoveryClient, err := discovery.NewDiscoveryClientForConfig(mgr.GetConfig())
+			if err != nil {
+				setupLog.Error(err, "unable to create discovery client")
+
+				return err
+			}
+
 			reconciler := &controllers.TenantControlPlaneReconciler{
 				Client:    mgr.GetClient(),
 				APIReader: mgr.GetAPIReader(),
 				Config: controllers.TenantControlPlaneReconcilerConfig{
-					ReconcileTimeout:     controllerReconcileTimeout,
-					DefaultDataStoreName: datastore,
-					KineContainerImage:   kineImage,
-					TmpBaseDirectory:     tmpDirectory,
+					DefaultDataStoreName:    datastore,
+					KineContainerImage:      kineImage,
+					TmpBaseDirectory:        tmpDirectory,
+					CertExpirationThreshold: certificateExpirationDeadline,
 				},
+				ReconcileTimeout:        controllerReconcileTimeout,
 				CertificateChan:         certChannel,
 				TriggerChan:             tcpChannel,
 				KamajiNamespace:         managerNamespace,
@@ -160,9 +166,10 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command {
 				KamajiService:           managerServiceName,
 				KamajiMigrateImage:      migrateJobImage,
 				MaxConcurrentReconciles: maxConcurrentReconciles,
+				DiscoveryClient:         discoveryClient,
 			}
 
-			if err = reconciler.SetupWithManager(mgr); err != nil {
+			if err = reconciler.SetupWithManager(ctx, mgr); err != nil {
 				setupLog.Error(err, "unable to create controller", "controller", "Namespace")
 
 				return err
@@ -191,7 +198,10 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command {
 				}
 			}
 
-			if err = (&controllers.CertificateLifecycle{Channel: certChannel, Deadline: certificateExpirationDeadline}).SetupWithManager(mgr); err != nil {
+			certController := &controllers.CertificateLifecycle{Channel: certChannel, Deadline: certificateExpirationDeadline}
+			certController.EnqueueFn = certController.EnqueueForTenantControlPlane
+
+			if err = certController.SetupWithManager(mgr); err != nil {
 				setupLog.Error(err, "unable to create controller", "controller", "CertificateLifecycle")
 
 				return err
@@ -209,10 +219,22 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command {
 				return err
 			}
 
+			// Only requires to look for the core api group.
+			if utilities.AreGatewayResourcesAvailable(ctx, mgr.GetClient(), discoveryClient) {
+				if err = (&kamajiv1alpha1.GatewayListener{}).SetupWithManager(ctx, mgr); err != nil {
+					setupLog.Error(err, "unable to create indexer", "indexer", "GatewayListener")
+
+					return err
+				}
+			}
+
 			err = webhook.Register(mgr, map[routes.Route][]handlers.Handler{
 				routes.TenantControlPlaneMigrate{}: {
 					handlers.Freeze{},
 				},
+				routes.TenantControlPlaneWritePermission{}: {
+					handlers.WritePermission{},
+				},
 				routes.TenantControlPlaneDefaults{}: {
 					handlers.TenantControlPlaneDefaults{
 						DefaultDatastore: datastore,
@@ -222,7 +244,6 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command {
 					handlers.TenantControlPlaneCertSANs{},
 					handlers.TenantControlPlaneName{},
 					handlers.TenantControlPlaneVersion{},
-					handlers.TenantControlPlaneKubeletAddresses{},
 					handlers.TenantControlPlaneDataStore{Client: mgr.GetClient()},
 					handlers.TenantControlPlaneDeployment{
 						Client: mgr.GetClient(),
@@ -236,6 +257,10 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command {
 					},
 					handlers.TenantControlPlaneServiceCIDR{},
 					handlers.TenantControlPlaneLoadBalancerSourceRanges{},
+					handlers.TenantControlPlaneGatewayValidation{
+						Client:          mgr.GetClient(),
+						DiscoveryClient: discoveryClient,
+					},
 				},
 				routes.TenantControlPlaneTelemetry{}: {
 					handlers.TenantControlPlaneTelemetry{
@@ -245,12 +270,6 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command {
 						KubernetesVersion: k8sVersion,
 					},
 				},
-				routes.DataStoreValidate{}: {
-					handlers.DataStoreValidation{Client: mgr.GetClient()},
-				},
-				routes.DataStoreSecrets{}: {
-					handlers.DataStoreSecretValidation{Client: mgr.GetClient()},
-				},
 			})
 			if err != nil {
 				setupLog.Error(err, "unable to create webhook")
@@ -306,7 +325,7 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command {
 	cmd.Flags().StringVar(&tmpDirectory, "tmp-directory", "/tmp/kamaji", "Directory which will be used to work with temporary files.")
 	cmd.Flags().StringVar(&kineImage, "kine-image", "rancher/kine:v0.11.10-amd64", "Container image along with tag to use for the Kine sidecar container (used only if etcd-storage-type is set to one of kine strategies).")
 	cmd.Flags().StringVar(&datastore, "datastore", "", "Optional, the default DataStore that should be used by Kamaji to setup the required storage of Tenant Control Planes with undeclared DataStore.")
-	cmd.Flags().StringVar(&migrateJobImage, "migrate-image", fmt.Sprintf("clastix/kamaji:%s", internal.GitTag), "Specify the container image to launch when a TenantControlPlane is migrated to a new datastore.")
+	cmd.Flags().StringVar(&migrateJobImage, "migrate-image", fmt.Sprintf("%s/clastix/kamaji:%s", internal.ContainerRepository, internal.GitTag), "Specify the container image to launch when a TenantControlPlane is migrated to a new datastore.")
 	cmd.Flags().IntVar(&maxConcurrentReconciles, "max-concurrent-tcp-reconciles", 1, "Specify the number of workers for the Tenant Control Plane controller (beware of CPU consumption)")
 	cmd.Flags().StringVar(&managerNamespace, "pod-namespace", os.Getenv("POD_NAMESPACE"), "The Kubernetes Namespace on which the Operator is running in, required for the TenantControlPlane migration jobs.")
 	cmd.Flags().StringVar(&managerServiceName, "webhook-service-name", "kamaji-webhook-service", "The Kamaji webhook server Service name which is used to get validation webhooks, required for the TenantControlPlane migration jobs.")

cmd/migrate/cmd.go

@@ -22,9 +22,10 @@ import (
 func NewCmd(scheme *runtime.Scheme) *cobra.Command {
 	// CLI flags
 	var (
-		tenantControlPlane string
-		targetDataStore    string
-		timeout            time.Duration
+		tenantControlPlane    string
+		targetDataStore       string
+		cleanupPriorMigration bool
+		timeout               time.Duration
 	)
 
 	cmd := &cobra.Command{
@@ -95,6 +96,20 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command {
 				return err
 			}
 			defer targetConnection.Close()
+
+			if cleanupPriorMigration {
+				log.Info("Checking if target DataStore should be clean-up prior migration")
+
+				if exists, _ := targetConnection.DBExists(ctx, tcp.Status.Storage.Setup.Schema); exists {
+					log.Info("A colliding schema on target DataStore is present, cleaning up")
+
+					if dErr := targetConnection.DeleteDB(ctx, tcp.Status.Storage.Setup.Schema); dErr != nil {
+						return fmt.Errorf("error cleaning up prior migration: %s", dErr.Error())
+					}
+
+					log.Info("Cleaning up prior migration has been completed")
+				}
+			}
 			// Start migrating from the old Datastore to the new one
 			log.Info("migration from origin to target started")
 
@@ -110,6 +125,7 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command {
 
 	cmd.Flags().StringVar(&tenantControlPlane, "tenant-control-plane", "", "Namespaced-name of the TenantControlPlane that must be migrated (e.g.: default/test)")
 	cmd.Flags().StringVar(&targetDataStore, "target-datastore", "", "Name of the Datastore to which the TenantControlPlane will be migrated")
+	cmd.Flags().BoolVar(&cleanupPriorMigration, "cleanup-prior-migration", false, "When set to true, migration job will drop existing data in the target DataStore: useful to avoid stale data when migrating back and forth between DataStores.")
 	cmd.Flags().DurationVar(&timeout, "timeout", 5*time.Minute, "Amount of time for the context timeout")
 
 	_ = cmd.MarkFlagRequired("tenant-control-plane")

cmd/root.go

@@ -10,6 +10,8 @@ import (
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	appsv1 "k8s.io/kubernetes/pkg/apis/apps/v1"
+	gatewayv1 "sigs.k8s.io/gateway-api/apis/v1"
+	gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2"
 
 	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
 )
@@ -22,6 +24,10 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command {
 			utilruntime.Must(clientgoscheme.AddToScheme(scheme))
 			utilruntime.Must(kamajiv1alpha1.AddToScheme(scheme))
 			utilruntime.Must(appsv1.RegisterDefaults(scheme))
+			// NOTE: This will succeed even if Gateway API is not installed in the cluster.
+			// Only registers the go types.
+			utilruntime.Must(gatewayv1.Install(scheme))
+			utilruntime.Must(gatewayv1alpha2.Install(scheme))
 		},
 	}
 }

cmd/utils/k8s_version.go

@@ -4,7 +4,8 @@
 package utils
 
 import (
-	"github.com/pkg/errors"
+	"fmt"
+
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 )
@@ -12,12 +13,12 @@ import (
 func KubernetesVersion(config *rest.Config) (string, error) {
 	cs, csErr := kubernetes.NewForConfig(config)
 	if csErr != nil {
-		return "", errors.Wrap(csErr, "cannot create kubernetes clientset")
+		return "", fmt.Errorf("cannot create kubernetes clientset: %w", csErr)
 	}
 
 	sv, svErr := cs.ServerVersion()
 	if svErr != nil {
-		return "", errors.Wrap(svErr, "cannot get Kubernetes version")
+		return "", fmt.Errorf("cannot get Kubernetes version: %w", svErr)
 	}
 
 	return sv.GitVersion, nil

config/capi/clusterclass-kubevirt-kamaji-external.yaml

@@ -0,0 +1,154 @@
+---
+apiVersion: bootstrap.cluster.x-k8s.io/v1beta2
+kind: KubeadmConfigTemplate
+metadata:
+  name: worker-external
+  namespace: default
+spec:
+  template:
+    spec:
+      users:
+      joinConfiguration:
+        nodeRegistration:
+          kubeletExtraArgs:
+          - name: feature-gates
+            value: "KubeletCrashLoopBackOffMax=true,KubeletEnsureSecretPulledImages=true"
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
+kind: KubevirtMachineTemplate
+metadata:
+  name: worker-external
+  namespace: default
+spec:
+  template:
+    spec:
+      virtualMachineBootstrapCheck:
+        checkStrategy: ssh
+      virtualMachineTemplate:
+        metadata:
+          namespace: default
+        spec:
+          runStrategy: Always
+          template:
+            spec:
+              dnsPolicy: None
+              dnsConfig:
+                nameservers:
+                  - 1.1.1.1
+                  - 8.8.8.8
+                searches: []
+                options:
+                  - name: ndots
+                    value: "1"
+              domain:
+                cpu:
+                  cores: 2
+                devices:
+                  interfaces:
+                  - name: default
+                    masquerade: {}
+                  disks:
+                  - disk:
+                      bus: virtio
+                    name: containervolume
+                  networkInterfaceMultiqueue: true
+                memory:
+                  guest: 4Gi
+              evictionStrategy: External
+              networks:
+              - name: default
+                pod: {}
+              volumes:
+              - containerDisk:
+                  image: quay.io/capk/ubuntu-2404-container-disk:v1.34.1
+                name: containervolume
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
+kind: KubevirtClusterTemplate
+metadata:
+  name: kubevirt-external
+  namespace: default
+spec:
+  template:
+    metadata:
+      annotations:
+        cluster.x-k8s.io/managed-by: kamaji
+    spec:
+      controlPlaneServiceTemplate:
+        spec:
+          type: LoadBalancer
+---
+apiVersion: controlplane.cluster.x-k8s.io/v1alpha1
+kind: KamajiControlPlaneTemplate
+metadata:
+  name: kamaji-controlplane-external
+  namespace: default
+spec:
+  template:
+    spec:
+      addons:
+        coreDNS: {}
+        konnectivity: {}
+        kubeProxy: {}
+      dataStoreName: "default" # reference to DataStore present on external cluster
+      deployment:
+        externalClusterReference:
+          deploymentNamespace: kamaji-tenants
+          kubeconfigSecretName: kind-external-kubeconfig
+          kubeconfigSecretKey: kubeconfig
+      network:
+        serviceType: LoadBalancer
+      kubelet:
+        cgroupfs: systemd
+        preferredAddressTypes:
+        - InternalIP
+      registry: "registry.k8s.io"
+---
+apiVersion: cluster.x-k8s.io/v1beta2
+kind: ClusterClass
+metadata:
+  name: kubevirt-kamaji-kubeadm-external
+  namespace: default
+spec:
+  controlPlane:
+    templateRef:
+      apiVersion: controlplane.cluster.x-k8s.io/v1alpha1
+      kind: KamajiControlPlaneTemplate
+      name: kamaji-controlplane-external
+  infrastructure:
+    templateRef:
+      apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
+      kind: KubevirtClusterTemplate
+      name: kubevirt-external
+  workers:
+    machineDeployments:
+      - class: small
+        bootstrap:
+          templateRef:
+            apiVersion: bootstrap.cluster.x-k8s.io/v1beta2
+            kind: KubeadmConfigTemplate
+            name: worker-external
+        infrastructure:
+          templateRef:
+            apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
+            kind: KubevirtMachineTemplate
+            name: worker-external
+---
+apiVersion: cluster.x-k8s.io/v1beta2
+kind: Cluster
+metadata:
+  name: demo-external
+  namespace: default
+spec:
+  topology:
+    classRef:
+      name: kubevirt-kamaji-kubeadm-external
+      namespace: default
+    version: v1.34.0
+    controlPlane:
+      replicas: 1
+    workers:
+      machineDeployments:
+      - class: small
+        name: md-small
+        replicas: 1

config/samples/kamaji_v1alpha1_kubeconfiggenerator.yaml

@@ -0,0 +1,21 @@
+apiVersion: kamaji.clastix.io/v1alpha1
+kind: KubeconfigGenerator
+metadata:
+  name: tenant
+spec:
+  controlPlaneEndpointFrom: admin.conf
+  groups:
+    - stringValue: custom.group.tld
+    - fromDefinition: metadata.namespace
+  namespaceSelector:
+    matchExpressions:
+      - key: kubernetes.io/metadata.name
+        operator: Exists
+        values: []
+  tenantControlPlaneSelector:
+    matchExpressions:
+      - key: tenant.clastix.io
+        operator: Exists
+        values: []
+  user:
+    fromDefinition: metadata.name

config/samples/kamaji_v1alpha1_tenantcontrolplane.yaml

@@ -1,9 +1,9 @@
 apiVersion: kamaji.clastix.io/v1alpha1
 kind: TenantControlPlane
 metadata:
-  name: k8s-130
+  name: k8s-133
   labels:
-    tenant.clastix.io: k8s-130
+    tenant.clastix.io: k8s-133
 spec:
   controlPlane:
     deployment:
@@ -11,9 +11,17 @@ spec:
     service:
       serviceType: LoadBalancer
   kubernetes:
-    version: "v1.30.0"
+    version: "v1.33.0"
     kubelet:
-      cgroupfs: systemd
+      configurationJSONPatches:
+        - op: add
+          path: /featureGates
+          value:
+            KubeletCrashLoopBackOffMax: false
+            KubeletEnsureSecretPulledImages: false
+        - op: replace
+          path: /cgroupDriver
+          value: systemd
   networkProfile:
     port: 6443
   addons:
@@ -22,3 +30,5 @@ spec:
     konnectivity:
       server:
         port: 8132
+      agent:
+        mode: DaemonSet

config/samples/kamaji_v1alpha1_tenantcontrolplane_gateway.yaml

@@ -0,0 +1,81 @@
+# Copyright 2022 Clastix Labs
+# SPDX-License-Identifier: Apache-2.0
+
+# This example demonstrates how to configure Gateway API support for a Tenant Control Plane.
+# 
+# Prerequisites:
+# 1. Gateway API CRDs must be installed (GatewayClass, Gateway, TLSRoute)
+# 2. A Gateway resource must exist with listeners for ports 6443 and 8132
+# 3. DNS(or worker nodes hosts entries) must be configured to resolve the hostname to the Gateway's external address
+#
+# Example GatewayClass and Gateway configuration:
+#
+# apiVersion: gateway.networking.k8s.io/v1
+# kind: GatewayClass
+# metadata:
+#   name: envoy-gw-class
+# spec:
+#   controllerName: gateway.envoyproxy.io/gatewayclass-controller
+# ---
+# apiVersion: gateway.networking.k8s.io/v1
+# kind: Gateway
+# metadata:
+#   name: gateway
+#   namespace: default
+# spec:
+#   gatewayClassName: envoy-gw-class
+#   listeners:
+#   - allowedRoutes:
+#       kinds:
+#       - group: gateway.networking.k8s.io
+#         kind: TLSRoute
+#       namespaces:
+#         from: All
+#     hostname: '*.cluster.dev'
+#     name: kube-apiserver
+#     port: 6443
+#     protocol: TLS
+#     tls:
+#       mode: Passthrough
+#   - allowedRoutes:
+#       kinds:
+#       - group: gateway.networking.k8s.io
+#         kind: TLSRoute
+#       namespaces:
+#         from: All
+#     hostname: '*.cluster.dev'
+#     name: konnectivity-server
+#     port: 8132
+#     protocol: TLS
+#     tls:
+#       mode: Passthrough
+
+apiVersion: kamaji.clastix.io/v1alpha1
+kind: TenantControlPlane
+metadata:
+  name: demo-tcp-1
+spec:
+  addons:
+    coreDNS: {}
+    kubeProxy: {}
+    konnectivity: {}
+  dataStore: default
+  controlPlane:
+    gateway:
+      hostname: "c11.cluster.dev"  # worker nodes or kubectl clients must be able to resolve this hostname to the Gateway's external address.
+      parentRefs:
+      - name: gateway
+        namespace: default
+    deployment:
+      replicas: 1
+    service:
+      serviceType: ClusterIP
+  kubernetes:
+    version: v1.32.0
+    kubelet:
+      cgroupfs: systemd
+  networkProfile:
+    port: 6443
+    certSANs:
+    - "c11.cluster.dev"
+

config/samples/kamaji_v1alpha1_tenantcontrolplane_konnectivity_hostnetwork.yaml

@@ -0,0 +1,36 @@
+apiVersion: kamaji.clastix.io/v1alpha1
+kind: TenantControlPlane
+metadata:
+  name: example-hostnetwork-tcp
+  namespace: tenant-system
+spec:
+  controlPlane:
+    deployment:
+      replicas: 2
+    service:
+      serviceType: LoadBalancer
+  kubernetes:
+    version: v1.29.0
+    kubelet:
+      cgroupfs: systemd
+      preferredAddressTypes: ["InternalIP", "ExternalIP"]
+  networkProfile:
+    address: "10.0.0.100"
+    port: 6443
+    serviceCidr: "10.96.0.0/16"
+    podCidr: "10.244.0.0/16"
+  addons:
+    coreDNS: {}
+    konnectivity:
+      server:
+        port: 8132
+      agent:
+        hostNetwork: true
+        tolerations:
+          - key: "CriticalAddonsOnly"
+            operator: "Exists"
+          - key: "node.kubernetes.io/not-ready"
+            operator: "Exists"
+            effect: "NoExecute"
+            tolerationSeconds: 300
+    kubeProxy: {}

controllers/cert_channel.go

@@ -1,10 +0,0 @@
-// Copyright 2022 Clastix Labs
-// SPDX-License-Identifier: Apache-2.0
-
-package controllers
-
-import (
-	"sigs.k8s.io/controller-runtime/pkg/event"
-)
-
-type CertificateChannel chan event.GenericEvent

controllers/certificate_lifecycle_controller.go

@@ -9,7 +9,6 @@ import (
 	"fmt"
 	"time"
 
-	"github.com/pkg/errors"
 	corev1 "k8s.io/api/core/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -24,14 +23,16 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 
 	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+	"github.com/clastix/kamaji/controllers/utils"
 	"github.com/clastix/kamaji/internal/constants"
 	"github.com/clastix/kamaji/internal/crypto"
 	"github.com/clastix/kamaji/internal/utilities"
 )
 
 type CertificateLifecycle struct {
-	Channel  CertificateChannel
-	Deadline time.Duration
+	Channel   chan event.GenericEvent
+	Deadline  time.Duration
+	EnqueueFn func(secret *corev1.Secret)
 
 	client client.Client
 }
@@ -41,19 +42,25 @@ func (s *CertificateLifecycle) Reconcile(ctx context.Context, request reconcile.
 
 	logger.Info("starting CertificateLifecycle handling")
 
-	secret := corev1.Secret{}
-	err := s.client.Get(ctx, request.NamespacedName, &secret)
-	if k8serrors.IsNotFound(err) {
-		logger.Info("resource have been deleted, skipping")
+	var secret corev1.Secret
+	if err := s.client.Get(ctx, request.NamespacedName, &secret); err != nil {
+		if k8serrors.IsNotFound(err) {
+			logger.Info("resource may have been deleted, skipping")
+
+			return reconcile.Result{}, nil
+		}
 
-		return reconcile.Result{}, nil
-	}
-	if err != nil {
 		logger.Error(err, "cannot retrieve the required resource")
 
 		return reconcile.Result{}, err
 	}
 
+	if utils.IsPaused(&secret) {
+		logger.Info("paused reconciliation, no further actions")
+
+		return reconcile.Result{}, nil
+	}
+
 	checkType, ok := secret.GetLabels()[constants.ControllerLabelResource]
 	if !ok {
 		logger.Info("missing controller label, shouldn't happen")
@@ -62,14 +69,15 @@ func (s *CertificateLifecycle) Reconcile(ctx context.Context, request reconcile.
 	}
 
 	var crt *x509.Certificate
+	var err error
 
 	switch checkType {
-	case "x509":
+	case utilities.CertificateX509Label:
 		crt, err = s.extractCertificateFromBareSecret(secret)
-	case "kubeconfig":
+	case utilities.CertificateKubeconfigLabel:
 		crt, err = s.extractCertificateFromKubeconfig(secret)
 	default:
-		err = fmt.Errorf("unsupported strategy, %s", checkType)
+		return reconcile.Result{}, fmt.Errorf("unsupported strategy, %q", checkType)
 	}
 
 	if err != nil {
@@ -83,12 +91,7 @@ func (s *CertificateLifecycle) Reconcile(ctx context.Context, request reconcile.
 	if deadline.After(crt.NotAfter) {
 		logger.Info("certificate near expiration, must be rotated")
 
-		s.Channel <- event.GenericEvent{Object: &kamajiv1alpha1.TenantControlPlane{
-			ObjectMeta: metav1.ObjectMeta{
-				Name:      secret.GetOwnerReferences()[0].Name,
-				Namespace: secret.Namespace,
-			},
-		}}
+		s.EnqueueFn(&secret)
 
 		logger.Info("certificate rotation triggered")
 
@@ -99,7 +102,36 @@ func (s *CertificateLifecycle) Reconcile(ctx context.Context, request reconcile.
 
 	logger.Info("certificate is still valid, enqueuing back", "after", after.String())
 
-	return reconcile.Result{Requeue: true, RequeueAfter: after}, nil
+	return reconcile.Result{RequeueAfter: after}, nil
+}
+
+func (s *CertificateLifecycle) EnqueueForTenantControlPlane(secret *corev1.Secret) {
+	for _, or := range secret.GetOwnerReferences() {
+		if or.Kind != "TenantControlPlane" {
+			continue
+		}
+
+		s.Channel <- event.GenericEvent{Object: &kamajiv1alpha1.TenantControlPlane{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      or.Name,
+				Namespace: secret.Namespace,
+			},
+		}}
+	}
+}
+
+func (s *CertificateLifecycle) EnqueueForKubeconfigGenerator(secret *corev1.Secret) {
+	for _, or := range secret.GetOwnerReferences() {
+		if or.Kind != "KubeconfigGenerator" {
+			continue
+		}
+
+		s.Channel <- event.GenericEvent{Object: &kamajiv1alpha1.TenantControlPlane{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: or.Name,
+			},
+		}}
+	}
 }
 
 func (s *CertificateLifecycle) extractCertificateFromBareSecret(secret corev1.Secret) (*x509.Certificate, error) {
@@ -135,7 +167,7 @@ func (s *CertificateLifecycle) extractCertificateFromKubeconfig(secret corev1.Se
 
 	crt, err := crypto.ParseCertificateBytes(kc.AuthInfos[0].AuthInfo.ClientCertificateData)
 	if err != nil {
-		return nil, errors.Wrap(err, "cannot parse kubeconfig certificate bytes")
+		return nil, fmt.Errorf("cannot parse kubeconfig certificate bytes: %w", err)
 	}
 
 	return crt, nil
@@ -144,7 +176,7 @@ func (s *CertificateLifecycle) extractCertificateFromKubeconfig(secret corev1.Se
 func (s *CertificateLifecycle) SetupWithManager(mgr controllerruntime.Manager) error {
 	s.client = mgr.GetClient()
 
-	supportedStrategies := sets.New[string]("x509", "kubeconfig")
+	supportedStrategies := sets.New[string](utilities.CertificateX509Label, utilities.CertificateKubeconfigLabel)
 
 	return controllerruntime.NewControllerManagedBy(mgr).
 		For(&corev1.Secret{}, builder.WithPredicates(predicate.NewPredicateFuncs(func(object client.Object) bool {

controllers/datastore_controller.go

@@ -5,81 +5,236 @@ package controllers
 
 import (
 	"context"
+	"fmt"
 
+	corev1 "k8s.io/api/core/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/api/meta"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/fields"
 	k8stypes "k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/client-go/util/workqueue"
 	controllerruntime "sigs.k8s.io/controller-runtime"
-	"sigs.k8s.io/controller-runtime/pkg/builder"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 	"sigs.k8s.io/controller-runtime/pkg/event"
 	"sigs.k8s.io/controller-runtime/pkg/handler"
 	"sigs.k8s.io/controller-runtime/pkg/log"
-	"sigs.k8s.io/controller-runtime/pkg/predicate"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 
 	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+	"github.com/clastix/kamaji/controllers/utils"
 )
 
 type DataStore struct {
 	Client client.Client
 	// TenantControlPlaneTrigger is the channel used to communicate across the controllers:
-	// if a Data Source is updated we have to be sure that the reconciliation of the certificates content
+	// if a Data Source is updated, we have to be sure that the reconciliation of the certificates content
 	// for each Tenant Control Plane is put in place properly.
-	TenantControlPlaneTrigger TenantControlPlaneChannel
+	TenantControlPlaneTrigger chan event.GenericEvent
 }
 
 //+kubebuilder:rbac:groups=kamaji.clastix.io,resources=datastores,verbs=get;list;watch;create;update;patch;delete
 //+kubebuilder:rbac:groups=kamaji.clastix.io,resources=datastores/status,verbs=get;update;patch
 
 func (r *DataStore) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) {
-	log := log.FromContext(ctx)
+	var err error
 
-	ds := &kamajiv1alpha1.DataStore{}
-	err := r.Client.Get(ctx, request.NamespacedName, ds)
-	if k8serrors.IsNotFound(err) {
-		log.Info("resource have been deleted, skipping")
+	logger := log.FromContext(ctx)
+
+	var ds kamajiv1alpha1.DataStore
+	if dsErr := r.Client.Get(ctx, request.NamespacedName, &ds); dsErr != nil {
+		if k8serrors.IsNotFound(dsErr) {
+			logger.Info("resource may have been deleted, skipping")
+
+			return reconcile.Result{}, nil
+		}
+
+		logger.Error(dsErr, "cannot retrieve the required resource")
+
+		return reconcile.Result{}, dsErr
+	}
+
+	if utils.IsPaused(&ds) {
+		logger.Info("paused reconciliation, no further actions")
 
 		return reconcile.Result{}, nil
 	}
-	if err != nil {
-		log.Error(err, "cannot retrieve the required resource")
 
-		return reconcile.Result{}, err
+	if ds.GetDeletionTimestamp() == nil && !controllerutil.ContainsFinalizer(&ds, kamajiv1alpha1.DataStoreTCPFinalizer) {
+		logger.Info("missing finalizer, adding it")
+
+		ds.SetFinalizers(append(ds.GetFinalizers(), kamajiv1alpha1.DataStoreTCPFinalizer))
+		if uErr := r.Client.Update(ctx, &ds); uErr != nil {
+			return reconcile.Result{}, uErr
+		}
+
+		return reconcile.Result{}, nil
 	}
+	// Extracting the list of TenantControlPlane objects referenced by the given DataStore:
+	// this data is used to reference these in the Status, as well as propagating changes
+	// that would be required, such as changing TLS Configuration, or Basic Auth.
+	var tcpList kamajiv1alpha1.TenantControlPlaneList
 
-	tcpList := kamajiv1alpha1.TenantControlPlaneList{}
-
-	if err := r.Client.List(ctx, &tcpList, client.MatchingFieldsSelector{
+	if lErr := r.Client.List(ctx, &tcpList, client.MatchingFieldsSelector{
 		Selector: fields.OneTermEqualSelector(kamajiv1alpha1.TenantControlPlaneUsedDataStoreKey, ds.GetName()),
-	}); err != nil {
-		log.Error(err, "cannot retrieve list of the Tenant Control Plane using the following instance")
-
-		return reconcile.Result{}, err
+	}); lErr != nil {
+		return reconcile.Result{}, fmt.Errorf("cannot retrieve list of the Tenant Control Plane using the following instance: %w", lErr)
 	}
-	// Updating the status with the list of Tenant Control Plane using the following Data Source
+
 	tcpSets := sets.NewString()
+
 	for _, tcp := range tcpList.Items {
 		tcpSets.Insert(getNamespacedName(tcp.GetNamespace(), tcp.GetName()).String())
 	}
 
 	ds.Status.UsedBy = tcpSets.List()
+	// Performing the status update only at the end of the reconciliation loop:
+	// this is performed in defer to avoid duplication of code,
+	// and triggering a reconciliation of depending on TenantControlPlanes only if the update was successful.
+	defer func() {
+		if meta.IsStatusConditionTrue(ds.Status.Conditions, kamajiv1alpha1.DataStoreConditionAllowedDeletionType) {
+			logger.Info("removing finalizer upon true condition")
 
-	if err := r.Client.Status().Update(ctx, ds); err != nil {
-		log.Error(err, "cannot update the status for the given instance")
+			controllerutil.RemoveFinalizer(&ds, kamajiv1alpha1.DataStoreTCPFinalizer)
+			if uErr := r.Client.Update(ctx, &ds); uErr != nil {
+				logger.Error(uErr, "cannot update object")
 
-		return reconcile.Result{}, err
-	}
-	// Triggering the reconciliation of the Tenant Control Plane upon a Secret change
-	for _, i := range tcpList.Items {
-		tcp := i
+				return
+			}
 
-		r.TenantControlPlaneTrigger <- event.GenericEvent{Object: &tcp}
+			logger.Info("finalizer removed successfully")
+
+			return
+		}
+
+		ds.Status.ObservedGeneration = ds.Generation
+		ds.Status.Ready = meta.IsStatusConditionTrue(ds.Status.Conditions, kamajiv1alpha1.DataStoreConditionValidType)
+
+		if err = r.Client.Status().Update(ctx, &ds); err != nil {
+			logger.Error(err, "cannot update the status for the given instance")
+
+			return
+		}
+
+		if !ds.Status.Ready {
+			logger.Info("skipping triggering, DataStore is not ready")
+
+			return
+		}
+
+		logger.Info("triggering cascading reconciliation for TenantControlPlanes")
+
+		for _, tcp := range tcpList.Items {
+			var shrunkTCP kamajiv1alpha1.TenantControlPlane
+
+			shrunkTCP.Name = tcp.Name
+			shrunkTCP.Namespace = tcp.Namespace
+
+			go utils.TriggerChannel(ctx, r.TenantControlPlaneTrigger, shrunkTCP)
+		}
+	}()
+
+	if ds.GetDeletionTimestamp() != nil {
+		if len(tcpList.Items) > 0 {
+			logger.Info("deletion is blocked due to DataStore still being referenced")
+
+			meta.SetStatusCondition(&ds.Status.Conditions, metav1.Condition{
+				Type:               kamajiv1alpha1.DataStoreConditionAllowedDeletionType,
+				Status:             metav1.ConditionFalse,
+				ObservedGeneration: ds.Generation,
+				Reason:             "DataStoreStillUsed",
+				Message:            "The DataStore is still used and referenced by one (or more) TenantControlPlane objects.",
+			})
+
+			return reconcile.Result{}, nil
+		}
+
+		if meta.IsStatusConditionFalse(ds.Status.Conditions, kamajiv1alpha1.DataStoreConditionAllowedDeletionType) {
+			logger.Info("DataStore is not used by any TenantControlPlane object")
+
+			meta.SetStatusCondition(&ds.Status.Conditions, metav1.Condition{
+				Type:               kamajiv1alpha1.DataStoreConditionAllowedDeletionType,
+				Status:             metav1.ConditionTrue,
+				ObservedGeneration: ds.Generation,
+				Reason:             "DataStoreUnused",
+				Message:            "",
+			})
+
+			return reconcile.Result{}, nil
+		}
+
+		logger.Info("DataStore can be safely deleted")
+
+		return reconcile.Result{}, nil
 	}
 
-	return reconcile.Result{}, nil
+	if exists := meta.FindStatusCondition(ds.Status.Conditions, kamajiv1alpha1.DataStoreConditionValidType); exists == nil {
+		logger.Info("missing starting condition")
+
+		meta.SetStatusCondition(&ds.Status.Conditions, metav1.Condition{
+			Type:               kamajiv1alpha1.DataStoreConditionValidType,
+			Status:             metav1.ConditionUnknown,
+			ObservedGeneration: ds.Generation,
+			Reason:             "MissingCondition",
+			Message:            "Controller will process the validation.",
+		})
+
+		if sErr := r.Client.Status().Update(ctx, &ds); sErr != nil {
+			return reconcile.Result{}, fmt.Errorf("cannot update the status for the given instance: %w", sErr)
+		}
+
+		return reconcile.Result{}, nil
+	}
+
+	if ds.Spec.BasicAuth != nil {
+		logger.Info("validating basic authentication")
+
+		if vErr := r.validateBasicAuth(ctx, ds); vErr != nil {
+			meta.SetStatusCondition(&ds.Status.Conditions, metav1.Condition{
+				Type:               kamajiv1alpha1.DataStoreConditionValidType,
+				Status:             metav1.ConditionFalse,
+				ObservedGeneration: ds.Generation,
+				Reason:             "BasicAuthValidationFailed",
+				Message:            vErr.Error(),
+			})
+
+			logger.Info("invalid basic authentication")
+
+			return reconcile.Result{}, nil
+		}
+
+		logger.Info("basic authentication is valid")
+	}
+
+	logger.Info("validating TLS configuration")
+
+	if vErr := r.validateTLSConfig(ctx, ds); vErr != nil {
+		meta.SetStatusCondition(&ds.Status.Conditions, metav1.Condition{
+			Type:               kamajiv1alpha1.DataStoreConditionValidType,
+			Status:             metav1.ConditionFalse,
+			ObservedGeneration: ds.Generation,
+			Reason:             "TLSConfigurationValidationFailed",
+			Message:            vErr.Error(),
+		})
+
+		logger.Info("invalid TLS configuration")
+
+		return reconcile.Result{}, nil
+	}
+
+	logger.Info("TLS configuration is valid")
+
+	meta.SetStatusCondition(&ds.Status.Conditions, metav1.Condition{
+		Type:               kamajiv1alpha1.DataStoreConditionValidType,
+		Status:             metav1.ConditionTrue,
+		ObservedGeneration: ds.Status.ObservedGeneration,
+		Reason:             "DataStoreIsValid",
+		Message:            "",
+	})
+
+	return reconcile.Result{}, err
 }
 
 func (r *DataStore) SetupWithManager(mgr controllerruntime.Manager) error {
@@ -94,9 +249,7 @@ func (r *DataStore) SetupWithManager(mgr controllerruntime.Manager) error {
 	}
 	//nolint:forcetypeassert
 	return controllerruntime.NewControllerManagedBy(mgr).
-		For(&kamajiv1alpha1.DataStore{}, builder.WithPredicates(
-			predicate.ResourceVersionChangedPredicate{},
-		)).
+		For(&kamajiv1alpha1.DataStore{}).
 		Watches(&kamajiv1alpha1.TenantControlPlane{}, handler.Funcs{
 			CreateFunc: func(_ context.Context, createEvent event.TypedCreateEvent[client.Object], w workqueue.TypedRateLimitingInterface[reconcile.Request]) {
 				enqueueFn(createEvent.Object.(*kamajiv1alpha1.TenantControlPlane), w)
@@ -111,3 +264,76 @@ func (r *DataStore) SetupWithManager(mgr controllerruntime.Manager) error {
 		}).
 		Complete(r)
 }
+
+func (r *DataStore) validateBasicAuth(ctx context.Context, ds kamajiv1alpha1.DataStore) error {
+	if err := r.validateContentReference(ctx, ds.Spec.BasicAuth.Password); err != nil {
+		return fmt.Errorf("basic-auth password is not valid, %w", err)
+	}
+
+	if err := r.validateContentReference(ctx, ds.Spec.BasicAuth.Username); err != nil {
+		return fmt.Errorf("basic-auth username is not valid, %w", err)
+	}
+
+	return nil
+}
+
+func (r *DataStore) validateTLSConfig(ctx context.Context, ds kamajiv1alpha1.DataStore) error {
+	if ds.Spec.TLSConfig == nil && ds.Spec.Driver != kamajiv1alpha1.EtcdDriver {
+		return nil
+	}
+
+	if err := r.validateContentReference(ctx, ds.Spec.TLSConfig.CertificateAuthority.Certificate); err != nil {
+		return fmt.Errorf("CA certificate is not valid, %w", err)
+	}
+
+	if ds.Spec.Driver == kamajiv1alpha1.EtcdDriver {
+		if ds.Spec.TLSConfig.CertificateAuthority.PrivateKey == nil {
+			return fmt.Errorf("CA private key is required when using the etcd driver")
+		}
+
+		if ds.Spec.TLSConfig.ClientCertificate == nil {
+			return fmt.Errorf("client certificate is required when using the etcd driver")
+		}
+	}
+
+	if ds.Spec.TLSConfig.CertificateAuthority.PrivateKey != nil {
+		if err := r.validateContentReference(ctx, *ds.Spec.TLSConfig.CertificateAuthority.PrivateKey); err != nil {
+			return fmt.Errorf("CA private key is not valid, %w", err)
+		}
+	}
+
+	if ds.Spec.TLSConfig.ClientCertificate != nil {
+		if err := r.validateContentReference(ctx, ds.Spec.TLSConfig.ClientCertificate.Certificate); err != nil {
+			return fmt.Errorf("client certificate is not valid, %w", err)
+		}
+
+		if err := r.validateContentReference(ctx, ds.Spec.TLSConfig.ClientCertificate.PrivateKey); err != nil {
+			return fmt.Errorf("client private key is not valid, %w", err)
+		}
+	}
+
+	return nil
+}
+
+func (r *DataStore) validateContentReference(ctx context.Context, ref kamajiv1alpha1.ContentRef) error {
+	switch {
+	case len(ref.Content) > 0:
+		return nil
+	case ref.SecretRef == nil:
+		return fmt.Errorf("the Secret reference is mandatory when bare content is not specified")
+	case len(ref.SecretRef.SecretReference.Name) == 0:
+		return fmt.Errorf("the Secret reference name is mandatory")
+	case len(ref.SecretRef.SecretReference.Namespace) == 0:
+		return fmt.Errorf("the Secret reference namespace is mandatory")
+	}
+
+	if err := r.Client.Get(ctx, k8stypes.NamespacedName{Name: ref.SecretRef.SecretReference.Name, Namespace: ref.SecretRef.SecretReference.Namespace}, &corev1.Secret{}); err != nil {
+		if k8serrors.IsNotFound(err) {
+			return fmt.Errorf("secret %s/%s is not found", ref.SecretRef.SecretReference.Namespace, ref.SecretRef.SecretReference.Name)
+		}
+
+		return err
+	}
+
+	return nil
+}

controllers/kubeconfiggenerator_controller.go

@@ -0,0 +1,444 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package controllers
+
+import (
+	"bytes"
+	"context"
+	"crypto/x509"
+	"fmt"
+	"sort"
+	"strings"
+	"time"
+
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/sets"
+	clientcmdapiv1 "k8s.io/client-go/tools/clientcmd/api/v1"
+	certutil "k8s.io/client-go/util/cert"
+	"k8s.io/client-go/util/keyutil"
+	"k8s.io/client-go/util/workqueue"
+	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
+	"k8s.io/kubernetes/cmd/kubeadm/app/util"
+	"k8s.io/kubernetes/cmd/kubeadm/app/util/pkiutil"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+	"sigs.k8s.io/controller-runtime/pkg/event"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
+	"sigs.k8s.io/controller-runtime/pkg/log"
+	"sigs.k8s.io/controller-runtime/pkg/manager"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+	"sigs.k8s.io/controller-runtime/pkg/source"
+
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+	"github.com/clastix/kamaji/controllers/utils"
+	"github.com/clastix/kamaji/internal/constants"
+	"github.com/clastix/kamaji/internal/crypto"
+	"github.com/clastix/kamaji/internal/resources"
+	"github.com/clastix/kamaji/internal/utilities"
+)
+
+type KubeconfigGeneratorReconciler struct {
+	Client            client.Client
+	NotValidThreshold time.Duration
+	CertificateChan   chan event.GenericEvent
+}
+
+//+kubebuilder:rbac:groups="",resources=namespaces,verbs=get;list;watch
+//+kubebuilder:rbac:groups=kamaji.clastix.io,resources=kubeconfiggenerators,verbs=get;list;watch;create;update;patch
+//+kubebuilder:rbac:groups=kamaji.clastix.io,resources=kubeconfiggenerators/status,verbs=get;update;patch
+//+kubebuilder:rbac:groups=kamaji.clastix.io,resources=kubeconfiggenerators/finalizers,verbs=update
+
+func (r *KubeconfigGeneratorReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	logger := log.FromContext(ctx)
+
+	logger.Info("reconciling resource")
+
+	var generator kamajiv1alpha1.KubeconfigGenerator
+	if err := r.Client.Get(ctx, req.NamespacedName, &generator); err != nil {
+		if apierrors.IsNotFound(err) {
+			logger.Info("resource may have been deleted, skipping")
+
+			return ctrl.Result{}, nil
+		}
+
+		logger.Error(err, "cannot retrieve the required resource")
+
+		return ctrl.Result{}, err
+	}
+
+	if utils.IsPaused(&generator) {
+		logger.Info("paused reconciliation, no further actions")
+
+		return ctrl.Result{}, nil
+	}
+
+	status, err := r.handle(ctx, &generator)
+	if err != nil {
+		logger.Error(err, "cannot handle the request")
+
+		return ctrl.Result{}, err
+	}
+
+	generator.Status = status
+	generator.Status.ObservedGeneration = generator.Generation
+
+	if statusErr := r.Client.Status().Update(ctx, &generator); statusErr != nil {
+		logger.Error(statusErr, "cannot update resource status")
+
+		return ctrl.Result{}, statusErr
+	}
+
+	logger.Info("reconciling completed")
+
+	return ctrl.Result{}, nil
+}
+
+func (r *KubeconfigGeneratorReconciler) handle(ctx context.Context, generator *kamajiv1alpha1.KubeconfigGenerator) (kamajiv1alpha1.KubeconfigGeneratorStatus, error) {
+	nsSelector, nsErr := metav1.LabelSelectorAsSelector(&generator.Spec.NamespaceSelector)
+	if nsErr != nil {
+		return kamajiv1alpha1.KubeconfigGeneratorStatus{}, fmt.Errorf("NamespaceSelector contains an error: %w", nsErr)
+	}
+
+	var namespaceList corev1.NamespaceList
+	if err := r.Client.List(ctx, &namespaceList, &client.ListOptions{LabelSelector: nsSelector}); err != nil {
+		return kamajiv1alpha1.KubeconfigGeneratorStatus{}, fmt.Errorf("cannot filter Namespace objects using provided selector: %w", err)
+	}
+
+	var targets []kamajiv1alpha1.TenantControlPlane
+
+	for _, ns := range namespaceList.Items {
+		tcpSelector, tcpErr := metav1.LabelSelectorAsSelector(&generator.Spec.TenantControlPlaneSelector)
+		if tcpErr != nil {
+			return kamajiv1alpha1.KubeconfigGeneratorStatus{}, fmt.Errorf("TenantControlPlaneSelector contains an error: %w", tcpErr)
+		}
+
+		var tcpList kamajiv1alpha1.TenantControlPlaneList
+		if err := r.Client.List(ctx, &tcpList, &client.ListOptions{Namespace: ns.GetName(), LabelSelector: tcpSelector}); err != nil {
+			return kamajiv1alpha1.KubeconfigGeneratorStatus{}, fmt.Errorf("cannot filter TenantControlPlane objects using provided selector: %w", err)
+		}
+
+		targets = append(targets, tcpList.Items...)
+	}
+
+	sort.Slice(targets, func(i, j int) bool {
+		return client.ObjectKeyFromObject(&targets[i]).String() < client.ObjectKeyFromObject(&targets[j]).String()
+	})
+
+	status := kamajiv1alpha1.KubeconfigGeneratorStatus{
+		Resources:          len(targets),
+		AvailableResources: len(targets),
+	}
+
+	for _, tcp := range targets {
+		if err := r.process(ctx, generator, tcp); err != nil {
+			status.Errors = append(status.Errors, *err)
+			status.AvailableResources--
+		}
+	}
+
+	return status, nil
+}
+
+func (r *KubeconfigGeneratorReconciler) process(ctx context.Context, generator *kamajiv1alpha1.KubeconfigGenerator, tcp kamajiv1alpha1.TenantControlPlane) *kamajiv1alpha1.KubeconfigGeneratorStatusError {
+	statusErr := kamajiv1alpha1.KubeconfigGeneratorStatusError{
+		Resource: client.ObjectKeyFromObject(&tcp).String(),
+	}
+
+	var adminSecret corev1.Secret
+
+	if tcp.Status.KubeConfig.Admin.SecretName == "" {
+		statusErr.Message = "the admin kubeconfig is not yet generated"
+
+		return &statusErr
+	}
+
+	if err := r.Client.Get(ctx, types.NamespacedName{Name: tcp.Status.KubeConfig.Admin.SecretName, Namespace: tcp.GetNamespace()}, &adminSecret); err != nil {
+		statusErr.Message = fmt.Sprintf("an error occurred retrieving the admin Kubeconfig: %s", err.Error())
+
+		return &statusErr
+	}
+
+	kubeconfigTmpl, kcErr := utilities.DecodeKubeconfig(adminSecret, generator.Spec.ControlPlaneEndpointFrom)
+	if kcErr != nil {
+		statusErr.Message = fmt.Sprintf("unable to decode Kubeconfig template: %s", kcErr.Error())
+
+		return &statusErr
+	}
+
+	uMap, uErr := runtime.DefaultUnstructuredConverter.ToUnstructured(&tcp)
+	if uErr != nil {
+		statusErr.Message = fmt.Sprintf("cannot convert the resource to a map: %s", uErr)
+
+		return &statusErr
+	}
+
+	var user string
+	groups := sets.New[string]()
+
+	for _, group := range generator.Spec.Groups {
+		switch {
+		case group.StringValue != "":
+			groups.Insert(group.StringValue)
+		case group.FromDefinition != "":
+			v, ok, vErr := unstructured.NestedString(uMap, strings.Split(group.FromDefinition, ".")...)
+			switch {
+			case vErr != nil:
+				statusErr.Message = fmt.Sprintf("cannot run NestedString %q due to an error: %s", group.FromDefinition, vErr.Error())
+
+				return &statusErr
+			case !ok:
+				statusErr.Message = fmt.Sprintf("provided dot notation %q is not found", group.FromDefinition)
+
+				return &statusErr
+			default:
+				groups.Insert(v)
+			}
+		default:
+			statusErr.Message = "at least a StringValue or FromDefinition Group value must be provided"
+
+			return &statusErr
+		}
+	}
+
+	switch {
+	case generator.Spec.User.StringValue != "":
+		user = generator.Spec.User.StringValue
+	case generator.Spec.User.FromDefinition != "":
+		v, ok, vErr := unstructured.NestedString(uMap, strings.Split(generator.Spec.User.FromDefinition, ".")...)
+
+		switch {
+		case vErr != nil:
+			statusErr.Message = fmt.Sprintf("cannot run NestedString %q due to an error: %s", generator.Spec.User.FromDefinition, vErr.Error())
+
+			return &statusErr
+		case !ok:
+			statusErr.Message = fmt.Sprintf("provided dot notation %q is not found", generator.Spec.User.FromDefinition)
+
+			return &statusErr
+		default:
+			user = v
+		}
+	default:
+		statusErr.Message = "at least a StringValue or FromDefinition for the user field must be provided"
+
+		return &statusErr
+	}
+
+	var resultSecret corev1.Secret
+	resultSecret.SetName(tcp.Name + "-" + generator.Name)
+	resultSecret.SetNamespace(tcp.Namespace)
+
+	objectKey := client.ObjectKeyFromObject(&resultSecret)
+
+	if err := r.Client.Get(ctx, objectKey, &resultSecret); err != nil {
+		if !apierrors.IsNotFound(err) {
+			statusErr.Message = fmt.Sprintf("the secret %q cannot be generated", objectKey.String())
+
+			return &statusErr
+		}
+
+		if generateErr := r.generate(ctx, generator, &resultSecret, kubeconfigTmpl, &tcp, groups, user); generateErr != nil {
+			statusErr.Message = fmt.Sprintf("an error occurred generating the %q Secret: %s", objectKey.String(), generateErr.Error())
+
+			return &statusErr
+		}
+
+		return nil
+	}
+
+	isValid, validateErr := r.isValid(&resultSecret, kubeconfigTmpl, groups, user)
+	switch {
+	case !isValid:
+		if generateErr := r.generate(ctx, generator, &resultSecret, kubeconfigTmpl, &tcp, groups, user); generateErr != nil {
+			statusErr.Message = fmt.Sprintf("an error occurred regenerating the %q Secret: %s", objectKey.String(), generateErr.Error())
+
+			return &statusErr
+		}
+
+		return nil
+	case validateErr != nil:
+		statusErr.Message = fmt.Sprintf("an error occurred checking validation for %q Secret: %s", objectKey.String(), validateErr.Error())
+
+		return &statusErr
+	default:
+		return nil
+	}
+}
+
+func (r *KubeconfigGeneratorReconciler) generate(ctx context.Context, generator *kamajiv1alpha1.KubeconfigGenerator, secret *corev1.Secret, tmpl *clientcmdapiv1.Config, tcp *kamajiv1alpha1.TenantControlPlane, groups sets.Set[string], user string) error {
+	_, config, err := resources.GetKubeadmManifestDeps(ctx, r.Client, tcp)
+	if err != nil {
+		return err
+	}
+
+	clientCertConfig := pkiutil.CertConfig{
+		Config: certutil.Config{
+			CommonName:   user,
+			Organization: groups.UnsortedList(),
+			Usages:       []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
+		},
+		NotAfter:            util.StartTimeUTC().Add(kubeadmconstants.CertificateValidityPeriod),
+		EncryptionAlgorithm: config.InitConfiguration.ClusterConfiguration.EncryptionAlgorithmType(),
+	}
+
+	var caSecret corev1.Secret
+	if caErr := r.Client.Get(ctx, types.NamespacedName{Namespace: tcp.Namespace, Name: tcp.Status.Certificates.CA.SecretName}, &caSecret); caErr != nil {
+		return fmt.Errorf("cannot retrieve Certificate Authority: %w", caErr)
+	}
+
+	caCert, crtErr := crypto.ParseCertificateBytes(caSecret.Data[kubeadmconstants.CACertName])
+	if crtErr != nil {
+		return fmt.Errorf("cannot parse Certificate Authority certificate: %w", crtErr)
+	}
+
+	caKey, keyErr := crypto.ParsePrivateKeyBytes(caSecret.Data[kubeadmconstants.CAKeyName])
+	if keyErr != nil {
+		return fmt.Errorf("cannot parse Certificate Authority key: %w", keyErr)
+	}
+
+	clientCert, clientKey, err := pkiutil.NewCertAndKey(caCert, caKey, &clientCertConfig)
+
+	contextUserName := generator.Name
+
+	for name := range tmpl.AuthInfos {
+		tmpl.AuthInfos[name].Name = contextUserName
+		tmpl.AuthInfos[name].AuthInfo.ClientCertificateData = pkiutil.EncodeCertPEM(clientCert)
+		tmpl.AuthInfos[name].AuthInfo.ClientKeyData, err = keyutil.MarshalPrivateKeyToPEM(clientKey)
+		if err != nil {
+			return fmt.Errorf("cannot marshal private key to PEM: %w", err)
+		}
+	}
+
+	for name := range tmpl.Contexts {
+		tmpl.Contexts[name].Name = contextUserName
+		tmpl.Contexts[name].Context.AuthInfo = contextUserName
+	}
+
+	tmpl.CurrentContext = contextUserName
+
+	_, err = utilities.CreateOrUpdateWithConflict(ctx, r.Client, secret, func() error {
+		labels := secret.GetLabels()
+		if labels == nil {
+			labels = map[string]string{}
+		}
+
+		labels[kamajiv1alpha1.ManagedByLabel] = generator.Name
+		labels[kamajiv1alpha1.ManagedForLabel] = tcp.Name
+		labels[constants.ControllerLabelResource] = utilities.CertificateKubeconfigLabel
+
+		secret.SetLabels(labels)
+
+		if secret.Data == nil {
+			secret.Data = make(map[string][]byte)
+		}
+
+		secret.Data["value"], err = utilities.EncodeToYaml(tmpl)
+		if err != nil {
+			return fmt.Errorf("cannot encode generated Kubeconfig to YAML: %w", err)
+		}
+
+		if utilities.IsRotationRequested(secret) {
+			utilities.SetLastRotationTimestamp(secret)
+		}
+
+		if orErr := controllerutil.SetOwnerReference(tcp, secret, r.Client.Scheme()); orErr != nil {
+			return orErr
+		}
+
+		return ctrl.SetControllerReference(tcp, secret, r.Client.Scheme())
+	})
+	if err != nil {
+		return fmt.Errorf("cannot create or update generated Kubeconfig: %w", err)
+	}
+
+	return nil
+}
+
+func (r *KubeconfigGeneratorReconciler) isValid(secret *corev1.Secret, tmpl *clientcmdapiv1.Config, groups sets.Set[string], user string) (bool, error) {
+	if utilities.IsRotationRequested(secret) {
+		return false, nil
+	}
+
+	concrete, decodeErr := utilities.DecodeKubeconfig(*secret, "value")
+	if decodeErr != nil {
+		return false, decodeErr
+	}
+	// Checking Certificate Authority validity
+	switch {
+	case len(concrete.Clusters) != len(tmpl.Clusters):
+		return false, nil
+	default:
+		for i := range tmpl.Clusters {
+			if !bytes.Equal(tmpl.Clusters[i].Cluster.CertificateAuthorityData, concrete.Clusters[i].Cluster.CertificateAuthorityData) {
+				return false, nil
+			}
+
+			if tmpl.Clusters[i].Cluster.Server != concrete.Clusters[i].Cluster.Server {
+				return false, nil
+			}
+		}
+	}
+
+	for _, auth := range concrete.AuthInfos {
+		valid, vErr := crypto.IsValidCertificateKeyPairBytes(auth.AuthInfo.ClientCertificateData, auth.AuthInfo.ClientKeyData, r.NotValidThreshold)
+		if vErr != nil {
+			return false, vErr
+		}
+		if !valid {
+			return false, nil
+		}
+
+		crt, crtErr := crypto.ParseCertificateBytes(auth.AuthInfo.ClientCertificateData)
+		if crtErr != nil {
+			return false, crtErr
+		}
+
+		if crt.Subject.CommonName != user {
+			return false, nil
+		}
+
+		if !sets.New[string](crt.Subject.Organization...).Equal(groups) {
+			return false, nil
+		}
+	}
+
+	return true, nil
+}
+
+func (r *KubeconfigGeneratorReconciler) SetupWithManager(mgr manager.Manager) error {
+	return ctrl.NewControllerManagedBy(mgr).
+		For(&kamajiv1alpha1.KubeconfigGenerator{}).
+		WatchesRawSource(source.Channel(r.CertificateChan, handler.Funcs{GenericFunc: func(_ context.Context, genericEvent event.TypedGenericEvent[client.Object], w workqueue.TypedRateLimitingInterface[reconcile.Request]) {
+			w.AddRateLimited(ctrl.Request{
+				NamespacedName: types.NamespacedName{
+					Name: genericEvent.Object.GetName(),
+				},
+			})
+		}})).
+		Watches(&corev1.Secret{}, handler.TypedEnqueueRequestsFromMapFunc(func(ctx context.Context, object client.Object) []ctrl.Request {
+			if object.GetLabels() == nil {
+				return nil
+			}
+
+			v, found := object.GetLabels()[kamajiv1alpha1.ManagedByLabel]
+			if !found {
+				return nil
+			}
+
+			return []ctrl.Request{
+				{
+					NamespacedName: types.NamespacedName{
+						Name: v,
+					},
+				},
+			}
+		})).
+		Complete(r)
+}

controllers/kubeconfiggenerator_watcher.go

@@ -0,0 +1,75 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package controllers
+
+import (
+	"context"
+
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/event"
+	"sigs.k8s.io/controller-runtime/pkg/log"
+	"sigs.k8s.io/controller-runtime/pkg/manager"
+
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+)
+
+type KubeconfigGeneratorWatcher struct {
+	Client        client.Client
+	GeneratorChan chan event.GenericEvent
+}
+
+func (r *KubeconfigGeneratorWatcher) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	logger := log.FromContext(ctx)
+
+	logger.Info("reconciling resource")
+
+	var tcp kamajiv1alpha1.TenantControlPlane
+	if err := r.Client.Get(ctx, req.NamespacedName, &tcp); err != nil {
+		if apierrors.IsNotFound(err) {
+			logger.Info("resource may have been deleted, skipping")
+
+			return ctrl.Result{}, nil
+		}
+
+		logger.Error(err, "cannot retrieve the required resource")
+
+		return ctrl.Result{}, err
+	}
+
+	var generators kamajiv1alpha1.KubeconfigGeneratorList
+	if err := r.Client.List(ctx, &generators); err != nil {
+		logger.Error(err, "cannot list generators")
+
+		return ctrl.Result{}, err
+	}
+
+	for _, generator := range generators.Items {
+		sel, err := metav1.LabelSelectorAsSelector(&generator.Spec.TenantControlPlaneSelector)
+		if err != nil {
+			logger.Error(err, "cannot validate Selector", "generator", generator.Name)
+
+			return ctrl.Result{}, err
+		}
+
+		if sel.Matches(labels.Set(tcp.Labels)) {
+			logger.Info("pushing Generator", "generator", generator.Name)
+
+			r.GeneratorChan <- event.GenericEvent{
+				Object: &generator,
+			}
+		}
+	}
+
+	return ctrl.Result{}, nil
+}
+
+func (r *KubeconfigGeneratorWatcher) SetupWithManager(mgr manager.Manager) error {
+	return ctrl.NewControllerManagedBy(mgr).
+		For(&kamajiv1alpha1.TenantControlPlane{}).
+		Complete(r)
+}

controllers/resources.go

@@ -4,11 +4,14 @@
 package controllers
 
 import (
+	"context"
 	"fmt"
+	"time"
 
 	"github.com/go-logr/logr"
 	"github.com/google/uuid"
 	k8stypes "k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/discovery"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 
@@ -19,19 +22,24 @@ import (
 	"github.com/clastix/kamaji/internal/resources"
 	ds "github.com/clastix/kamaji/internal/resources/datastore"
 	"github.com/clastix/kamaji/internal/resources/konnectivity"
+	"github.com/clastix/kamaji/internal/utilities"
 )
 
 type GroupResourceBuilderConfiguration struct {
-	client               client.Client
-	log                  logr.Logger
-	tcpReconcilerConfig  TenantControlPlaneReconcilerConfig
-	tenantControlPlane   kamajiv1alpha1.TenantControlPlane
-	Connection           datastore.Connection
-	DataStore            kamajiv1alpha1.DataStore
-	KamajiNamespace      string
-	KamajiServiceAccount string
-	KamajiService        string
-	KamajiMigrateImage   string
+	client                        client.Client
+	log                           logr.Logger
+	tcpReconcilerConfig           TenantControlPlaneReconcilerConfig
+	tenantControlPlane            kamajiv1alpha1.TenantControlPlane
+	ExpirationThreshold           time.Duration
+	Connection                    datastore.Connection
+	DataStore                     kamajiv1alpha1.DataStore
+	DataStoreOverrides            []builder.DataStoreOverrides
+	DataStoreOverriedsConnections map[string]datastore.Connection
+	KamajiNamespace               string
+	KamajiServiceAccount          string
+	KamajiService                 string
+	KamajiMigrateImage            string
+	DiscoveryClient               discovery.DiscoveryInterface
 }
 
 type GroupDeletableResourceBuilderConfiguration struct {
@@ -46,8 +54,30 @@ type GroupDeletableResourceBuilderConfiguration struct {
 // GetResources returns a list of resources that will be used to provide tenant control planes
 // Currently there is only a default approach
 // TODO: the idea of this function is to become a factory to return the group of resources according to the given configuration.
-func GetResources(config GroupResourceBuilderConfiguration) []resources.Resource {
-	return getDefaultResources(config)
+func GetResources(ctx context.Context, config GroupResourceBuilderConfiguration) []resources.Resource {
+	resources := []resources.Resource{}
+
+	resources = append(resources, getDataStoreMigratingResources(config.client, config.KamajiNamespace, config.KamajiMigrateImage, config.KamajiServiceAccount, config.KamajiService)...)
+	resources = append(resources, getUpgradeResources(config.client)...)
+	resources = append(resources, getKubernetesServiceResources(config.client)...)
+	resources = append(resources, getKubeadmConfigResources(config.client, getTmpDirectory(config.tcpReconcilerConfig.TmpBaseDirectory, config.tenantControlPlane), config.DataStore)...)
+	resources = append(resources, getKubernetesCertificatesResources(config.client, config.tcpReconcilerConfig, config.tenantControlPlane)...)
+	resources = append(resources, getKubeconfigResources(config.client, config.tcpReconcilerConfig, config.tenantControlPlane)...)
+	resources = append(resources, getKubernetesStorageResources(config.client, config.Connection, config.DataStore, config.ExpirationThreshold)...)
+	resources = append(resources, getKubernetesAdditionalStorageResources(config.client, config.DataStoreOverriedsConnections, config.DataStoreOverrides, config.ExpirationThreshold)...)
+	resources = append(resources, getKonnectivityServerRequirementsResources(config.client, config.ExpirationThreshold)...)
+	resources = append(resources, getKubernetesDeploymentResources(config.client, config.tcpReconcilerConfig, config.DataStore, config.DataStoreOverrides)...)
+	resources = append(resources, getKonnectivityServerPatchResources(config.client)...)
+	resources = append(resources, getDataStoreMigratingCleanup(config.client, config.KamajiNamespace)...)
+	resources = append(resources, getKubernetesIngressResources(config.client)...)
+
+	// Conditionally add Gateway resources
+	if utilities.AreGatewayResourcesAvailable(ctx, config.client, config.DiscoveryClient) {
+		resources = append(resources, getKubernetesGatewayResources(config.client)...)
+		resources = append(resources, getKonnectivityGatewayResources(config.client)...)
+	}
+
+	return resources
 }
 
 // GetDeletableResources returns a list of resources that have to be deleted when tenant control planes are deleted
@@ -71,23 +101,6 @@ func GetDeletableResources(tcp *kamajiv1alpha1.TenantControlPlane, config GroupD
 	return res
 }
 
-func getDefaultResources(config GroupResourceBuilderConfiguration) []resources.Resource {
-	resources := getDataStoreMigratingResources(config.client, config.KamajiNamespace, config.KamajiMigrateImage, config.KamajiServiceAccount, config.KamajiService)
-	resources = append(resources, getUpgradeResources(config.client)...)
-	resources = append(resources, getKubernetesServiceResources(config.client)...)
-	resources = append(resources, getKubeadmConfigResources(config.client, getTmpDirectory(config.tcpReconcilerConfig.TmpBaseDirectory, config.tenantControlPlane), config.DataStore)...)
-	resources = append(resources, getKubernetesCertificatesResources(config.client, config.tcpReconcilerConfig, config.tenantControlPlane)...)
-	resources = append(resources, getKubeconfigResources(config.client, config.tcpReconcilerConfig, config.tenantControlPlane)...)
-	resources = append(resources, getKubernetesStorageResources(config.client, config.Connection, config.DataStore)...)
-	resources = append(resources, getKonnectivityServerRequirementsResources(config.client)...)
-	resources = append(resources, getKubernetesDeploymentResources(config.client, config.tcpReconcilerConfig, config.DataStore)...)
-	resources = append(resources, getKonnectivityServerPatchResources(config.client)...)
-	resources = append(resources, getDataStoreMigratingCleanup(config.client, config.KamajiNamespace)...)
-	resources = append(resources, getKubernetesIngressResources(config.client)...)
-
-	return resources
-}
-
 func getDataStoreMigratingCleanup(c client.Client, kamajiNamespace string) []resources.Resource {
 	return []resources.Resource{
 		&ds.Migrate{
@@ -126,6 +139,22 @@ func getKubernetesServiceResources(c client.Client) []resources.Resource {
 	}
 }
 
+func getKubernetesGatewayResources(c client.Client) []resources.Resource {
+	return []resources.Resource{
+		&resources.KubernetesGatewayResource{
+			Client: c,
+		},
+	}
+}
+
+func getKonnectivityGatewayResources(c client.Client) []resources.Resource {
+	return []resources.Resource{
+		&konnectivity.KubernetesKonnectivityGatewayResource{
+			Client: c,
+		},
+	}
+}
+
 func getKubeadmConfigResources(c client.Client, tmpDirectory string, dataStore kamajiv1alpha1.DataStore) []resources.Resource {
 	var endpoints []string
 
@@ -148,28 +177,33 @@ func getKubeadmConfigResources(c client.Client, tmpDirectory string, dataStore k
 func getKubernetesCertificatesResources(c client.Client, tcpReconcilerConfig TenantControlPlaneReconcilerConfig, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource {
 	return []resources.Resource{
 		&resources.CACertificate{
-			Client:       c,
-			TmpDirectory: getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
+			Client:                  c,
+			TmpDirectory:            getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
+			CertExpirationThreshold: tcpReconcilerConfig.CertExpirationThreshold,
 		},
 		&resources.FrontProxyCACertificate{
-			Client:       c,
-			TmpDirectory: getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
+			Client:                  c,
+			TmpDirectory:            getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
+			CertExpirationThreshold: tcpReconcilerConfig.CertExpirationThreshold,
 		},
 		&resources.SACertificate{
 			Client:       c,
 			TmpDirectory: getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
 		},
 		&resources.APIServerCertificate{
-			Client:       c,
-			TmpDirectory: getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
+			Client:                  c,
+			TmpDirectory:            getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
+			CertExpirationThreshold: tcpReconcilerConfig.CertExpirationThreshold,
 		},
 		&resources.APIServerKubeletClientCertificate{
-			Client:       c,
-			TmpDirectory: getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
+			Client:                  c,
+			TmpDirectory:            getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
+			CertExpirationThreshold: tcpReconcilerConfig.CertExpirationThreshold,
 		},
 		&resources.FrontProxyClientCertificate{
-			Client:       c,
-			TmpDirectory: getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
+			Client:                  c,
+			TmpDirectory:            getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
+			CertExpirationThreshold: tcpReconcilerConfig.CertExpirationThreshold,
 		},
 	}
 }
@@ -177,33 +211,37 @@ func getKubernetesCertificatesResources(c client.Client, tcpReconcilerConfig Ten
 func getKubeconfigResources(c client.Client, tcpReconcilerConfig TenantControlPlaneReconcilerConfig, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource {
 	return []resources.Resource{
 		&resources.KubeconfigResource{
-			Name:               "admin-kubeconfig",
-			Client:             c,
-			KubeConfigFileName: resources.AdminKubeConfigFileName,
-			TmpDirectory:       getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
+			Client:                  c,
+			Name:                    "admin-kubeconfig",
+			KubeConfigFileName:      resources.AdminKubeConfigFileName,
+			TmpDirectory:            getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
+			CertExpirationThreshold: tcpReconcilerConfig.CertExpirationThreshold,
 		},
 		&resources.KubeconfigResource{
-			Name:               "admin-kubeconfig",
-			Client:             c,
-			KubeConfigFileName: resources.SuperAdminKubeConfigFileName,
-			TmpDirectory:       getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
+			Client:                  c,
+			Name:                    "admin-kubeconfig",
+			KubeConfigFileName:      resources.SuperAdminKubeConfigFileName,
+			TmpDirectory:            getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
+			CertExpirationThreshold: tcpReconcilerConfig.CertExpirationThreshold,
 		},
 		&resources.KubeconfigResource{
-			Name:               "controller-manager-kubeconfig",
-			Client:             c,
-			KubeConfigFileName: resources.ControllerManagerKubeConfigFileName,
-			TmpDirectory:       getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
+			Client:                  c,
+			Name:                    "controller-manager-kubeconfig",
+			KubeConfigFileName:      resources.ControllerManagerKubeConfigFileName,
+			TmpDirectory:            getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
+			CertExpirationThreshold: tcpReconcilerConfig.CertExpirationThreshold,
 		},
 		&resources.KubeconfigResource{
-			Name:               "scheduler-kubeconfig",
-			Client:             c,
-			KubeConfigFileName: resources.SchedulerKubeConfigFileName,
-			TmpDirectory:       getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
+			Client:                  c,
+			Name:                    "scheduler-kubeconfig",
+			KubeConfigFileName:      resources.SchedulerKubeConfigFileName,
+			TmpDirectory:            getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
+			CertExpirationThreshold: tcpReconcilerConfig.CertExpirationThreshold,
 		},
 	}
 }
 
-func getKubernetesStorageResources(c client.Client, dbConnection datastore.Connection, datastore kamajiv1alpha1.DataStore) []resources.Resource {
+func getKubernetesStorageResources(c client.Client, dbConnection datastore.Connection, datastore kamajiv1alpha1.DataStore, threshold time.Duration) []resources.Resource {
 	return []resources.Resource{
 		&ds.MultiTenancy{
 			DataStore: datastore,
@@ -219,18 +257,49 @@ func getKubernetesStorageResources(c client.Client, dbConnection datastore.Conne
 			DataStore:  datastore,
 		},
 		&ds.Certificate{
-			Client:    c,
-			DataStore: datastore,
+			Client:                  c,
+			DataStore:               datastore,
+			CertExpirationThreshold: threshold,
 		},
 	}
 }
 
-func getKubernetesDeploymentResources(c client.Client, tcpReconcilerConfig TenantControlPlaneReconcilerConfig, dataStore kamajiv1alpha1.DataStore) []resources.Resource {
+func getKubernetesAdditionalStorageResources(c client.Client, dbConnections map[string]datastore.Connection, dataStoreOverrides []builder.DataStoreOverrides, threshold time.Duration) []resources.Resource {
+	res := make([]resources.Resource, 0, len(dataStoreOverrides))
+	for _, dso := range dataStoreOverrides {
+		datastore := dso.DataStore
+		res = append(res,
+			&ds.MultiTenancy{
+				DataStore: datastore,
+			},
+			&ds.Config{
+				Client:     c,
+				ConnString: dbConnections[dso.Resource].GetConnectionString(),
+				DataStore:  datastore,
+				IsOverride: true,
+			},
+			&ds.Setup{
+				Client:     c,
+				Connection: dbConnections[dso.Resource],
+				DataStore:  datastore,
+			},
+			&ds.Certificate{
+				Client:                  c,
+				DataStore:               datastore,
+				CertExpirationThreshold: threshold,
+			})
+	}
+
+	return res
+}
+
+func getKubernetesDeploymentResources(c client.Client, tcpReconcilerConfig TenantControlPlaneReconcilerConfig, dataStore kamajiv1alpha1.DataStore, dataStoreOverrides []builder.DataStoreOverrides) []resources.Resource {
 	return []resources.Resource{
 		&resources.KubernetesDeploymentResource{
 			Client:             c,
 			DataStore:          dataStore,
 			KineContainerImage: tcpReconcilerConfig.KineContainerImage,
+			DataStoreOverrides: dataStoreOverrides,
 		},
 	}
 }
@@ -251,10 +320,10 @@ func GetExternalKonnectivityResources(c client.Client) []resources.Resource {
 	}
 }
 
-func getKonnectivityServerRequirementsResources(c client.Client) []resources.Resource {
+func getKonnectivityServerRequirementsResources(c client.Client, threshold time.Duration) []resources.Resource {
 	return []resources.Resource{
 		&konnectivity.EgressSelectorConfigurationResource{Client: c},
-		&konnectivity.CertificateResource{Client: c},
+		&konnectivity.CertificateResource{Client: c, CertExpirationThreshold: threshold},
 		&konnectivity.KubeconfigResource{Client: c},
 	}
 }

controllers/soot/controllers/coredns.go

@@ -5,6 +5,7 @@ package controllers
 
 import (
 	"context"
+	"errors"
 
 	"github.com/go-logr/logr"
 	appsv1 "k8s.io/api/apps/v1"
@@ -23,6 +24,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 	"sigs.k8s.io/controller-runtime/pkg/source"
 
+	sooterrors "github.com/clastix/kamaji/controllers/soot/controllers/errors"
 	"github.com/clastix/kamaji/controllers/utils"
 	"github.com/clastix/kamaji/internal/kubeadm"
 	"github.com/clastix/kamaji/internal/resources"
@@ -30,54 +32,56 @@ import (
 )
 
 type CoreDNS struct {
-	logger logr.Logger
-
+	Logger                    logr.Logger
 	AdminClient               client.Client
 	GetTenantControlPlaneFunc utils.TenantControlPlaneRetrievalFn
 	TriggerChannel            chan event.GenericEvent
+	ControllerName            string
 }
 
 func (c *CoreDNS) Reconcile(ctx context.Context, _ reconcile.Request) (reconcile.Result, error) {
 	tcp, err := c.GetTenantControlPlaneFunc()
 	if err != nil {
-		c.logger.Error(err, "cannot retrieve TenantControlPlane")
+		if errors.Is(err, sooterrors.ErrPausedReconciliation) {
+			c.Logger.Info(err.Error())
+
+			return reconcile.Result{}, nil
+		}
 
 		return reconcile.Result{}, err
 	}
 
-	c.logger.Info("start processing")
+	c.Logger.Info("start processing")
 
 	resource := &addons.CoreDNS{Client: c.AdminClient}
 
 	result, handlingErr := resources.Handle(ctx, resource, tcp)
 	if handlingErr != nil {
-		c.logger.Error(handlingErr, "resource process failed", "resource", resource.GetName())
+		c.Logger.Error(handlingErr, "resource process failed", "resource", resource.GetName())
 
 		return reconcile.Result{}, handlingErr
 	}
 
 	if result == controllerutil.OperationResultNone {
-		c.logger.Info("reconciliation completed")
+		c.Logger.Info("reconciliation completed")
 
 		return reconcile.Result{}, nil
 	}
 
 	if err = utils.UpdateStatus(ctx, c.AdminClient, tcp, resource); err != nil {
-		c.logger.Error(err, "update status failed", "resource", resource.GetName())
+		c.Logger.Error(err, "update status failed", "resource", resource.GetName())
 
 		return reconcile.Result{}, err
 	}
 
-	c.logger.Info("reconciliation processed")
+	c.Logger.Info("reconciliation processed")
 
 	return reconcile.Result{}, nil
 }
 
 func (c *CoreDNS) SetupWithManager(mgr manager.Manager) error {
-	c.logger = mgr.GetLogger().WithName("coredns")
-	c.TriggerChannel = make(chan event.GenericEvent)
-
 	return controllerruntime.NewControllerManagedBy(mgr).
+		Named(c.ControllerName).
 		WithOptions(controller.TypedOptions[reconcile.Request]{SkipNameValidation: ptr.To(true)}).
 		For(&rbacv1.ClusterRoleBinding{}, builder.WithPredicates(predicate.NewPredicateFuncs(func(object client.Object) bool {
 			return object.GetName() == kubeadm.CoreDNSClusterRoleBindingName

controllers/soot/controllers/errors/paused_reconciliation.go

@@ -0,0 +1,8 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package errors
+
+import "errors"
+
+var ErrPausedReconciliation = errors.New("paused reconciliation, no further actions")

controllers/soot/controllers/konnectivity.go

@@ -5,6 +5,7 @@ package controllers
 
 import (
 	"context"
+	"errors"
 
 	"github.com/go-logr/logr"
 	appsv1 "k8s.io/api/apps/v1"
@@ -25,60 +26,69 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/source"
 
 	"github.com/clastix/kamaji/controllers"
+	sooterrors "github.com/clastix/kamaji/controllers/soot/controllers/errors"
 	"github.com/clastix/kamaji/controllers/utils"
 	"github.com/clastix/kamaji/internal/resources"
 	"github.com/clastix/kamaji/internal/resources/konnectivity"
 )
 
 type KonnectivityAgent struct {
-	logger logr.Logger
-
+	Logger                    logr.Logger
 	AdminClient               client.Client
 	GetTenantControlPlaneFunc utils.TenantControlPlaneRetrievalFn
 	TriggerChannel            chan event.GenericEvent
+	ControllerName            string
 }
 
 func (k *KonnectivityAgent) Reconcile(ctx context.Context, _ reconcile.Request) (reconcile.Result, error) {
 	tcp, err := k.GetTenantControlPlaneFunc()
 	if err != nil {
-		k.logger.Error(err, "cannot retrieve TenantControlPlane")
+		if errors.Is(err, sooterrors.ErrPausedReconciliation) {
+			k.Logger.Info(err.Error())
+
+			return reconcile.Result{}, nil
+		}
+
+		k.Logger.Error(err, "cannot retrieve TenantControlPlane")
 
 		return reconcile.Result{}, err
 	}
 
+	if tcp.Spec.Addons.Konnectivity == nil {
+		return reconcile.Result{}, nil
+	}
+
 	for _, resource := range controllers.GetExternalKonnectivityResources(k.AdminClient) {
-		k.logger.Info("start processing", "resource", resource.GetName())
+		k.Logger.Info("start processing", "resource", resource.GetName())
 
 		result, handlingErr := resources.Handle(ctx, resource, tcp)
 		if handlingErr != nil {
-			k.logger.Error(handlingErr, "resource process failed", "resource", resource.GetName())
+			k.Logger.Error(handlingErr, "resource process failed", "resource", resource.GetName())
 
 			return reconcile.Result{}, handlingErr
 		}
 
 		if result == controllerutil.OperationResultNone {
-			k.logger.Info("resource processed", "resource", resource.GetName())
+			k.Logger.Info("resource processed", "resource", resource.GetName())
 
 			continue
 		}
 
 		if err = utils.UpdateStatus(ctx, k.AdminClient, tcp, resource); err != nil {
-			k.logger.Error(err, "update status failed", "resource", resource.GetName())
+			k.Logger.Error(err, "update status failed", "resource", resource.GetName())
 
 			return reconcile.Result{}, err
 		}
 	}
 
-	k.logger.Info("reconciliation completed")
+	k.Logger.Info("reconciliation completed")
 
 	return reconcile.Result{}, nil
 }
 
 func (k *KonnectivityAgent) SetupWithManager(mgr manager.Manager) error {
-	k.logger = mgr.GetLogger().WithName("konnectivity_agent")
-	k.TriggerChannel = make(chan event.GenericEvent)
-
 	return controllerruntime.NewControllerManagedBy(mgr).
+		Named(k.ControllerName).
 		WithOptions(controller.TypedOptions[reconcile.Request]{SkipNameValidation: ptr.To(true)}).
 		For(&appsv1.DaemonSet{}, builder.WithPredicates(predicate.NewPredicateFuncs(func(object client.Object) bool {
 			return object.GetName() == konnectivity.AgentName && object.GetNamespace() == konnectivity.AgentNamespace

controllers/soot/controllers/kubeadm_phase.go

@@ -5,6 +5,7 @@ package controllers
 
 import (
 	"context"
+	"errors"
 
 	"github.com/go-logr/logr"
 	"k8s.io/utils/ptr"
@@ -19,6 +20,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 	"sigs.k8s.io/controller-runtime/pkg/source"
 
+	sooterrors "github.com/clastix/kamaji/controllers/soot/controllers/errors"
 	"github.com/clastix/kamaji/controllers/utils"
 	"github.com/clastix/kamaji/internal/resources"
 )
@@ -27,6 +29,7 @@ type KubeadmPhase struct {
 	GetTenantControlPlaneFunc utils.TenantControlPlaneRetrievalFn
 	TriggerChannel            chan event.GenericEvent
 	Phase                     resources.KubeadmPhaseResource
+	ControllerName            string
 
 	logger logr.Logger
 }
@@ -34,6 +37,12 @@ type KubeadmPhase struct {
 func (k *KubeadmPhase) Reconcile(ctx context.Context, _ reconcile.Request) (reconcile.Result, error) {
 	tcp, err := k.GetTenantControlPlaneFunc()
 	if err != nil {
+		if errors.Is(err, sooterrors.ErrPausedReconciliation) {
+			k.logger.Info(err.Error())
+
+			return reconcile.Result{}, nil
+		}
+
 		return reconcile.Result{}, err
 	}
 
@@ -65,9 +74,9 @@ func (k *KubeadmPhase) Reconcile(ctx context.Context, _ reconcile.Request) (reco
 
 func (k *KubeadmPhase) SetupWithManager(mgr manager.Manager) error {
 	k.logger = mgr.GetLogger().WithName(k.Phase.GetName())
-	k.TriggerChannel = make(chan event.GenericEvent)
 
 	return controllerruntime.NewControllerManagedBy(mgr).
+		Named(k.ControllerName).
 		WithOptions(controller.TypedOptions[reconcile.Request]{SkipNameValidation: ptr.To(true)}).
 		For(k.Phase.GetWatchedObject(), builder.WithPredicates(predicate.NewPredicateFuncs(k.Phase.GetPredicateFunc()))).
 		WatchesRawSource(source.Channel(k.TriggerChannel, &handler.EnqueueRequestForObject{})).

controllers/soot/controllers/kubeproxy.go

@@ -5,6 +5,7 @@ package controllers
 
 import (
 	"context"
+	"errors"
 
 	"github.com/go-logr/logr"
 	appsv1 "k8s.io/api/apps/v1"
@@ -23,6 +24,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 	"sigs.k8s.io/controller-runtime/pkg/source"
 
+	sooterrors "github.com/clastix/kamaji/controllers/soot/controllers/errors"
 	"github.com/clastix/kamaji/controllers/utils"
 	"github.com/clastix/kamaji/internal/kubeadm"
 	"github.com/clastix/kamaji/internal/resources"
@@ -30,54 +32,58 @@ import (
 )
 
 type KubeProxy struct {
+	Logger                    logr.Logger
 	AdminClient               client.Client
 	GetTenantControlPlaneFunc utils.TenantControlPlaneRetrievalFn
 	TriggerChannel            chan event.GenericEvent
-
-	logger logr.Logger
+	ControllerName            string
 }
 
 func (k *KubeProxy) Reconcile(ctx context.Context, _ reconcile.Request) (reconcile.Result, error) {
 	tcp, err := k.GetTenantControlPlaneFunc()
 	if err != nil {
-		k.logger.Error(err, "cannot retrieve TenantControlPlane")
+		if errors.Is(err, sooterrors.ErrPausedReconciliation) {
+			k.Logger.Info(err.Error())
+
+			return reconcile.Result{}, nil
+		}
+
+		k.Logger.Error(err, "cannot retrieve TenantControlPlane")
 
 		return reconcile.Result{}, err
 	}
 
-	k.logger.Info("start processing")
+	k.Logger.Info("start processing")
 
 	resource := &addons.KubeProxy{Client: k.AdminClient}
 
 	result, handlingErr := resources.Handle(ctx, resource, tcp)
 	if handlingErr != nil {
-		k.logger.Error(handlingErr, "resource process failed", "resource", resource.GetName())
+		k.Logger.Error(handlingErr, "resource process failed", "resource", resource.GetName())
 
 		return reconcile.Result{}, handlingErr
 	}
 
 	if result == controllerutil.OperationResultNone {
-		k.logger.Info("reconciliation completed")
+		k.Logger.Info("reconciliation completed")
 
 		return reconcile.Result{}, nil
 	}
 
 	if err = utils.UpdateStatus(ctx, k.AdminClient, tcp, resource); err != nil {
-		k.logger.Error(err, "update status failed")
+		k.Logger.Error(err, "update status failed")
 
 		return reconcile.Result{}, err
 	}
 
-	k.logger.Info("reconciliation processed")
+	k.Logger.Info("reconciliation processed")
 
 	return reconcile.Result{}, nil
 }
 
 func (k *KubeProxy) SetupWithManager(mgr manager.Manager) error {
-	k.logger = mgr.GetLogger().WithName("kube_proxy")
-	k.TriggerChannel = make(chan event.GenericEvent)
-
 	return controllerruntime.NewControllerManagedBy(mgr).
+		Named(k.ControllerName).
 		WithOptions(controller.TypedOptions[reconcile.Request]{SkipNameValidation: ptr.To(true)}).
 		For(&rbacv1.ClusterRoleBinding{}, builder.WithPredicates(predicate.NewPredicateFuncs(func(object client.Object) bool {
 			return object.GetName() == kubeadm.KubeProxyClusterRoleBindingName

controllers/soot/controllers/migrate.go

@@ -5,11 +5,13 @@ package controllers
 
 import (
 	"context"
+	"errors"
 	"fmt"
+	"time"
 
 	"github.com/go-logr/logr"
 	admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
-	"k8s.io/apimachinery/pkg/api/errors"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	pointer "k8s.io/utils/ptr"
 	controllerruntime "sigs.k8s.io/controller-runtime"
@@ -24,29 +26,36 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/source"
 
 	"github.com/clastix/kamaji/api/v1alpha1"
+	sooterrors "github.com/clastix/kamaji/controllers/soot/controllers/errors"
 	"github.com/clastix/kamaji/controllers/utils"
 	"github.com/clastix/kamaji/internal/utilities"
 )
 
 type Migrate struct {
-	client client.Client
-	logger logr.Logger
-
+	Client                    client.Client
+	Logger                    logr.Logger
 	GetTenantControlPlaneFunc utils.TenantControlPlaneRetrievalFn
 	WebhookNamespace          string
 	WebhookServiceName        string
 	WebhookCABundle           []byte
 	TriggerChannel            chan event.GenericEvent
+	ControllerName            string
 }
 
 func (m *Migrate) Reconcile(ctx context.Context, _ reconcile.Request) (reconcile.Result, error) {
 	tcp, err := m.GetTenantControlPlaneFunc()
 	if err != nil {
+		if errors.Is(err, sooterrors.ErrPausedReconciliation) {
+			m.Logger.Info(err.Error())
+
+			return reconcile.Result{}, nil
+		}
+
 		return reconcile.Result{}, err
 	}
 	// Cannot detect the status of the TenantControlPlane, enqueuing back
 	if tcp.Status.Kubernetes.Version.Status == nil {
-		return reconcile.Result{Requeue: true}, nil
+		return reconcile.Result{RequeueAfter: time.Second}, nil
 	}
 
 	switch *tcp.Status.Kubernetes.Version.Status {
@@ -57,7 +66,7 @@ func (m *Migrate) Reconcile(ctx context.Context, _ reconcile.Request) (reconcile
 	}
 
 	if err != nil {
-		m.logger.Error(err, "reconciliation failed")
+		m.Logger.Error(err, "reconciliation failed")
 
 		return reconcile.Result{}, err
 	}
@@ -66,8 +75,8 @@ func (m *Migrate) Reconcile(ctx context.Context, _ reconcile.Request) (reconcile
 }
 
 func (m *Migrate) cleanup(ctx context.Context) error {
-	if err := m.client.Delete(ctx, m.object()); err != nil {
-		if errors.IsNotFound(err) {
+	if err := m.Client.Delete(ctx, m.object()); err != nil {
+		if apierrors.IsNotFound(err) {
 			return nil
 		}
 
@@ -80,7 +89,7 @@ func (m *Migrate) cleanup(ctx context.Context) error {
 func (m *Migrate) createOrUpdate(ctx context.Context) error {
 	obj := m.object()
 
-	_, err := utilities.CreateOrUpdateWithConflict(ctx, m.client, obj, func() error {
+	_, err := utilities.CreateOrUpdateWithConflict(ctx, m.Client, obj, func() error {
 		obj.Webhooks = []admissionregistrationv1.ValidatingWebhook{
 			{
 				Name: "leases.migrate.kamaji.clastix.io",
@@ -178,11 +187,10 @@ func (m *Migrate) createOrUpdate(ctx context.Context) error {
 }
 
 func (m *Migrate) SetupWithManager(mgr manager.Manager) error {
-	m.client = mgr.GetClient()
-	m.logger = mgr.GetLogger().WithName("migrate")
 	m.TriggerChannel = make(chan event.GenericEvent)
 
 	return controllerruntime.NewControllerManagedBy(mgr).
+		Named(m.ControllerName).
 		WithOptions(controller.TypedOptions[reconcile.Request]{SkipNameValidation: pointer.To(true)}).
 		For(&admissionregistrationv1.ValidatingWebhookConfiguration{}, builder.WithPredicates(predicate.NewPredicateFuncs(func(object client.Object) bool {
 			vwc := m.object()

controllers/soot/controllers/write_permissions.go

@@ -0,0 +1,209 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package controllers
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/go-logr/logr"
+	admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/kubernetes/cmd/kubeadm/app/util/errors"
+	"k8s.io/utils/ptr"
+	controllerruntime "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/builder"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller"
+	"sigs.k8s.io/controller-runtime/pkg/event"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
+	"sigs.k8s.io/controller-runtime/pkg/manager"
+	"sigs.k8s.io/controller-runtime/pkg/predicate"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+	"sigs.k8s.io/controller-runtime/pkg/source"
+
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+	sooterrors "github.com/clastix/kamaji/controllers/soot/controllers/errors"
+	"github.com/clastix/kamaji/controllers/utils"
+	"github.com/clastix/kamaji/internal/utilities"
+)
+
+type WritePermissions struct {
+	Logger                    logr.Logger
+	Client                    client.Client
+	GetTenantControlPlaneFunc utils.TenantControlPlaneRetrievalFn
+	WebhookNamespace          string
+	WebhookServiceName        string
+	WebhookCABundle           []byte
+	TriggerChannel            chan event.GenericEvent
+	ControllerName            string
+}
+
+func (r *WritePermissions) Reconcile(ctx context.Context, _ reconcile.Request) (reconcile.Result, error) {
+	tcp, err := r.GetTenantControlPlaneFunc()
+	if err != nil {
+		if errors.Is(err, sooterrors.ErrPausedReconciliation) {
+			r.Logger.Info(err.Error())
+
+			return reconcile.Result{}, nil
+		}
+
+		return reconcile.Result{}, err
+	}
+	// Cannot detect the status of the TenantControlPlane, enqueuing back
+	if tcp.Status.Kubernetes.Version.Status == nil {
+		return reconcile.Result{RequeueAfter: time.Second}, nil
+	}
+
+	switch {
+	case ptr.Deref(tcp.Status.Kubernetes.Version.Status, kamajiv1alpha1.VersionUnknown) == kamajiv1alpha1.VersionWriteLimited &&
+		tcp.Spec.WritePermissions.HasAnyLimitation():
+		err = r.createOrUpdate(ctx, tcp.Spec.WritePermissions)
+	default:
+		err = r.cleanup(ctx)
+	}
+
+	if err != nil {
+		r.Logger.Error(err, "reconciliation failed")
+
+		return reconcile.Result{}, err
+	}
+
+	return reconcile.Result{}, nil
+}
+
+func (r *WritePermissions) createOrUpdate(ctx context.Context, writePermissions kamajiv1alpha1.Permissions) error {
+	obj := r.object().DeepCopy()
+
+	_, err := utilities.CreateOrUpdateWithConflict(ctx, r.Client, obj, func() error {
+		obj.Webhooks = []admissionregistrationv1.ValidatingWebhook{
+			{
+				Name: "leases.write-permissions.kamaji.clastix.io",
+				ClientConfig: admissionregistrationv1.WebhookClientConfig{
+					URL:      ptr.To(fmt.Sprintf("https://%s.%s.svc:443/write-permission", r.WebhookServiceName, r.WebhookNamespace)),
+					CABundle: r.WebhookCABundle,
+				},
+				Rules: []admissionregistrationv1.RuleWithOperations{
+					{
+						Operations: []admissionregistrationv1.OperationType{
+							admissionregistrationv1.Create,
+							admissionregistrationv1.Delete,
+						},
+						Rule: admissionregistrationv1.Rule{
+							APIGroups:   []string{"*"},
+							APIVersions: []string{"*"},
+							Resources:   []string{"*"},
+							Scope:       ptr.To(admissionregistrationv1.NamespacedScope),
+						},
+					},
+				},
+				FailurePolicy: ptr.To(admissionregistrationv1.Fail),
+				MatchPolicy:   ptr.To(admissionregistrationv1.Equivalent),
+				NamespaceSelector: &metav1.LabelSelector{
+					MatchExpressions: []metav1.LabelSelectorRequirement{
+						{
+							Key:      "kubernetes.io/metadata.name",
+							Operator: metav1.LabelSelectorOpIn,
+							Values: []string{
+								"kube-node-lease",
+							},
+						},
+					},
+				},
+				SideEffects:             ptr.To(admissionregistrationv1.SideEffectClassNoneOnDryRun),
+				AdmissionReviewVersions: []string{"v1"},
+			},
+			{
+				Name: "catchall.write-permissions.kamaji.clastix.io",
+				ClientConfig: admissionregistrationv1.WebhookClientConfig{
+					URL:      ptr.To(fmt.Sprintf("https://%s.%s.svc:443/write-permission", r.WebhookServiceName, r.WebhookNamespace)),
+					CABundle: r.WebhookCABundle,
+				},
+				Rules: []admissionregistrationv1.RuleWithOperations{
+					{
+						Operations: func() []admissionregistrationv1.OperationType {
+							var ops []admissionregistrationv1.OperationType
+
+							if writePermissions.BlockCreate {
+								ops = append(ops, admissionregistrationv1.Create)
+							}
+
+							if writePermissions.BlockUpdate {
+								ops = append(ops, admissionregistrationv1.Update)
+							}
+
+							if writePermissions.BlockDelete {
+								ops = append(ops, admissionregistrationv1.Delete)
+							}
+
+							return ops
+						}(),
+						Rule: admissionregistrationv1.Rule{
+							APIGroups:   []string{"*"},
+							APIVersions: []string{"*"},
+							Resources:   []string{"*"},
+							Scope:       ptr.To(admissionregistrationv1.AllScopes),
+						},
+					},
+				},
+				FailurePolicy: ptr.To(admissionregistrationv1.Fail),
+				MatchPolicy:   ptr.To(admissionregistrationv1.Equivalent),
+				NamespaceSelector: &metav1.LabelSelector{
+					MatchExpressions: []metav1.LabelSelectorRequirement{
+						{
+							Key:      "kubernetes.io/metadata.name",
+							Operator: metav1.LabelSelectorOpNotIn,
+							Values: []string{
+								"kube-system",
+								"kube-node-lease",
+							},
+						},
+					},
+				},
+				SideEffects:             ptr.To(admissionregistrationv1.SideEffectClassNoneOnDryRun),
+				TimeoutSeconds:          nil,
+				AdmissionReviewVersions: []string{"v1"},
+			},
+		}
+
+		return nil
+	})
+
+	return err
+}
+
+func (r *WritePermissions) cleanup(ctx context.Context) error {
+	if err := r.Client.Delete(ctx, r.object()); err != nil {
+		if apierrors.IsNotFound(err) {
+			return nil
+		}
+
+		return fmt.Errorf("unable to clean-up ValidationWebhook required for write permissions: %w", err)
+	}
+
+	return nil
+}
+
+func (r *WritePermissions) SetupWithManager(mgr manager.Manager) error {
+	r.TriggerChannel = make(chan event.GenericEvent)
+
+	return controllerruntime.NewControllerManagedBy(mgr).
+		Named(r.ControllerName).
+		WithOptions(controller.TypedOptions[reconcile.Request]{SkipNameValidation: ptr.To(true)}).
+		For(&admissionregistrationv1.ValidatingWebhookConfiguration{}, builder.WithPredicates(predicate.NewPredicateFuncs(func(object client.Object) bool {
+			return object.GetName() == r.object().GetName()
+		}))).
+		WatchesRawSource(source.Channel(r.TriggerChannel, &handler.EnqueueRequestForObject{})).
+		Complete(r)
+}
+
+func (r *WritePermissions) object() *admissionregistrationv1.ValidatingWebhookConfiguration {
+	return &admissionregistrationv1.ValidatingWebhookConfiguration{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "kamaji-write-permissions",
+		},
+	}
+}

controllers/soot/manager.go

@@ -6,8 +6,9 @@ package soot
 import (
 	"context"
 	"fmt"
+	"time"
 
-	"k8s.io/apimachinery/pkg/api/errors"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/util/retry"
 	"k8s.io/utils/ptr"
@@ -28,20 +29,26 @@ import (
 	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
 	"github.com/clastix/kamaji/controllers/finalizers"
 	"github.com/clastix/kamaji/controllers/soot/controllers"
+	"github.com/clastix/kamaji/controllers/soot/controllers/errors"
 	"github.com/clastix/kamaji/controllers/utils"
 	"github.com/clastix/kamaji/internal/resources"
 	"github.com/clastix/kamaji/internal/utilities"
 )
 
 type sootItem struct {
-	triggers []chan event.GenericEvent
-	cancelFn context.CancelFunc
+	triggers    []chan event.GenericEvent
+	cancelFn    context.CancelFunc
+	completedCh chan struct{}
 }
 
 type sootMap map[string]sootItem
 
+const (
+	sootManagerAnnotation       = "kamaji.clastix.io/soot"
+	sootManagerFailedAnnotation = "failed"
+)
+
 type Manager struct {
-	client  client.Client
 	sootMap sootMap
 	// sootManagerErrChan is the channel that is going to be used
 	// when the soot manager cannot start due to any kind of problem.
@@ -59,10 +66,14 @@ func (m *Manager) retrieveTenantControlPlane(ctx context.Context, request reconc
 	return func() (*kamajiv1alpha1.TenantControlPlane, error) {
 		tcp := &kamajiv1alpha1.TenantControlPlane{}
 
-		if err := m.client.Get(ctx, request.NamespacedName, tcp); err != nil {
+		if err := m.AdminClient.Get(ctx, request.NamespacedName, tcp); err != nil {
 			return nil, err
 		}
 
+		if utils.IsPaused(tcp) {
+			return nil, errors.ErrPausedReconciliation
+		}
+
 		return tcp, nil
 	}
 }
@@ -93,39 +104,82 @@ func (m *Manager) cleanup(ctx context.Context, req reconcile.Request, tenantCont
 	}
 
 	v.cancelFn()
+	// TODO(prometherion): the 10 seconds is an hardcoded number,
+	// it's widely used across the code base as a timeout with the API Server.
+	// Evaluate if we would need to make this configurable globally.
+	deadlineCtx, deadlineFn := context.WithTimeout(ctx, 10*time.Second)
+	defer deadlineFn()
+
+	select {
+	case _, open := <-v.completedCh:
+		if !open {
+			log.FromContext(ctx).Info("soot manager completed its process")
+
+			break
+		}
+	case <-deadlineCtx.Done():
+		log.FromContext(ctx).Error(deadlineCtx.Err(), "soot manager didn't exit to timeout")
+
+		break
+	}
 
 	delete(m.sootMap, tcpName)
 
 	return nil
 }
 
+func (m *Manager) retryTenantControlPlaneAnnotations(ctx context.Context, request reconcile.Request, modifierFn func(annotations map[string]string)) error {
+	return retry.RetryOnConflict(retry.DefaultRetry, func() error {
+		tcp, err := m.retrieveTenantControlPlane(ctx, request)()
+		if err != nil {
+			return err
+		}
+
+		if tcp.Annotations == nil {
+			tcp.Annotations = map[string]string{}
+		}
+
+		modifierFn(tcp.Annotations)
+
+		tcp.SetAnnotations(tcp.Annotations)
+
+		return m.AdminClient.Update(ctx, tcp)
+	})
+}
+
+//nolint:maintidx
 func (m *Manager) Reconcile(ctx context.Context, request reconcile.Request) (res reconcile.Result, err error) {
 	// Retrieving the TenantControlPlane:
 	// in case of deletion, we must be sure to properly remove from the memory the soot manager.
 	tcp := &kamajiv1alpha1.TenantControlPlane{}
-	if err = m.client.Get(ctx, request.NamespacedName, tcp); err != nil {
-		if errors.IsNotFound(err) {
+	if err = m.AdminClient.Get(ctx, request.NamespacedName, tcp); err != nil {
+		if apierrors.IsNotFound(err) {
 			return reconcile.Result{}, m.cleanup(ctx, request, nil)
 		}
 
 		return reconcile.Result{}, err
 	}
-	// Handling finalizer if the TenantControlPlane is marked for deletion:
+	tcpStatus := ptr.Deref(tcp.Status.Kubernetes.Version.Status, kamajiv1alpha1.VersionProvisioning)
+	// Handling finalizer if the TenantControlPlane is marked for deletion or scaled to zero:
 	// the clean-up function is already taking care to stop the manager, if this exists.
-	if tcp.GetDeletionTimestamp() != nil {
+	if tcp.GetDeletionTimestamp() != nil || tcpStatus == kamajiv1alpha1.VersionSleeping {
 		if controllerutil.ContainsFinalizer(tcp, finalizers.SootFinalizer) {
 			return reconcile.Result{}, m.cleanup(ctx, request, tcp)
 		}
 
 		return reconcile.Result{}, nil
 	}
-
-	tcpStatus := *tcp.Status.Kubernetes.Version.Status
 	// Triggering the reconciliation of the underlying controllers of
 	// the soot manager if this is already registered.
 	v, ok := m.sootMap[request.String()]
 	if ok {
 		switch {
+		case tcp.Annotations != nil && tcp.Annotations[sootManagerAnnotation] == sootManagerFailedAnnotation:
+			delete(m.sootMap, request.String())
+
+			return reconcile.Result{}, m.retryTenantControlPlaneAnnotations(ctx, request, func(annotations map[string]string) {
+				delete(annotations, sootManagerAnnotation)
+			})
 		case tcpStatus == kamajiv1alpha1.VersionCARotating:
 			// The TenantControlPlane CA has been rotated, it means the running manager
 			// must be restarted to avoid certificate signed by unknown authority errors.
@@ -137,7 +191,12 @@ func (m *Manager) Reconcile(ctx context.Context, request reconcile.Request) (res
 			return reconcile.Result{}, m.cleanup(ctx, request, tcp)
 		default:
 			for _, trigger := range v.triggers {
-				trigger <- event.GenericEvent{Object: tcp}
+				var shrunkTCP kamajiv1alpha1.TenantControlPlane
+
+				shrunkTCP.Name = tcp.Name
+				shrunkTCP.Namespace = tcp.Namespace
+
+				go utils.TriggerChannel(ctx, trigger, shrunkTCP)
 			}
 		}
 
@@ -145,7 +204,7 @@ func (m *Manager) Reconcile(ctx context.Context, request reconcile.Request) (res
 	}
 	// No need to start a soot manager if the TenantControlPlane is not ready:
 	// enqueuing back is not required since we're going to get that event once ready.
-	if tcpStatus == kamajiv1alpha1.VersionNotReady || tcpStatus == kamajiv1alpha1.VersionCARotating {
+	if tcpStatus == kamajiv1alpha1.VersionNotReady || tcpStatus == kamajiv1alpha1.VersionCARotating || tcpStatus == kamajiv1alpha1.VersionSleeping {
 		log.FromContext(ctx).Info("skipping start of the soot manager for a not ready instance")
 
 		return reconcile.Result{}, nil
@@ -159,11 +218,11 @@ func (m *Manager) Reconcile(ctx context.Context, request reconcile.Request) (res
 			return nil
 		})
 
-		return reconcile.Result{Requeue: true}, finalizerErr
+		return reconcile.Result{RequeueAfter: time.Second}, finalizerErr
 	}
 	// Generating the manager and starting it:
 	// in case of any error, reconciling the request to start it back from the beginning.
-	tcpRest, err := utilities.GetRESTClientConfig(ctx, m.client, tcp)
+	tcpRest, err := utilities.GetRESTClientConfig(ctx, m.AdminClient, tcp)
 	if err != nil {
 		return reconcile.Result{}, err
 	}
@@ -178,14 +237,14 @@ func (m *Manager) Reconcile(ctx context.Context, request reconcile.Request) (res
 
 	mgr, err := controllerruntime.NewManager(tcpRest, controllerruntime.Options{
 		Logger: log.Log.WithName(fmt.Sprintf("soot_%s_%s", tcp.GetNamespace(), tcp.GetName())),
-		Scheme: m.client.Scheme(),
+		Scheme: m.AdminClient.Scheme(),
 		Metrics: metricsserver.Options{
 			BindAddress: "0",
 		},
-		NewClient: func(config *rest.Config, _ client.Options) (client.Client, error) {
-			return client.New(config, client.Options{
-				Scheme: m.client.Scheme(),
-			})
+		NewClient: func(config *rest.Config, opts client.Options) (client.Client, error) {
+			opts.Scheme = m.AdminClient.Scheme()
+
+			return client.New(config, opts)
 		},
 	})
 	if err != nil {
@@ -194,11 +253,31 @@ func (m *Manager) Reconcile(ctx context.Context, request reconcile.Request) (res
 	//
 	// Register all the controllers of the soot here:
 	//
+	// Generate unique controller name prefix from TenantControlPlane to avoid metric conflicts
+	controllerNamePrefix := fmt.Sprintf("%s-%s", tcp.GetNamespace(), tcp.GetName())
+
+	writePermissions := &controllers.WritePermissions{
+		Logger:                    mgr.GetLogger().WithName("writePermissions"),
+		Client:                    mgr.GetClient(),
+		GetTenantControlPlaneFunc: m.retrieveTenantControlPlane(tcpCtx, request),
+		WebhookNamespace:          m.MigrateServiceNamespace,
+		WebhookServiceName:        m.MigrateServiceName,
+		WebhookCABundle:           m.MigrateCABundle,
+		TriggerChannel:            nil,
+		ControllerName:            fmt.Sprintf("%s-writepermissions", controllerNamePrefix),
+	}
+	if err = writePermissions.SetupWithManager(mgr); err != nil {
+		return reconcile.Result{}, err
+	}
+
 	migrate := &controllers.Migrate{
 		WebhookNamespace:          m.MigrateServiceNamespace,
 		WebhookServiceName:        m.MigrateServiceName,
 		WebhookCABundle:           m.MigrateCABundle,
 		GetTenantControlPlaneFunc: m.retrieveTenantControlPlane(tcpCtx, request),
+		Client:                    mgr.GetClient(),
+		Logger:                    mgr.GetLogger().WithName("migrate"),
+		ControllerName:            fmt.Sprintf("%s-migrate", controllerNamePrefix),
 	}
 	if err = migrate.SetupWithManager(mgr); err != nil {
 		return reconcile.Result{}, err
@@ -207,6 +286,9 @@ func (m *Manager) Reconcile(ctx context.Context, request reconcile.Request) (res
 	konnectivityAgent := &controllers.KonnectivityAgent{
 		AdminClient:               m.AdminClient,
 		GetTenantControlPlaneFunc: m.retrieveTenantControlPlane(tcpCtx, request),
+		Logger:                    mgr.GetLogger().WithName("konnectivity_agent"),
+		TriggerChannel:            make(chan event.GenericEvent),
+		ControllerName:            fmt.Sprintf("%s-konnectivity", controllerNamePrefix),
 	}
 	if err = konnectivityAgent.SetupWithManager(mgr); err != nil {
 		return reconcile.Result{}, err
@@ -215,6 +297,9 @@ func (m *Manager) Reconcile(ctx context.Context, request reconcile.Request) (res
 	kubeProxy := &controllers.KubeProxy{
 		AdminClient:               m.AdminClient,
 		GetTenantControlPlaneFunc: m.retrieveTenantControlPlane(tcpCtx, request),
+		Logger:                    mgr.GetLogger().WithName("kube_proxy"),
+		TriggerChannel:            make(chan event.GenericEvent),
+		ControllerName:            fmt.Sprintf("%s-kubeproxy", controllerNamePrefix),
 	}
 	if err = kubeProxy.SetupWithManager(mgr); err != nil {
 		return reconcile.Result{}, err
@@ -223,6 +308,9 @@ func (m *Manager) Reconcile(ctx context.Context, request reconcile.Request) (res
 	coreDNS := &controllers.CoreDNS{
 		AdminClient:               m.AdminClient,
 		GetTenantControlPlaneFunc: m.retrieveTenantControlPlane(tcpCtx, request),
+		Logger:                    mgr.GetLogger().WithName("coredns"),
+		TriggerChannel:            make(chan event.GenericEvent),
+		ControllerName:            fmt.Sprintf("%s-coredns", controllerNamePrefix),
 	}
 	if err = coreDNS.SetupWithManager(mgr); err != nil {
 		return reconcile.Result{}, err
@@ -234,6 +322,8 @@ func (m *Manager) Reconcile(ctx context.Context, request reconcile.Request) (res
 			Client: m.AdminClient,
 			Phase:  resources.PhaseUploadConfigKubeadm,
 		},
+		TriggerChannel: make(chan event.GenericEvent),
+		ControllerName: fmt.Sprintf("%s-kubeadmconfig", controllerNamePrefix),
 	}
 	if err = uploadKubeadmConfig.SetupWithManager(mgr); err != nil {
 		return reconcile.Result{}, err
@@ -245,6 +335,8 @@ func (m *Manager) Reconcile(ctx context.Context, request reconcile.Request) (res
 			Client: m.AdminClient,
 			Phase:  resources.PhaseUploadConfigKubelet,
 		},
+		TriggerChannel: make(chan event.GenericEvent),
+		ControllerName: fmt.Sprintf("%s-kubeletconfig", controllerNamePrefix),
 	}
 	if err = uploadKubeletConfig.SetupWithManager(mgr); err != nil {
 		return reconcile.Result{}, err
@@ -256,6 +348,8 @@ func (m *Manager) Reconcile(ctx context.Context, request reconcile.Request) (res
 			Client: m.AdminClient,
 			Phase:  resources.PhaseBootstrapToken,
 		},
+		TriggerChannel: make(chan event.GenericEvent),
+		ControllerName: fmt.Sprintf("%s-bootstraptoken", controllerNamePrefix),
 	}
 	if err = bootstrapToken.SetupWithManager(mgr); err != nil {
 		return reconcile.Result{}, err
@@ -267,23 +361,41 @@ func (m *Manager) Reconcile(ctx context.Context, request reconcile.Request) (res
 			Client: m.AdminClient,
 			Phase:  resources.PhaseClusterAdminRBAC,
 		},
+		TriggerChannel: make(chan event.GenericEvent),
+		ControllerName: fmt.Sprintf("%s-kubeadmrbac", controllerNamePrefix),
 	}
 	if err = kubeadmRbac.SetupWithManager(mgr); err != nil {
 		return reconcile.Result{}, err
 	}
+	completedCh := make(chan struct{})
 	// Starting the manager
 	go func() {
 		if err = mgr.Start(tcpCtx); err != nil {
 			log.FromContext(ctx).Error(err, "unable to start soot manager")
+			// The sootManagerAnnotation is used to propagate the error between reconciliations with its state:
+			// this is required to avoid mutex and prevent concurrent read/write on the soot map
+			annotationErr := m.retryTenantControlPlaneAnnotations(ctx, request, func(annotations map[string]string) {
+				annotations[sootManagerAnnotation] = sootManagerFailedAnnotation
+			})
+			if annotationErr != nil {
+				log.FromContext(ctx).Error(err, "unable to update TenantControlPlane for soot failed annotation")
+			}
 			// When the manager cannot start we're enqueuing back the request to take advantage of the backoff factor
 			// of the queue: this is a goroutine and cannot return an error since the manager is running on its own,
 			// using the sootManagerErrChan channel we can trigger a reconciliation although the TCP hadn't any change.
-			m.sootManagerErrChan <- event.GenericEvent{Object: tcp}
+			var shrunkTCP kamajiv1alpha1.TenantControlPlane
+
+			shrunkTCP.Name = tcp.Name
+			shrunkTCP.Namespace = tcp.Namespace
+
+			m.sootManagerErrChan <- event.GenericEvent{Object: &shrunkTCP}
 		}
+		close(completedCh)
 	}()
 
 	m.sootMap[request.NamespacedName.String()] = sootItem{
 		triggers: []chan event.GenericEvent{
+			writePermissions.TriggerChannel,
 			migrate.TriggerChannel,
 			konnectivityAgent.TriggerChannel,
 			kubeProxy.TriggerChannel,
@@ -292,14 +404,14 @@ func (m *Manager) Reconcile(ctx context.Context, request reconcile.Request) (res
 			uploadKubeletConfig.TriggerChannel,
 			bootstrapToken.TriggerChannel,
 		},
-		cancelFn: tcpCancelFn,
+		cancelFn:    tcpCancelFn,
+		completedCh: completedCh,
 	}
 
-	return reconcile.Result{Requeue: true}, nil
+	return reconcile.Result{RequeueAfter: time.Second}, nil
 }
 
 func (m *Manager) SetupWithManager(mgr manager.Manager) error {
-	m.client = mgr.GetClient()
 	m.sootManagerErrChan = make(chan event.GenericEvent)
 	m.sootMap = make(map[string]sootItem)
 

controllers/tcp_channel.go

@@ -1,8 +0,0 @@
-// Copyright 2022 Clastix Labs
-// SPDX-License-Identifier: Apache-2.0
-
-package controllers
-
-import "sigs.k8s.io/controller-runtime/pkg/event"
-
-type TenantControlPlaneChannel chan event.GenericEvent

controllers/telemetry_controller.go

@@ -5,13 +5,14 @@ package controllers
 
 import (
 	"context"
+	"fmt"
 	"time"
 
 	"github.com/clastix/kamaji-telemetry/api"
 	telemetry "github.com/clastix/kamaji-telemetry/pkg/client"
-	"github.com/pkg/errors"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/utils/ptr"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/log"
 
@@ -30,7 +31,7 @@ type TelemetryController struct {
 func (m *TelemetryController) retrieveControllerUID(ctx context.Context) (string, error) {
 	var defaultSvc corev1.Service
 	if err := m.Client.Get(ctx, types.NamespacedName{Name: "kubernetes", Namespace: "default"}, &defaultSvc); err != nil {
-		return "", errors.Wrap(err, "cannot start the telemetry controller")
+		return "", fmt.Errorf("cannot start the telemetry controller: %w", err)
 	}
 
 	return string(defaultSvc.UID), nil
@@ -81,7 +82,7 @@ func (m *TelemetryController) collectStats(ctx context.Context, uid string) {
 
 	for _, tcp := range tcpList.Items {
 		switch {
-		case tcp.Spec.ControlPlane.Deployment.Replicas == nil || *tcp.Spec.ControlPlane.Deployment.Replicas == 0:
+		case ptr.Deref(tcp.Status.Kubernetes.Version.Status, kamajiv1alpha1.VersionProvisioning) == kamajiv1alpha1.VersionSleeping:
 			stats.TenantControlPlanes.Sleeping++
 		case tcp.Status.Kubernetes.Version.Status != nil && *tcp.Status.Kubernetes.Version.Status == kamajiv1alpha1.VersionNotReady:
 			stats.TenantControlPlanes.NotReady++

controllers/tenantcontrolplane_controller.go

@@ -5,18 +5,20 @@ package controllers
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"strings"
 	"time"
 
 	"github.com/juju/mutex/v2"
-	"github.com/pkg/errors"
 	appsv1 "k8s.io/api/apps/v1"
 	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
 	networkingv1 "k8s.io/api/networking/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	k8stypes "k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/discovery"
+	"k8s.io/client-go/util/retry"
 	"k8s.io/client-go/util/workqueue"
 	"k8s.io/utils/clock"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -30,13 +32,17 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/predicate"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 	"sigs.k8s.io/controller-runtime/pkg/source"
+	gatewayv1 "sigs.k8s.io/gateway-api/apis/v1"
+	gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2"
 
 	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
 	"github.com/clastix/kamaji/controllers/finalizers"
 	"github.com/clastix/kamaji/controllers/utils"
+	controlplanebuilder "github.com/clastix/kamaji/internal/builders/controlplane"
 	"github.com/clastix/kamaji/internal/datastore"
 	kamajierrors "github.com/clastix/kamaji/internal/errors"
 	"github.com/clastix/kamaji/internal/resources"
+	"github.com/clastix/kamaji/internal/utilities"
 )
 
 // TenantControlPlaneReconciler reconciles a TenantControlPlane object.
@@ -44,26 +50,28 @@ type TenantControlPlaneReconciler struct {
 	Client                  client.Client
 	APIReader               client.Reader
 	Config                  TenantControlPlaneReconcilerConfig
-	TriggerChan             TenantControlPlaneChannel
+	TriggerChan             chan event.GenericEvent
 	KamajiNamespace         string
 	KamajiServiceAccount    string
 	KamajiService           string
 	KamajiMigrateImage      string
 	MaxConcurrentReconciles int
+	ReconcileTimeout        time.Duration
+	DiscoveryClient         discovery.DiscoveryInterface
 	// CertificateChan is the channel used by the CertificateLifecycleController that is checking for
 	// certificates and kubeconfig user certs validity: a generic event for the given TCP will be triggered
 	// once the validity threshold for the given certificate is reached.
-	CertificateChan CertificateChannel
+	CertificateChan chan event.GenericEvent
 
 	clock mutex.Clock
 }
 
 // TenantControlPlaneReconcilerConfig gives the necessary configuration for TenantControlPlaneReconciler.
 type TenantControlPlaneReconcilerConfig struct {
-	ReconcileTimeout     time.Duration
-	DefaultDataStoreName string
-	KineContainerImage   string
-	TmpBaseDirectory     string
+	DefaultDataStoreName    string
+	KineContainerImage      string
+	TmpBaseDirectory        string
+	CertExpirationThreshold time.Duration
 }
 
 //+kubebuilder:rbac:groups=kamaji.clastix.io,resources=tenantcontrolplanes,verbs=get;list;watch;create;update;patch;delete
@@ -75,17 +83,22 @@ type TenantControlPlaneReconcilerConfig struct {
 //+kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete
 //+kubebuilder:rbac:groups=networking.k8s.io,resources=ingresses,verbs=get;list;watch;create;update;patch;delete
 //+kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;watch;create;delete
+//+kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=httproutes,verbs=get;list;watch;create;update;patch;delete
+//+kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=grpcroutes,verbs=get;list;watch;create;update;patch;delete
+//+kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=tlsroutes,verbs=get;list;watch;create;update;patch;delete
+//+kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=gateways,verbs=get;list;watch
 
+//nolint:maintidx
 func (r *TenantControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
 	log := log.FromContext(ctx)
 
 	var cancelFn context.CancelFunc
-	ctx, cancelFn = context.WithTimeout(ctx, r.Config.ReconcileTimeout)
+	ctx, cancelFn = context.WithTimeout(ctx, r.ReconcileTimeout)
 	defer cancelFn()
 
 	tenantControlPlane, err := r.getTenantControlPlane(ctx, req.NamespacedName)()
 	if k8serrors.IsNotFound(err) {
-		log.Info("resource have been deleted, skipping")
+		log.Info("resource may have been deleted, skipping")
 
 		return reconcile.Result{}, nil
 	}
@@ -95,17 +108,23 @@ func (r *TenantControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R
 		return reconcile.Result{}, err
 	}
 
+	if utils.IsPaused(tenantControlPlane) {
+		log.Info("paused reconciliation, no further actions")
+
+		return ctrl.Result{}, nil
+	}
+
 	releaser, err := mutex.Acquire(r.mutexSpec(tenantControlPlane))
 	if err != nil {
 		switch {
-		case errors.As(err, &mutex.ErrTimeout):
+		case errors.Is(err, mutex.ErrTimeout):
 			log.Info("acquire timed out, current process is blocked by another reconciliation")
 
-			return ctrl.Result{Requeue: true}, nil
-		case errors.As(err, &mutex.ErrCancelled):
+			return ctrl.Result{RequeueAfter: time.Second}, nil
+		case errors.Is(err, mutex.ErrCancelled):
 			log.Info("acquire cancelled")
 
-			return ctrl.Result{Requeue: true}, nil
+			return ctrl.Result{RequeueAfter: time.Second}, nil
 		default:
 			log.Error(err, "acquire failed")
 
@@ -119,13 +138,15 @@ func (r *TenantControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R
 	if markedToBeDeleted && !controllerutil.ContainsFinalizer(tenantControlPlane, finalizers.DatastoreFinalizer) {
 		return ctrl.Result{}, nil
 	}
-	// Retrieving the DataStore to use for the current reconciliation
-	ds, err := r.dataStore(ctx, tenantControlPlane)
-	if err != nil {
-		if errors.Is(err, ErrMissingDataStore) {
-			log.Info(err.Error())
+	// DataStore preflight checks:
+	// 1. DataStore must exist
+	// 2. it must be ready
+	ds, dsErr := r.dataStore(ctx, tenantControlPlane)
+	if dsErr != nil {
+		if errors.Is(dsErr, ErrMissingDataStore) {
+			log.Info(dsErr.Error())
 
-			return ctrl.Result{Requeue: true}, nil
+			return ctrl.Result{RequeueAfter: time.Second}, nil
 		}
 
 		log.Error(err, "cannot retrieve the DataStore for the given instance")
@@ -133,6 +154,12 @@ func (r *TenantControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R
 		return ctrl.Result{}, err
 	}
 
+	if !ds.Status.Ready {
+		log.Info("cannot reconcile since DataStore is not ready")
+
+		return ctrl.Result{RequeueAfter: time.Second}, nil
+	}
+
 	dsConnection, err := datastore.NewStorageConnection(ctx, r.Client, *ds)
 	if err != nil {
 		log.Error(err, "cannot generate the DataStore connection for the given instance")
@@ -141,6 +168,25 @@ func (r *TenantControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R
 	}
 	defer dsConnection.Close()
 
+	dso, err := r.dataStoreOverride(ctx, tenantControlPlane)
+	if err != nil {
+		log.Error(err, "cannot retrieve the DataStoreOverrides for the given instance")
+
+		return ctrl.Result{}, err
+	}
+	dsoConnections := make(map[string]datastore.Connection, len(dso))
+	for _, ds := range dso {
+		dsoConnection, err := datastore.NewStorageConnection(ctx, r.Client, ds.DataStore)
+		if err != nil {
+			log.Error(err, "cannot generate the DataStoreOverride connection for the given instance")
+
+			return ctrl.Result{}, err
+		}
+		defer dsoConnection.Close()
+
+		dsoConnections[ds.Resource] = dsoConnection
+	}
+
 	if markedToBeDeleted && controllerutil.ContainsFinalizer(tenantControlPlane, finalizers.DatastoreFinalizer) {
 		log.Info("marked for deletion, performing clean-up")
 
@@ -167,18 +213,21 @@ func (r *TenantControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R
 	}
 
 	groupResourceBuilderConfiguration := GroupResourceBuilderConfiguration{
-		client:               r.Client,
-		log:                  log,
-		tcpReconcilerConfig:  r.Config,
-		tenantControlPlane:   *tenantControlPlane,
-		Connection:           dsConnection,
-		DataStore:            *ds,
-		KamajiNamespace:      r.KamajiNamespace,
-		KamajiServiceAccount: r.KamajiServiceAccount,
-		KamajiService:        r.KamajiService,
-		KamajiMigrateImage:   r.KamajiMigrateImage,
+		client:                        r.Client,
+		log:                           log,
+		tcpReconcilerConfig:           r.Config,
+		tenantControlPlane:            *tenantControlPlane,
+		Connection:                    dsConnection,
+		DataStore:                     *ds,
+		DataStoreOverrides:            dso,
+		DataStoreOverriedsConnections: dsoConnections,
+		KamajiNamespace:               r.KamajiNamespace,
+		KamajiServiceAccount:          r.KamajiServiceAccount,
+		KamajiService:                 r.KamajiService,
+		KamajiMigrateImage:            r.KamajiMigrateImage,
+		DiscoveryClient:               r.DiscoveryClient,
 	}
-	registeredResources := GetResources(groupResourceBuilderConfiguration)
+	registeredResources := GetResources(ctx, groupResourceBuilderConfiguration)
 
 	for _, resource := range registeredResources {
 		result, err := resources.Handle(ctx, resource, tenantControlPlane)
@@ -186,7 +235,7 @@ func (r *TenantControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R
 			if kamajierrors.ShouldReconcileErrorBeIgnored(err) {
 				log.V(1).Info("sentinel error, enqueuing back request", "error", err.Error())
 
-				return ctrl.Result{Requeue: true}, nil
+				return ctrl.Result{RequeueAfter: time.Second}, nil
 			}
 
 			log.Error(err, "handling of resource failed", "resource", resource.GetName())
@@ -202,7 +251,7 @@ func (r *TenantControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R
 			if kamajierrors.ShouldReconcileErrorBeIgnored(err) {
 				log.V(1).Info("sentinel error, enqueuing back request", "error", err.Error())
 
-				return ctrl.Result{Requeue: true}, nil
+				return ctrl.Result{RequeueAfter: time.Second}, nil
 			}
 
 			log.Error(err, "update of the resource failed", "resource", resource.GetName())
@@ -215,12 +264,29 @@ func (r *TenantControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R
 		if result == resources.OperationResultEnqueueBack {
 			log.Info("requested enqueuing back", "resources", resource.GetName())
 
-			return ctrl.Result{Requeue: true}, nil
+			return ctrl.Result{RequeueAfter: time.Second}, nil
 		}
 	}
 
 	log.Info(fmt.Sprintf("%s has been reconciled", tenantControlPlane.GetName()))
 
+	// Set ObservedGeneration only on successful reconciliation completion.
+	// This follows Cluster API conventions where ObservedGeneration indicates
+	// the controller has fully processed the given generation.
+	if err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
+		if getErr := r.Client.Get(ctx, req.NamespacedName, tenantControlPlane); getErr != nil {
+			return getErr
+		}
+
+		tenantControlPlane.Status.ObservedGeneration = tenantControlPlane.Generation
+
+		return r.Client.Status().Update(ctx, tenantControlPlane)
+	}); err != nil {
+		log.Error(err, "failed to update ObservedGeneration")
+
+		return ctrl.Result{}, err
+	}
+
 	return ctrl.Result{}, nil
 }
 
@@ -235,10 +301,10 @@ func (r *TenantControlPlaneReconciler) mutexSpec(obj client.Object) mutex.Spec {
 }
 
 // SetupWithManager sets up the controller with the Manager.
-func (r *TenantControlPlaneReconciler) SetupWithManager(mgr ctrl.Manager) error {
+func (r *TenantControlPlaneReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager) error {
 	r.clock = clock.RealClock{}
 
-	return ctrl.NewControllerManagedBy(mgr).
+	controllerBuilder := ctrl.NewControllerManagedBy(mgr).
 		WatchesRawSource(source.Channel(r.CertificateChan, handler.Funcs{GenericFunc: func(_ context.Context, genericEvent event.TypedGenericEvent[client.Object], w workqueue.TypedRateLimitingInterface[reconcile.Request]) {
 			w.AddRateLimited(ctrl.Request{
 				NamespacedName: k8stypes.NamespacedName{
@@ -288,7 +354,20 @@ func (r *TenantControlPlaneReconciler) SetupWithManager(mgr ctrl.Manager) error
 			v, ok := labels["kamaji.clastix.io/component"]
 
 			return ok && v == "migrate"
-		}))).
+		})))
+
+	// Conditionally add Gateway API ownership if available
+	if utilities.AreGatewayResourcesAvailable(ctx, r.Client, r.DiscoveryClient) {
+		controllerBuilder = controllerBuilder.
+			Owns(&gatewayv1.HTTPRoute{}).
+			Owns(&gatewayv1.GRPCRoute{}).
+			Owns(&gatewayv1alpha2.TLSRoute{}).
+			Watches(&gatewayv1.Gateway{}, handler.EnqueueRequestsFromMapFunc(func(_ context.Context, object client.Object) []reconcile.Request {
+				return nil
+			}))
+	}
+
+	return controllerBuilder.
 		WithOptions(controller.Options{
 			MaxConcurrentReconciles: r.MaxConcurrentReconciles,
 		}).
@@ -327,8 +406,26 @@ func (r *TenantControlPlaneReconciler) dataStore(ctx context.Context, tenantCont
 
 	var ds kamajiv1alpha1.DataStore
 	if err := r.Client.Get(ctx, k8stypes.NamespacedName{Name: tenantControlPlane.Spec.DataStore}, &ds); err != nil {
-		return nil, errors.Wrap(err, "cannot retrieve *kamajiv1alpha.DataStore object")
+		return nil, fmt.Errorf("cannot retrieve *kamajiv1alpha.DataStore object: %w", err)
 	}
 
 	return &ds, nil
 }
+
+func (r *TenantControlPlaneReconciler) dataStoreOverride(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) ([]controlplanebuilder.DataStoreOverrides, error) {
+	datastores := make([]controlplanebuilder.DataStoreOverrides, 0, len(tenantControlPlane.Spec.DataStoreOverrides))
+
+	for _, dso := range tenantControlPlane.Spec.DataStoreOverrides {
+		var ds kamajiv1alpha1.DataStore
+		if err := r.Client.Get(ctx, k8stypes.NamespacedName{Name: dso.DataStore}, &ds); err != nil {
+			return nil, fmt.Errorf("cannot retrieve *kamajiv1alpha.DataStore object: %w", err)
+		}
+		if ds.Spec.Driver != kamajiv1alpha1.EtcdDriver {
+			return nil, errors.New("DataStoreOverrides can only use ETCD driver")
+		}
+
+		datastores = append(datastores, controlplanebuilder.DataStoreOverrides{Resource: dso.Resource, DataStore: ds})
+	}
+
+	return datastores, nil
+}

controllers/utils/is_paused.go

@@ -0,0 +1,19 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package utils
+
+import (
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	"github.com/clastix/kamaji/api/v1alpha1"
+)
+
+func IsPaused(obj client.Object) bool {
+	if obj.GetAnnotations() == nil {
+		return false
+	}
+	_, paused := obj.GetAnnotations()[v1alpha1.PausedReconciliationAnnotation]
+
+	return paused
+}

controllers/utils/trigger_channel.go

@@ -0,0 +1,26 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package utils
+
+import (
+	"context"
+	"time"
+
+	"sigs.k8s.io/controller-runtime/pkg/event"
+	"sigs.k8s.io/controller-runtime/pkg/log"
+
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+)
+
+func TriggerChannel(ctx context.Context, receiver chan event.GenericEvent, tcp kamajiv1alpha1.TenantControlPlane) {
+	deadlineCtx, cancelFn := context.WithTimeout(ctx, 10*time.Second)
+	defer cancelFn()
+
+	select {
+	case receiver <- event.GenericEvent{Object: &tcp}:
+		return
+	case <-deadlineCtx.Done():
+		log.FromContext(ctx).Error(deadlineCtx.Err(), "cannot send due to timeout")
+	}
+}

deploy/kamaji-aws.env

@@ -18,7 +18,7 @@ export KAMAJI_NAMESPACE=kamaji-system
 export TENANT_NAMESPACE=tenant-00
 export TENANT_NAME=tenant-00
 export TENANT_DOMAIN=internal.kamaji.aws.com
-export TENANT_VERSION=v1.30.2
+export TENANT_VERSION=v1.31.0
 export TENANT_PORT=6443 # port used to expose the tenant api server
 export TENANT_PROXY_PORT=8132 # port used to expose the konnectivity server
 export TENANT_POD_CIDR=10.36.0.0/16

deploy/kamaji-azure.env

@@ -15,7 +15,7 @@ export KAMAJI_NAMESPACE=kamaji-system
 export TENANT_NAMESPACE=default
 export TENANT_NAME=tenant-00
 export TENANT_DOMAIN=$KAMAJI_REGION.cloudapp.azure.com
-export TENANT_VERSION=v1.26.0
+export TENANT_VERSION=v1.31.0
 export TENANT_PORT=6443 # port used to expose the tenant api server
 export TENANT_PROXY_PORT=8132 # port used to expose the konnectivity server
 export TENANT_POD_CIDR=10.36.0.0/16

deploy/kamaji.env

@@ -5,7 +5,7 @@ export KAMAJI_NAMESPACE=kamaji-system
 export TENANT_NAMESPACE=default
 export TENANT_NAME=tenant-00
 export TENANT_DOMAIN=clastix.labs
-export TENANT_VERSION=v1.26.0
+export TENANT_VERSION=v1.31.0
 export TENANT_PORT=6443 # port used to expose the tenant api server
 export TENANT_PROXY_PORT=8132 # port used to expose the konnectivity server
 export TENANT_POD_CIDR=10.36.0.0/16

deploy/kind/Makefile

@@ -1,36 +0,0 @@
-kind_path := $(patsubst %/,%,$(dir $(abspath $(lastword $(MAKEFILE_LIST)))))
-
-include ../etcd/Makefile
-
-.PHONY: kind ingress-nginx
-
-.DEFAULT_GOAL := kamaji
-
-prometheus-stack:
-	helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
-	helm repo update
-	helm install prometheus-stack --create-namespace -n monitoring prometheus-community/kube-prometheus-stack
-
-reqs: kind ingress-nginx cert-manager
-
-cert-manager:
-	@kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.10.1/cert-manager.yaml
-
-kamaji: reqs
-	helm install kamaji --create-namespace -n kamaji-system $(kind_path)/../../charts/kamaji
-
-destroy: kind/destroy etcd-certificates/cleanup
-
-kind:
-	@kind create cluster --config $(kind_path)/kind-kamaji.yaml
-
-kind/destroy:
-	@kind delete cluster --name kamaji
-
-ingress-nginx: ingress-nginx-install
-
-ingress-nginx-install:
-	kubectl apply -f $(kind_path)/nginx-deploy.yaml
-
-kamaji-kind-worker-join:
-	$(kind_path)/join-node.bash

deploy/kind/join-node.bash

@@ -1,36 +0,0 @@
-#!/bin/bash
-
-set -e
-
-# Constants
-export DOCKER_IMAGE_NAME="kindest/node"
-export DOCKER_NETWORK="kind"
-
-# Variables
-export KUBERNETES_VERSION=${1:-v1.23.4}
-export KUBECONFIG="${KUBECONFIG:-/tmp/kubeconfig}"
-
-if [ -z $2 ]
-then
-    MAPPING_PORT=""
-else
-    MAPPING_PORT="-p ${2}:80"
-fi
-
-clear
-echo "Welcome to join a new node to the Kind network"
-
-echo -ne "\nChecking right kubeconfig\n"
-kubectl cluster-info
-echo "Are you pointing to the right tenant control plane? (Type return to continue)"
-read
-
-JOIN_CMD="$(kubeadm --kubeconfig=${KUBECONFIG} token create --print-join-command) --ignore-preflight-errors=SystemVerification"
-echo "Deploying new node..."
-NODE=$(docker run -d --privileged -v /lib/modules:/lib/modules:ro -v /var --net $DOCKER_NETWORK $MAPPING_PORT $DOCKER_IMAGE_NAME:$KUBERNETES_VERSION)
-sleep 10
-echo "Joining new node..."
-docker exec -e JOIN_CMD="$JOIN_CMD" $NODE /bin/bash -c "$JOIN_CMD"
-
-echo "Node has joined! Remember to install the kind-net CNI by issuing the following command:"
-echo "  $: kubectl apply -f https://raw.githubusercontent.com/aojea/kindnet/master/install-kindnet.yaml"

deploy/kind/kind-kamaji.yaml

@@ -1,37 +0,0 @@
-kind: Cluster
-apiVersion: kind.x-k8s.io/v1alpha4
-name: kamaji
-nodes:
-- role: control-plane
-  image: kindest/node:v1.23.4
-  kubeadmConfigPatches:
-  - |
-    kind: InitConfiguration
-    nodeRegistration:
-      kubeletExtraArgs:
-        node-labels: "ingress-ready=true"
-  ## required for Cluster API local development
-  extraMounts:
-    - hostPath: /var/run/docker.sock
-      containerPath: /var/run/docker.sock
-  extraPortMappings:
-    ## expose port 80 of the node to port 80 on the host
-  - containerPort: 80
-    hostPort: 80
-    protocol: TCP
-    ## expose port 443 of the node to port 443 on the host
-  - containerPort: 443
-    hostPort: 443
-    protocol: TCP
-    ## expose port 31132 of the node to port 31132 on the host for konnectivity
-  - containerPort: 31132
-    hostPort: 31132
-    protocol: TCP
-    ## expose port 31443 of the node to port 31443 on the host
-  - containerPort: 31443
-    hostPort: 31443
-    protocol: TCP
-    ## expose port 6443 of the node to port 8443 on the host
-  - containerPort: 6443
-    hostPort: 8443
-    protocol: TCP

deploy/kind/nginx-deploy.yaml

@@ -1,694 +0,0 @@
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: ingress-nginx
-  labels:
-    app.kubernetes.io/name: ingress-nginx
-    app.kubernetes.io/instance: ingress-nginx
-
----
-# Source: ingress-nginx/templates/controller-serviceaccount.yaml
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  labels:
-    helm.sh/chart: ingress-nginx-4.0.10
-    app.kubernetes.io/name: ingress-nginx
-    app.kubernetes.io/instance: ingress-nginx
-    app.kubernetes.io/version: 1.1.0
-    app.kubernetes.io/managed-by: Helm
-    app.kubernetes.io/component: controller
-  name: ingress-nginx
-  namespace: ingress-nginx
-automountServiceAccountToken: true
----
-# Source: ingress-nginx/templates/controller-configmap.yaml
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  labels:
-    helm.sh/chart: ingress-nginx-4.0.10
-    app.kubernetes.io/name: ingress-nginx
-    app.kubernetes.io/instance: ingress-nginx
-    app.kubernetes.io/version: 1.1.0
-    app.kubernetes.io/managed-by: Helm
-    app.kubernetes.io/component: controller
-  name: ingress-nginx-controller
-  namespace: ingress-nginx
-data:
-  allow-snippet-annotations: 'true'
----
-# Source: ingress-nginx/templates/clusterrole.yaml
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  labels:
-    helm.sh/chart: ingress-nginx-4.0.10
-    app.kubernetes.io/name: ingress-nginx
-    app.kubernetes.io/instance: ingress-nginx
-    app.kubernetes.io/version: 1.1.0
-    app.kubernetes.io/managed-by: Helm
-  name: ingress-nginx
-rules:
-  - apiGroups:
-      - ''
-    resources:
-      - configmaps
-      - endpoints
-      - nodes
-      - pods
-      - secrets
-      - namespaces
-    verbs:
-      - list
-      - watch
-  - apiGroups:
-      - ''
-    resources:
-      - nodes
-    verbs:
-      - get
-  - apiGroups:
-      - ''
-    resources:
-      - services
-    verbs:
-      - get
-      - list
-      - watch
-  - apiGroups:
-      - networking.k8s.io
-    resources:
-      - ingresses
-    verbs:
-      - get
-      - list
-      - watch
-  - apiGroups:
-      - ''
-    resources:
-      - events
-    verbs:
-      - create
-      - patch
-  - apiGroups:
-      - networking.k8s.io
-    resources:
-      - ingresses/status
-    verbs:
-      - update
-  - apiGroups:
-      - networking.k8s.io
-    resources:
-      - ingressclasses
-    verbs:
-      - get
-      - list
-      - watch
----
-# Source: ingress-nginx/templates/clusterrolebinding.yaml
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  labels:
-    helm.sh/chart: ingress-nginx-4.0.10
-    app.kubernetes.io/name: ingress-nginx
-    app.kubernetes.io/instance: ingress-nginx
-    app.kubernetes.io/version: 1.1.0
-    app.kubernetes.io/managed-by: Helm
-  name: ingress-nginx
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: ingress-nginx
-subjects:
-  - kind: ServiceAccount
-    name: ingress-nginx
-    namespace: ingress-nginx
----
-# Source: ingress-nginx/templates/controller-role.yaml
-apiVersion: rbac.authorization.k8s.io/v1
-kind: Role
-metadata:
-  labels:
-    helm.sh/chart: ingress-nginx-4.0.10
-    app.kubernetes.io/name: ingress-nginx
-    app.kubernetes.io/instance: ingress-nginx
-    app.kubernetes.io/version: 1.1.0
-    app.kubernetes.io/managed-by: Helm
-    app.kubernetes.io/component: controller
-  name: ingress-nginx
-  namespace: ingress-nginx
-rules:
-  - apiGroups:
-      - ''
-    resources:
-      - namespaces
-    verbs:
-      - get
-  - apiGroups:
-      - ''
-    resources:
-      - configmaps
-      - pods
-      - secrets
-      - endpoints
-    verbs:
-      - get
-      - list
-      - watch
-  - apiGroups:
-      - ''
-    resources:
-      - services
-    verbs:
-      - get
-      - list
-      - watch
-  - apiGroups:
-      - networking.k8s.io
-    resources:
-      - ingresses
-    verbs:
-      - get
-      - list
-      - watch
-  - apiGroups:
-      - networking.k8s.io
-    resources:
-      - ingresses/status
-    verbs:
-      - update
-  - apiGroups:
-      - networking.k8s.io
-    resources:
-      - ingressclasses
-    verbs:
-      - get
-      - list
-      - watch
-  - apiGroups:
-      - ''
-    resources:
-      - configmaps
-    resourceNames:
-      - ingress-controller-leader
-    verbs:
-      - get
-      - update
-  - apiGroups:
-      - ''
-    resources:
-      - configmaps
-    verbs:
-      - create
-  - apiGroups:
-      - ''
-    resources:
-      - events
-    verbs:
-      - create
-      - patch
----
-# Source: ingress-nginx/templates/controller-rolebinding.yaml
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  labels:
-    helm.sh/chart: ingress-nginx-4.0.10
-    app.kubernetes.io/name: ingress-nginx
-    app.kubernetes.io/instance: ingress-nginx
-    app.kubernetes.io/version: 1.1.0
-    app.kubernetes.io/managed-by: Helm
-    app.kubernetes.io/component: controller
-  name: ingress-nginx
-  namespace: ingress-nginx
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: Role
-  name: ingress-nginx
-subjects:
-  - kind: ServiceAccount
-    name: ingress-nginx
-    namespace: ingress-nginx
----
-# Source: ingress-nginx/templates/controller-service-webhook.yaml
-apiVersion: v1
-kind: Service
-metadata:
-  labels:
-    helm.sh/chart: ingress-nginx-4.0.10
-    app.kubernetes.io/name: ingress-nginx
-    app.kubernetes.io/instance: ingress-nginx
-    app.kubernetes.io/version: 1.1.0
-    app.kubernetes.io/managed-by: Helm
-    app.kubernetes.io/component: controller
-  name: ingress-nginx-controller-admission
-  namespace: ingress-nginx
-spec:
-  type: ClusterIP
-  ports:
-    - name: https-webhook
-      port: 443
-      targetPort: webhook
-      appProtocol: https
-  selector:
-    app.kubernetes.io/name: ingress-nginx
-    app.kubernetes.io/instance: ingress-nginx
-    app.kubernetes.io/component: controller
----
-# Source: ingress-nginx/templates/controller-service.yaml
-apiVersion: v1
-kind: Service
-metadata:
-  labels:
-    helm.sh/chart: ingress-nginx-4.0.10
-    app.kubernetes.io/name: ingress-nginx
-    app.kubernetes.io/instance: ingress-nginx
-    app.kubernetes.io/version: 1.1.0
-    app.kubernetes.io/managed-by: Helm
-    app.kubernetes.io/component: controller
-  name: ingress-nginx-controller
-  namespace: ingress-nginx
-spec:
-  type: NodePort
-  ipFamilyPolicy: SingleStack
-  ipFamilies:
-    - IPv4
-  ports:
-    - name: http
-      port: 80
-      protocol: TCP
-      targetPort: http
-      appProtocol: http
-    - name: https
-      port: 443
-      protocol: TCP
-      targetPort: https
-      appProtocol: https
-  selector:
-    app.kubernetes.io/name: ingress-nginx
-    app.kubernetes.io/instance: ingress-nginx
-    app.kubernetes.io/component: controller
----
-# Source: ingress-nginx/templates/controller-deployment.yaml
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  labels:
-    helm.sh/chart: ingress-nginx-4.0.10
-    app.kubernetes.io/name: ingress-nginx
-    app.kubernetes.io/instance: ingress-nginx
-    app.kubernetes.io/version: 1.1.0
-    app.kubernetes.io/managed-by: Helm
-    app.kubernetes.io/component: controller
-  name: ingress-nginx-controller
-  namespace: ingress-nginx
-spec:
-  selector:
-    matchLabels:
-      app.kubernetes.io/name: ingress-nginx
-      app.kubernetes.io/instance: ingress-nginx
-      app.kubernetes.io/component: controller
-  revisionHistoryLimit: 10
-  strategy:
-    rollingUpdate:
-      maxUnavailable: 1
-    type: RollingUpdate
-  minReadySeconds: 0
-  template:
-    metadata:
-      labels:
-        app.kubernetes.io/name: ingress-nginx
-        app.kubernetes.io/instance: ingress-nginx
-        app.kubernetes.io/component: controller
-    spec:
-      dnsPolicy: ClusterFirst
-      containers:
-        - name: controller
-          image: k8s.gcr.io/ingress-nginx/controller:v1.1.0@sha256:f766669fdcf3dc26347ed273a55e754b427eb4411ee075a53f30718b4499076a
-          imagePullPolicy: IfNotPresent
-          lifecycle:
-            preStop:
-              exec:
-                command:
-                  - /wait-shutdown
-          args:
-            - /nginx-ingress-controller
-            - --election-id=ingress-controller-leader
-            - --controller-class=k8s.io/ingress-nginx
-            - --configmap=$(POD_NAMESPACE)/ingress-nginx-controller
-            - --validating-webhook=:8443
-            - --validating-webhook-certificate=/usr/local/certificates/cert
-            - --validating-webhook-key=/usr/local/certificates/key
-            - --watch-ingress-without-class=true
-            - --publish-status-address=localhost
-            - --enable-ssl-passthrough=true
-          securityContext:
-            capabilities:
-              drop:
-                - ALL
-              add:
-                - NET_BIND_SERVICE
-            runAsUser: 101
-            allowPrivilegeEscalation: true
-          env:
-            - name: POD_NAME
-              valueFrom:
-                fieldRef:
-                  fieldPath: metadata.name
-            - name: POD_NAMESPACE
-              valueFrom:
-                fieldRef:
-                  fieldPath: metadata.namespace
-            - name: LD_PRELOAD
-              value: /usr/local/lib/libmimalloc.so
-          livenessProbe:
-            failureThreshold: 5
-            httpGet:
-              path: /healthz
-              port: 10254
-              scheme: HTTP
-            initialDelaySeconds: 10
-            periodSeconds: 10
-            successThreshold: 1
-            timeoutSeconds: 1
-          readinessProbe:
-            failureThreshold: 3
-            httpGet:
-              path: /healthz
-              port: 10254
-              scheme: HTTP
-            initialDelaySeconds: 10
-            periodSeconds: 10
-            successThreshold: 1
-            timeoutSeconds: 1
-          ports:
-            - name: http
-              containerPort: 80
-              protocol: TCP
-              hostPort: 80
-            - name: https
-              containerPort: 443
-              protocol: TCP
-              hostPort: 443
-            - name: webhook
-              containerPort: 8443
-              protocol: TCP
-          volumeMounts:
-            - name: webhook-cert
-              mountPath: /usr/local/certificates/
-              readOnly: true
-          resources:
-            requests:
-              cpu: 100m
-              memory: 90Mi
-      nodeSelector:
-        ingress-ready: 'true'
-        kubernetes.io/os: linux
-      tolerations:
-        - effect: NoSchedule
-          key: node-role.kubernetes.io/master
-          operator: Equal
-      serviceAccountName: ingress-nginx
-      terminationGracePeriodSeconds: 0
-      volumes:
-        - name: webhook-cert
-          secret:
-            secretName: ingress-nginx-admission
----
-# Source: ingress-nginx/templates/controller-ingressclass.yaml
-# We don't support namespaced ingressClass yet
-# So a ClusterRole and a ClusterRoleBinding is required
-apiVersion: networking.k8s.io/v1
-kind: IngressClass
-metadata:
-  labels:
-    helm.sh/chart: ingress-nginx-4.0.10
-    app.kubernetes.io/name: ingress-nginx
-    app.kubernetes.io/instance: ingress-nginx
-    app.kubernetes.io/version: 1.1.0
-    app.kubernetes.io/managed-by: Helm
-    app.kubernetes.io/component: controller
-  name: nginx
-  namespace: ingress-nginx
-spec:
-  controller: k8s.io/ingress-nginx
----
-# Source: ingress-nginx/templates/admission-webhooks/validating-webhook.yaml
-# before changing this value, check the required kubernetes version
-# https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#prerequisites
-apiVersion: admissionregistration.k8s.io/v1
-kind: ValidatingWebhookConfiguration
-metadata:
-  labels:
-    helm.sh/chart: ingress-nginx-4.0.10
-    app.kubernetes.io/name: ingress-nginx
-    app.kubernetes.io/instance: ingress-nginx
-    app.kubernetes.io/version: 1.1.0
-    app.kubernetes.io/managed-by: Helm
-    app.kubernetes.io/component: admission-webhook
-  name: ingress-nginx-admission
-webhooks:
-  - name: validate.nginx.ingress.kubernetes.io
-    matchPolicy: Equivalent
-    rules:
-      - apiGroups:
-          - networking.k8s.io
-        apiVersions:
-          - v1
-        operations:
-          - CREATE
-          - UPDATE
-        resources:
-          - ingresses
-    failurePolicy: Fail
-    sideEffects: None
-    admissionReviewVersions:
-      - v1
-    clientConfig:
-      service:
-        namespace: ingress-nginx
-        name: ingress-nginx-controller-admission
-        path: /networking/v1/ingresses
----
-# Source: ingress-nginx/templates/admission-webhooks/job-patch/serviceaccount.yaml
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: ingress-nginx-admission
-  namespace: ingress-nginx
-  annotations:
-    helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade
-    helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
-  labels:
-    helm.sh/chart: ingress-nginx-4.0.10
-    app.kubernetes.io/name: ingress-nginx
-    app.kubernetes.io/instance: ingress-nginx
-    app.kubernetes.io/version: 1.1.0
-    app.kubernetes.io/managed-by: Helm
-    app.kubernetes.io/component: admission-webhook
----
-# Source: ingress-nginx/templates/admission-webhooks/job-patch/clusterrole.yaml
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: ingress-nginx-admission
-  annotations:
-    helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade
-    helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
-  labels:
-    helm.sh/chart: ingress-nginx-4.0.10
-    app.kubernetes.io/name: ingress-nginx
-    app.kubernetes.io/instance: ingress-nginx
-    app.kubernetes.io/version: 1.1.0
-    app.kubernetes.io/managed-by: Helm
-    app.kubernetes.io/component: admission-webhook
-rules:
-  - apiGroups:
-      - admissionregistration.k8s.io
-    resources:
-      - validatingwebhookconfigurations
-    verbs:
-      - get
-      - update
----
-# Source: ingress-nginx/templates/admission-webhooks/job-patch/clusterrolebinding.yaml
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: ingress-nginx-admission
-  annotations:
-    helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade
-    helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
-  labels:
-    helm.sh/chart: ingress-nginx-4.0.10
-    app.kubernetes.io/name: ingress-nginx
-    app.kubernetes.io/instance: ingress-nginx
-    app.kubernetes.io/version: 1.1.0
-    app.kubernetes.io/managed-by: Helm
-    app.kubernetes.io/component: admission-webhook
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: ingress-nginx-admission
-subjects:
-  - kind: ServiceAccount
-    name: ingress-nginx-admission
-    namespace: ingress-nginx
----
-# Source: ingress-nginx/templates/admission-webhooks/job-patch/role.yaml
-apiVersion: rbac.authorization.k8s.io/v1
-kind: Role
-metadata:
-  name: ingress-nginx-admission
-  namespace: ingress-nginx
-  annotations:
-    helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade
-    helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
-  labels:
-    helm.sh/chart: ingress-nginx-4.0.10
-    app.kubernetes.io/name: ingress-nginx
-    app.kubernetes.io/instance: ingress-nginx
-    app.kubernetes.io/version: 1.1.0
-    app.kubernetes.io/managed-by: Helm
-    app.kubernetes.io/component: admission-webhook
-rules:
-  - apiGroups:
-      - ''
-    resources:
-      - secrets
-    verbs:
-      - get
-      - create
----
-# Source: ingress-nginx/templates/admission-webhooks/job-patch/rolebinding.yaml
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: ingress-nginx-admission
-  namespace: ingress-nginx
-  annotations:
-    helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade
-    helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
-  labels:
-    helm.sh/chart: ingress-nginx-4.0.10
-    app.kubernetes.io/name: ingress-nginx
-    app.kubernetes.io/instance: ingress-nginx
-    app.kubernetes.io/version: 1.1.0
-    app.kubernetes.io/managed-by: Helm
-    app.kubernetes.io/component: admission-webhook
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: Role
-  name: ingress-nginx-admission
-subjects:
-  - kind: ServiceAccount
-    name: ingress-nginx-admission
-    namespace: ingress-nginx
----
-# Source: ingress-nginx/templates/admission-webhooks/job-patch/job-createSecret.yaml
-apiVersion: batch/v1
-kind: Job
-metadata:
-  name: ingress-nginx-admission-create
-  namespace: ingress-nginx
-  annotations:
-    helm.sh/hook: pre-install,pre-upgrade
-    helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
-  labels:
-    helm.sh/chart: ingress-nginx-4.0.10
-    app.kubernetes.io/name: ingress-nginx
-    app.kubernetes.io/instance: ingress-nginx
-    app.kubernetes.io/version: 1.1.0
-    app.kubernetes.io/managed-by: Helm
-    app.kubernetes.io/component: admission-webhook
-spec:
-  template:
-    metadata:
-      name: ingress-nginx-admission-create
-      labels:
-        helm.sh/chart: ingress-nginx-4.0.10
-        app.kubernetes.io/name: ingress-nginx
-        app.kubernetes.io/instance: ingress-nginx
-        app.kubernetes.io/version: 1.1.0
-        app.kubernetes.io/managed-by: Helm
-        app.kubernetes.io/component: admission-webhook
-    spec:
-      containers:
-        - name: create
-          image: k8s.gcr.io/ingress-nginx/kube-webhook-certgen:v1.1.1@sha256:64d8c73dca984af206adf9d6d7e46aa550362b1d7a01f3a0a91b20cc67868660
-          imagePullPolicy: IfNotPresent
-          args:
-            - create
-            - --host=ingress-nginx-controller-admission,ingress-nginx-controller-admission.$(POD_NAMESPACE).svc
-            - --namespace=$(POD_NAMESPACE)
-            - --secret-name=ingress-nginx-admission
-          env:
-            - name: POD_NAMESPACE
-              valueFrom:
-                fieldRef:
-                  fieldPath: metadata.namespace
-          securityContext:
-            allowPrivilegeEscalation: false
-      restartPolicy: OnFailure
-      serviceAccountName: ingress-nginx-admission
-      nodeSelector:
-        kubernetes.io/os: linux
-      securityContext:
-        runAsNonRoot: true
-        runAsUser: 2000
----
-# Source: ingress-nginx/templates/admission-webhooks/job-patch/job-patchWebhook.yaml
-apiVersion: batch/v1
-kind: Job
-metadata:
-  name: ingress-nginx-admission-patch
-  namespace: ingress-nginx
-  annotations:
-    helm.sh/hook: post-install,post-upgrade
-    helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
-  labels:
-    helm.sh/chart: ingress-nginx-4.0.10
-    app.kubernetes.io/name: ingress-nginx
-    app.kubernetes.io/instance: ingress-nginx
-    app.kubernetes.io/version: 1.1.0
-    app.kubernetes.io/managed-by: Helm
-    app.kubernetes.io/component: admission-webhook
-spec:
-  template:
-    metadata:
-      name: ingress-nginx-admission-patch
-      labels:
-        helm.sh/chart: ingress-nginx-4.0.10
-        app.kubernetes.io/name: ingress-nginx
-        app.kubernetes.io/instance: ingress-nginx
-        app.kubernetes.io/version: 1.1.0
-        app.kubernetes.io/managed-by: Helm
-        app.kubernetes.io/component: admission-webhook
-    spec:
-      containers:
-        - name: patch
-          image: k8s.gcr.io/ingress-nginx/kube-webhook-certgen:v1.1.1@sha256:64d8c73dca984af206adf9d6d7e46aa550362b1d7a01f3a0a91b20cc67868660
-          imagePullPolicy: IfNotPresent
-          args:
-            - patch
-            - --webhook-name=ingress-nginx-admission
-            - --namespace=$(POD_NAMESPACE)
-            - --patch-mutating=false
-            - --secret-name=ingress-nginx-admission
-            - --patch-failure-policy=Fail
-          env:
-            - name: POD_NAMESPACE
-              valueFrom:
-                fieldRef:
-                  fieldPath: metadata.namespace
-          securityContext:
-            allowPrivilegeEscalation: false
-      restartPolicy: OnFailure
-      serviceAccountName: ingress-nginx-admission
-      nodeSelector:
-        kubernetes.io/os: linux
-      securityContext:
-        runAsNonRoot: true
-        runAsUser: 2000

deploy/kine/nats/Makefile

@@ -5,7 +5,7 @@ NAMESPACE:=nats-system
 .PHONY: helm
 HELM = $(shell pwd)/../../../bin/helm
 helm: ## Download helm locally if necessary.
-	$(call go-install-tool,$(HELM),helm.sh/helm/v3/cmd/helm@v3.9.0)
+	$(call go-install-tool,$(HELM),helm.sh/helm/v3/cmd/helm@v4.1.1)
 
 nats: nats-certificates nats-secret nats-deployment
 

docs/content/cluster-api/cluster-autoscaler.md

@@ -0,0 +1,174 @@
+# Cluster Autoscaler
+
+The [Cluster Autoscaler](https://github.com/kubernetes/autoscaler) is a tool that automatically adjusts the size of a Kubernetes cluster so that all pods have a place to run and no unneeded nodes remain.
+
+When pods are unschedulable because there are not enough resources, the Cluster Autoscaler scales up the cluster. When nodes are underutilized, the Cluster Autoscaler scales the cluster down.
+
+Cluster API supports the Cluster Autoscaler. See the [Cluster Autoscaler on Cluster API](https://cluster-api.sigs.k8s.io/tasks/automated-machine-management/autoscaling) for more information.
+
+## Getting started with the Cluster Autoscaler on Kamaji
+
+Kamaji supports the Cluster Autoscaler through Cluster API. There are several ways to run the Cluster Autoscaler with Cluster API. In this guide, we leverage the unique features of Kamaji to run the Cluster Autoscaler as part of the Hosted Control Plane.
+
+In other words, the Cluster Autoscaler runs as a pod in the Kamaji Management Cluster, alongside the Tenant Control Plane pods, and connects directly to the API server of the workload cluster. This approach hides sensitive data from the tenant. It works by mounting the kubeconfig of the tenant cluster into the Cluster Autoscaler pod.
+
+### Create the workload cluster
+
+Create a workload cluster using the Kamaji Control Plane Provider and the Infrastructure Provider of your choice. The following example creates a workload cluster using the vSphere Infrastructure Provider.
+
+The template file [`capi-kamaji-vsphere-autoscaler-template.yaml`](https://raw.githubusercontent.com/clastix/cluster-api-control-plane-provider-kamaji/master/templates/vsphere/capi-kamaji-vsphere-autoscaler-template.yaml) provides a full example of a cluster with the autoscaler enabled. You can generate the cluster manifest using `clusterctl`.
+
+Before doing so, list all the variables in the template file:
+
+```bash
+cat capi-kamaji-vsphere-autoscaler-template.yaml | clusterctl generate yaml --list-variables
+```
+
+Fill them with the desired values and generate the manifest:
+
+```bash
+clusterctl generate yaml \
+    --from capi-kamaji-vsphere-autoscaler-template.yaml \
+    > capi-kamaji-vsphere-cluster.yaml
+```
+
+Apply the generated manifest to create the ClusterClass:
+
+```bash
+kubectl apply -f capi-kamaji-vsphere-cluster.yaml
+```
+
+### Install the Cluster Autoscaler
+
+Install the Cluster Autoscaler via Helm in the Management Cluster, in the same namespace where the workload cluster is deployed.
+
+!!! info "Options for installing the Cluster Autoscaler"
+    The Cluster Autoscaler works on a single cluster, meaning every cluster must have its own Cluster Autoscaler instance. This can be addressed by leveraging Project Sveltos automations to deploy a Cluster Autoscaler instance for each Kamaji Cluster API instance.
+
+```bash
+helm repo add autoscaler https://kubernetes.github.io/autoscaler
+helm repo update
+helm upgrade --install ${CLUSTER_NAME}-autoscaler autoscaler/cluster-autoscaler \
+    --set cloudProvider=clusterapi \
+    --set autodiscvovery.namespace=default \
+    --set "autoDiscovery.labels[0].autoscaling=enabled" \
+    --set clusterAPIKubeconfigSecret=${CLUSTER_NAME}-kubeconfig \
+    --set clusterAPIMode=kubeconfig-incluster
+```
+
+The `autoDiscovery.labels` values are used to dynamically select clusters to autoscale.
+
+These labels must be set on the workload cluster, specifically in the `Cluster` and `MachineDeployment` resources.
+
+```yaml
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: Cluster
+metadata:
+  labels:
+    cluster.x-k8s.io/cluster-name: sample
+    # Cluster Autoscaler labels
+    autoscaling: enabled
+  name: sample
+
+# other fields omitted for brevity
+---
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: MachineDeployment
+metadata:
+  annotations:
+    # Cluster Autoscaler annotations
+    cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size: "0"
+    cluster.x-k8s.io/cluster-api-autoscaler-node-group-max-size: "6"
+    capacity.cluster-autoscaler.kubernetes.io/cpu: "2"  # YMMV
+    capacity.cluster-autoscaler.kubernetes.io/memory: 4Gi  # YMMV
+    capacity.cluster-autoscaler.kubernetes.io/maxPods: "110"  # YMMV
+  labels:
+    cluster.x-k8s.io/cluster-name: sample
+    # Cluster Autoscaler labels
+    autoscaling: enabled
+  name: sample-md-0
+
+# other fields omitted for brevity
+---
+# other Cluster API resources omitted for brevity
+```
+
+### Verify the Cluster Autoscaler
+
+To verify that the Cluster Autoscaler is working as expected, deploy a workload in the Tenant cluster with specific CPU requirements to simulate resource demand.
+
+```yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: hello-node
+  name: hello-node
+  namespace: default
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: hello-node
+  template:
+    metadata:
+      labels:
+        app: hello-node
+    spec:
+      containers:
+      - image: quay.io/google-containers/pause-amd64:3.0
+        imagePullPolicy: IfNotPresent
+        name: pause-amd64
+        resources:
+          limits:
+            cpu: 500m
+```
+
+Apply the workload to the Tenant cluster and simulate a load spike by increasing the number of replicas. The Cluster Autoscaler should scale up the cluster to accommodate the workload. Cooldown times must be configured correctly on a per-cluster basis.
+
+!!! warning "Possible Resource Wastage"
+    With the Cluster Autoscaler, new machines may be created very quickly, which can lead to over-provisioning and potentially wasted resources. The official Cluster Autoscaler documentation should be consulted to configure appropriate values based on your infrastructure and provisioning times.
+
+##  `ProvisioningRequest` support
+
+The [ProvisioningRequest](https://github.com/kubernetes/autoscaler/blob/cluster-autoscaler-1.34.1/cluster-autoscaler/proposals/provisioning-request.md) introduces a Kubernetes-native way for Cluster Autoscaler to request new capacity without talking directly to cloud provider APIs.
+Instead of embedding provider-specific logic, the autoscaler simply describes the capacity it needs, and an external provisioner decides how to create the required nodes.
+This makes scaling portable across clouds, on-prem platforms, and custom provisioning systems, while greatly reducing complexity inside the autoscaler.
+
+Once the cluster has been provisioned, install the `ProvisioningRequest` definition.
+
+```
+kubectl kamaji kubeconfig get capi-quickstart-kubevirt > /tmp/capi-quickstart-kubevirt
+KUBECONFIG=/tmp/capi-quickstart-kubevirt kubectl apply -f https://raw.githubusercontent.com/kubernetes/autoscaler/refs/tags/cluster-autoscaler-1.34.1/cluster-autoscaler/apis/config/crd/autoscaling.x-k8s.io_provisioningrequests.yaml
+```
+
+Proceed with the installation of Cluster Autoscaler by enabling some additional parameters: YMMV.
+
+```yaml
+cloudProvider: clusterapi
+autoDiscovery:
+  namespace: default
+  labels:
+  - autoscaling.x-k8s.io: enabled
+
+clusterAPIKubeconfigSecret: capi-quickstart-kubeconfig
+clusterAPIMode: kubeconfig-incluster
+
+extraArgs:
+  enable-provisioning-requests: true
+  kube-api-content-type: "application/json"
+  cloud-config: /etc/kubernetes/management/kubeconfig
+
+extraVolumeSecrets:
+  # Mount the management kubeconfig to talk with the management cluster:
+  # the in-rest configuration doesn't work
+  management-kubeconfig:
+    name: management-kubeconfig
+    mountPath: /etc/kubernetes/management
+    items:
+    - key: kubeconfig
+      path: kubeconfig
+```
+
+The Cluster Autoscaler should be up and running, enabled to connect to the management and tenant cluster API Server:
+follow the [official example](https://github.com/kubernetes/autoscaler/blob/cluster-autoscaler-1.34.1/cluster-autoscaler/FAQ.md#example-usage) from the repository to assess the `ProvisioningRequest` feature.

docs/content/cluster-api/cluster-class.md

@@ -0,0 +1,642 @@
+# Cluster Class with Kamaji
+
+`ClusterClass` is a Cluster API feature that enables template-based cluster creation. When combined with Kamaji's hosted control plane architecture, `ClusterClass` provides a powerful pattern for standardizing Kubernetes cluster deployments across multiple infrastructure providers while maintaining consistent control plane configurations.
+
+!!! warning "Experimental Feature"
+    ClusterClass is still an experimental feature of Cluster API. As with any experimental features it should be used with caution. Read more about ClusterClass in the [Cluster API documentation](https://cluster-api.sigs.k8s.io/tasks/experimental-features/cluster-class/).
+
+## Understanding Cluster Class
+
+`ClusterClass` reduces configuration boilerplate by defining reusable cluster templates. Instead of creating individual resources for each cluster, you define a `ClusterClass` once and create multiple clusters from it with minimal configuration.
+
+With Kamaji, this pattern becomes even more powerful:
+- **Shared Control Plane Templates**: The same KamajiControlPlaneTemplate works across all infrastructure providers
+- **Infrastructure Flexibility**: Deploy worker nodes on vSphere, AWS, Azure, or any supported provider while maintaining consistent control planes
+- **Simplified Management**: Hosted control planes reduce the complexity of `ClusterClass` templates
+
+## Enabling Cluster Class
+
+To use `ClusterClass` with Kamaji, you need to enable the cluster topology feature gate before initializing the management cluster:
+
+```bash
+export CLUSTER_TOPOLOGY=true
+clusterctl init --control-plane kamaji --infrastructure vsphere
+```
+
+This will install:
+- Cluster API core components with `ClusterClass` support
+- Kamaji Control Plane Provider
+- Your chosen infrastructure provider (vSphere in this example)
+
+Verify the installation:
+
+```bash
+kubectl get deployments -A | grep -E "capi|kamaji"
+```
+
+## Template Architecture with Kamaji
+
+A `ClusterClass` with Kamaji consists of four main components:
+
+1. Control Plane Template (KamajiControlPlaneTemplate): Defines the hosted control plane configuration that remains consistent across infrastructure providers.
+
+2. Infrastructure Template (VSphereClusterTemplate): Provider-specific infrastructure configuration for the cluster.
+
+3. Bootstrap Template (KubeadmConfigTemplate): Node initialization configuration that works across providers.
+
+4. Machine Template (VSphereMachineTemplate): Provider-specific machine configuration for worker nodes.
+
+Here's how these components relate in a `ClusterClass`:
+
+```yaml
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: ClusterClass
+metadata:
+  name: kamaji-vsphere-class
+spec:
+  # Infrastructure provider template
+  infrastructure:
+    ref:
+      apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+      kind: VSphereClusterTemplate
+      name: vsphere-cluster-template
+  
+  # Kamaji control plane template - reusable across providers
+  controlPlane:
+    ref:
+      apiVersion: controlplane.cluster.x-k8s.io/v1alpha1
+      kind: KamajiControlPlaneTemplate
+      name: kamaji-control-plane-template
+    
+  # Worker configuration
+  workers:
+    machineDeployments:
+    - class: default-worker
+      template:
+        bootstrap:
+          ref:
+            apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
+            kind: KubeadmConfigTemplate
+            name: worker-bootstrap-template
+        infrastructure:
+          ref:
+            apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+            kind: VSphereMachineTemplate
+            name: vsphere-worker-template
+```
+
+The key advantage: the KamajiControlPlaneTemplate and KubeadmConfigTemplate can be shared across different infrastructure providers, while only the infrastructure-specific templates need to change.
+
+## Creating a Cluster Class
+
+Let's create a `ClusterClass` for vSphere with Kamaji. First, define the shared templates:
+
+### KamajiControlPlaneTemplate
+
+This template defines the hosted control plane configuration:
+
+```yaml
+apiVersion: controlplane.cluster.x-k8s.io/v1alpha1
+kind: KamajiControlPlaneTemplate
+metadata:
+  name: kamaji-controlplane
+  namespace: capi-templates-vsphere
+spec:
+  template:
+    spec:
+      dataStoreName: "default"  # Default datastore for etcd
+      
+      network:
+        serviceType: LoadBalancer
+        serviceAddress: ""
+        certSANs: []
+        
+      addons:
+        coreDNS: {}
+        kubeProxy: {}
+        konnectivity: {}
+
+      apiServer:
+        extraArgs: []
+        resources:
+          requests: {}
+      controllerManager:
+        extraArgs: []
+        resources:
+          requests: {}
+      scheduler:
+        extraArgs: []
+        resources:
+          requests: {}
+            
+      kubelet:
+        cgroupfs: systemd
+        preferredAddressTypes:
+        - InternalIP
+        
+      registry: "registry.k8s.io"
+```
+
+### KubeadmConfigTemplate
+
+This bootstrap template configures worker nodes:
+
+```yaml
+apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
+kind: KubeadmConfigTemplate
+metadata:
+  name: worker-bootstrap-template
+spec:
+  template:
+    spec:
+    
+      # Configuration for kubeadm join
+      joinConfiguration:
+        discovery: {}
+        nodeRegistration:
+          criSocket: /var/run/containerd/containerd.sock
+          imagePullPolicy: IfNotPresent
+          name: '{{ local_hostname }}'
+          kubeletExtraArgs:
+            cloud-provider: external
+            node-ip: "{{ ds.meta_data.local_ipv4 }}"       
+
+      # Commands to run before kubeadm join
+      preKubeadmCommands:
+      - hostnamectl set-hostname "{{ ds.meta_data.hostname }}"
+      - echo "127.0.0.1 {{ ds.meta_data.hostname }}" >> /etc/hosts
+      
+      # Commands to run after kubeadm join
+      postKubeadmCommands: []
+
+      # Users to create on worker nodes
+      users: []
+```
+
+### VSphereClusterTemplate
+
+Infrastructure-specific template for vSphere:
+
+```yaml
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: VSphereClusterTemplate
+metadata:
+  name: vsphere
+  namespace: capi-templates-vsphere
+spec:
+  template:
+    spec:
+      server: "vcenter.sample.com"  # vCenter server address
+      thumbprint: ""                # vCenter certificate thumbprint
+      
+      identityRef:
+        kind: VSphereClusterIdentity  
+        name: "vsphere-cluster-identity"
+      
+      failureDomainSelector: {} 
+      clusterModules: []
+```
+
+### VSphereMachineTemplate
+
+Machine template for vSphere workers:
+
+```yaml
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: VSphereMachineTemplate
+metadata:
+  name: vsphere-vm-base
+  namespace: capi-templates-vsphere
+spec:
+  template:
+    spec:
+      # Resources will be patched by ClusterClass based on variables
+      # numCPUs, memoryMiB, diskGiB are dynamically set
+      
+      # Infrastructure defaults - will be patched by ClusterClass
+      server: "vcenter.sample.com"
+      datacenter: "datacenter"
+      datastore: "datastore"
+      resourcePool: "Resources"
+      folder: "vm-folder"
+      template: "ubuntu-2404-kube-v1.32.0"
+      storagePolicyName: ""
+      thumbprint: ""
+      
+      # Network configuration (IPAM by default)
+      network:
+        devices:
+        - networkName: "k8s-network"
+          dhcp4: false
+          addressesFromPools:
+          - apiGroup: ipam.cluster.x-k8s.io
+            kind: InClusterIPPool
+            name: "{{ .builtin.cluster.name }}"  # Uses cluster name
+```
+
+### Variables and Patching in Cluster Class
+
+`ClusterClass` becomes powerful through its variable system and JSON patching capabilities. This allows the same templates to be customized for different use cases without duplicating YAML.
+
+#### Variable System
+
+Variables in `ClusterClass` define the parameters users can customize when creating clusters. Each variable has:
+
+- **Schema Definition**: OpenAPI v3 schema that validates input
+- **Required/Optional**: Whether the variable must be provided
+- **Default Values**: Fallback values when not specified
+- **Type Constraints**: Data types, ranges, and enum values
+
+Here's how variables work in practice:
+
+**Control Plane Variables:**
+```yaml
+variables:
+- name: kamajiControlPlane
+  required: true
+  schema:
+    openAPIV3Schema:
+      type: object
+      properties:
+        dataStoreName:
+          type: string
+          description: "Datastore name for etcd"
+          default: "default"
+        network:
+          type: object
+          properties:
+            serviceType:
+              type: string
+              enum: ["ClusterIP", "NodePort", "LoadBalancer"]
+              default: "LoadBalancer"
+            serviceAddress:
+              type: string
+              description: "Pre-assigned VIP address"
+```
+
+**Machine Resource Variables:**
+```yaml
+- name: machineSpecs
+  required: true
+  schema:
+    openAPIV3Schema:
+      type: object
+      properties:
+        numCPUs:
+          type: integer
+          minimum: 2
+          maximum: 64
+          default: 4
+        memoryMiB:
+          type: integer
+          minimum: 4096
+          maximum: 131072
+          default: 8192
+        diskGiB:
+          type: integer
+          minimum: 40
+          maximum: 2048
+          default: 100
+```
+
+#### JSON Patching System
+
+Patches apply variable values to the base templates at cluster creation time. This enables the same template to serve different configurations.
+
+**Control Plane Patching:**
+```yaml
+patches:
+- name: controlPlaneConfig
+  definitions:
+  - selector:
+      apiVersion: controlplane.cluster.x-k8s.io/v1alpha1
+      kind: KamajiControlPlaneTemplate
+      matchResources:
+        controlPlane: true
+    jsonPatches:
+    - op: replace
+      path: /spec/template/spec/dataStoreName
+      valueFrom:
+        variable: kamajiControlPlane.dataStoreName
+    - op: replace
+      path: /spec/template/spec/network/serviceType
+      valueFrom:
+        variable: kamajiControlPlane.network.serviceType
+```
+
+**Machine Resource Patching:**
+```yaml
+- name: machineResources
+  definitions:
+  - selector:
+      apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+      kind: VSphereMachineTemplate
+      matchResources:
+        machineDeploymentClass:
+          names: ["default-worker"]
+    jsonPatches:
+    - op: add  # Resources are not in base template
+      path: /spec/template/spec/numCPUs
+      valueFrom:
+        variable: machineSpecs.numCPUs
+    - op: add
+      path: /spec/template/spec/memoryMiB
+      valueFrom:
+        variable: machineSpecs.memoryMiB
+```
+
+#### Advanced Patching Patterns
+
+**Conditional Patching:**
+```yaml
+- name: optionalVIP
+  definitions:
+  - selector:
+      apiVersion: controlplane.cluster.x-k8s.io/v1alpha1
+      kind: KamajiControlPlaneTemplate
+    jsonPatches:
+    - op: replace
+      path: /spec/template/spec/network/serviceAddress
+      valueFrom:
+        variable: kamajiControlPlane.network.serviceAddress
+      # Only applies if serviceAddress is not empty
+      enabledIf: "{{ ne .kamajiControlPlane.network.serviceAddress \"\" }}"
+```
+
+**Infrastructure Patching:**
+```yaml
+- name: infrastructureConfig
+  definitions:
+  - selector:
+      apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+      kind: VSphereMachineTemplate
+    jsonPatches:
+    - op: replace
+      path: /spec/template/spec/datacenter
+      valueFrom:
+        variable: infrastructure.datacenter
+    - op: replace
+      path: /spec/template/spec/datastore
+      valueFrom:
+        variable: infrastructure.datastore
+    - op: replace
+      path: /spec/template/spec/template
+      valueFrom:
+        variable: infrastructure.vmTemplate
+```
+
+### Complete Cluster Class with Variables
+
+For a comprehensive example with all variables and patches configured, see the [vsphere-kamaji-clusterclass.yaml](https://raw.githubusercontent.com/clastix/cluster-api-control-plane-provider-kamaji/master/templates/vsphere/capi-kamaji-vsphere-class-template.yaml) template.
+
+## Creating a Cluster from Cluster Class
+
+With the `ClusterClass` defined, creating a cluster becomes remarkably simple:
+
+```yaml
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: Cluster
+metadata:
+  name: my-cluster
+  namespace: default
+spec:
+  # Network configuration defined at cluster level
+  clusterNetwork:
+    pods:
+      cidrBlocks: ["10.244.0.0/16"]
+    services:
+      cidrBlocks: ["10.96.0.0/12"]
+    serviceDomain: "cluster.local"
+  
+  topology:
+    class: vsphere-standard
+    classNamespace: capi-templates-vsphere
+    version: v1.32.0
+    
+    controlPlane:
+      replicas: 2
+    
+    workers:
+      machineDeployments:
+      - class: default-worker
+        name: worker-nodes
+        replicas: 3
+    
+    variables:
+    - name: kamajiControlPlane
+      value:
+        dataStoreName: "etcd"
+        network:
+          serviceType: "LoadBalancer"
+          serviceAddress: ""  # Auto-assigned if empty
+    
+    - name: machineSpecs
+      value:
+        numCPUs: 8
+        memoryMiB: 16384
+        diskGiB: 60
+    
+    - name: infrastructure
+      value:
+        vmTemplate: "ubuntu-2404-kube-v1.32.0"
+        datacenter: "K8s-TI-dtc"
+        datastore: "K8s-N01td-01"
+        resourcePool: "rp-kamaji-dev"
+        folder: "my-cluster-vms"
+    
+    - name: networking
+      value:
+        networkName: "VM-K8s-TI-cpmgmt"
+        nameservers: ["8.8.8.8", "1.1.1.1"]
+        dhcp4: false  # Using IPAM
+```
+
+Create the cluster:
+
+```bash
+kubectl apply -f my-cluster.yaml
+```
+
+Monitor cluster creation:
+
+```bash
+clusterctl describe cluster my-cluster
+kubectl get cluster,kamajicontrolplane,machinedeployment -n default
+```
+
+With this approach, the same `KamajiControlPlaneTemplate` and `KubeadmConfigTemplate` can be reused when creating `ClusterClasses` for AWS, Azure, or any other provider. Only the infrastructure-specific templates need to change.
+
+## Cross-Provider Template Reuse
+
+One of Kamaji's key advantages with `ClusterClass` is template modularity across providers. Here's how to leverage this:
+
+### Shared Templates Repository
+
+Create a namespace for shared templates:
+
+```bash
+kubectl create namespace cluster-templates
+```
+
+Deploy shared Kamaji and bootstrap templates once:
+
+```bash
+kubectl apply -n cluster-templates -f kamaji-controlplane-template.yaml
+kubectl apply -n cluster-templates -f kubeadm-config-template.yaml
+```
+
+### Provider-Specific Cluster Classes
+
+For each infrastructure provider, create a `ClusterClass` that references the shared templates:
+
+#### AWS Cluster Class
+
+```yaml
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: ClusterClass
+metadata:
+  name: kamaji-aws-class
+spec:
+  controlPlane:
+    ref:
+      apiVersion: controlplane.cluster.x-k8s.io/v1alpha1
+      kind: KamajiControlPlaneTemplate
+      name: kamaji-controlplane
+      namespace: cluster-templates  # Shared template
+  
+  infrastructure:
+    ref:
+      apiVersion: infrastructure.cluster.x-k8s.io/v1beta2
+      kind: AWSClusterTemplate
+      name: aws-cluster-template  # AWS-specific
+  
+  workers:
+    machineDeployments:
+    - class: default-worker
+      template:
+        bootstrap:
+          ref:
+            apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
+            kind: KubeadmConfigTemplate
+            name: kubeadm
+            namespace: cluster-templates  # Shared template
+        infrastructure:
+          ref:
+            apiVersion: infrastructure.cluster.x-k8s.io/v1beta2
+            kind: AWSMachineTemplate
+            name: aws-worker-template  # AWS-specific
+```
+
+### Azure Cluster Class
+
+```yaml
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: ClusterClass
+metadata:
+  name: kamaji-azure-class
+spec:
+  controlPlane:
+    ref:
+      apiVersion: controlplane.cluster.x-k8s.io/v1alpha1
+      kind: KamajiControlPlaneTemplate
+      name: kamaji-control-plane-template
+      namespace: cluster-templates  # Same shared template
+  
+  infrastructure:
+    ref:
+      apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+      kind: AzureClusterTemplate
+      name: azure-cluster-template  # Azure-specific
+  
+  workers:
+    machineDeployments:
+    - class: default-worker
+      template:
+        bootstrap:
+          ref:
+            apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
+            kind: KubeadmConfigTemplate
+            name: worker-bootstrap-template
+            namespace: cluster-templates  # Same shared template
+        infrastructure:
+          ref:
+            apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+            kind: AzureMachineTemplate
+            name: azure-worker-template  # Azure-specific
+```
+
+## Managing Cluster Class Lifecycle
+
+### Listing Available Cluster Classes
+
+```bash
+kubectl get clusterclasses -A
+```
+
+### Viewing Cluster Class Details
+
+```bash
+kubectl describe clusterclass vsphere-standard -n capi-templates-vsphere
+```
+
+### Updating a Cluster Class
+
+A `ClusterClass` update affects only new clusters. Existing clusters continue using their original configuration:
+
+```bash
+kubectl edit clusterclass vsphere-standard -n capi-templates-vsphere
+```
+
+### Deleting Clusters Created from Cluster Class
+
+Always delete clusters before removing the `ClusterClass`:
+
+```bash
+# Delete the cluster
+kubectl delete cluster my-cluster
+
+# Wait for cleanup
+kubectl wait --for=delete cluster/my-cluster --timeout=10m
+
+# Then safe to delete ClusterClass if no longer needed
+kubectl delete clusterclass vsphere-standard -n capi-templates-vsphere
+```
+
+## Template Versioning Strategies
+
+When managing `ClusterClasses` across environments, consider these versioning approaches:
+
+### Semantic Versioning in Names
+
+```yaml
+metadata:
+  name: vsphere-standard-v1-2-0
+  namespace: capi-templates-vsphere
+```
+
+### Using Labels for Version Tracking
+
+```yaml
+metadata:
+  name: vsphere-standard
+  namespace: capi-templates-vsphere
+  labels:
+    version: "1.2.0"
+    stability: "stable"
+    tier: "standard"
+```
+
+### Namespace Separation
+
+```bash
+kubectl create namespace clusterclass-v1
+kubectl create namespace clusterclass-v2
+```
+
+This enables gradual migration between `ClusterClass` versions while maintaining compatibility.
+
+## Further Reading
+
+  - [Cluster API ClusterClass Documentation](https://cluster-api.sigs.k8s.io/tasks/experimental-features/cluster-class/)
+  - [Kamaji Control Plane Provider Reference](https://doc.crds.dev/github.com/clastix/cluster-api-control-plane-provider-kamaji)
+  - [CAPI Provider Integration](https://github.com/clastix/cluster-api-control-plane-provider-kamaji)

docs/content/cluster-api/control-plane-provider.md

@@ -0,0 +1,98 @@
+# Kamaji Control Plane Provider
+
+Kamaji can act as a Cluster API Control Plane provider using the `KamajiControlPlane` custom resource, which defines the control plane of a Tenant Cluster.
+
+Here is an example of a `KamajiControlPlane`:
+
+```yaml
+kind: KamajiControlPlane
+apiVersion: controlplane.cluster.x-k8s.io/v1alpha1
+metadata:
+  name: '${CLUSTER_NAME}'
+  namespace: '${CLUSTER_NAMESPACE}'
+spec:
+  apiServer:
+    extraArgs:
+      - --cloud-provider=external
+  controllerManager:
+    extraArgs:
+      - --cloud-provider=external
+  dataStoreName: default
+  addons:
+    coreDNS: {}
+    kubeProxy: {}
+    konnectivity: {}
+  kubelet:
+    cgroupfs: systemd
+    preferredAddressTypes:
+      - InternalIP
+  network:
+    serviceType: LoadBalancer
+  version: ${KUBERNETES_VERSION}
+```
+
+You can use this as reference in a standard `Cluster` custom resource as controlplane provider:
+
+```yaml
+kind: Cluster
+apiVersion: cluster.x-k8s.io/v1beta1
+metadata:
+  labels:
+    cluster.x-k8s.io/cluster-name: '${CLUSTER_NAME}'
+  name: '${CLUSTER_NAME}'
+  namespace: '${CLUSTER_NAMESPACE}'
+spec:
+  controlPlaneRef:
+    apiVersion: controlplane.cluster.x-k8s.io/v1beta1
+    kind: KamajiControlPlane
+    name: '${CLUSTER_NAME}'
+  clusterNetwork:
+    pods:
+      cidrBlocks:
+        - '${PODS_CIDR}'
+    services:
+      cidrBlocks:
+        - '${SERVICES_CIDR}'
+  infrastructureRef:
+    apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+    kind: ... # your infrastructure kind may vary
+    name: '${CLUSTER_NAME}'
+```
+
+!!! info "Full Reference"
+    For a full reference of the `KamajiControlPlane` custom resource, please see the [Reference APIs](https://doc.crds.dev/github.com/clastix/cluster-api-control-plane-provider-kamaji/controlplane.cluster.x-k8s.io/KamajiControlPlane/v1alpha1).
+
+## Getting started with the Kamaji Control Plane Provider
+
+Cluster API Provider Kamaji is compliant with the `clusterctl` contract, which means you can use it with the `clusterctl` CLI to create and manage your Kamaji based clusters.
+
+!!! info "Options for install Cluster API"
+    There are two ways to getting started with Cluster API:
+
+    * using `clusterctl` to install the Cluster API components.
+    * using the Cluster API Operator. Please refer to the [Cluster API Operator](https://cluster-api-operator.sigs.k8s.io/) guide for this option.
+
+### Prerequisites
+
+* [`clusterctl`](https://cluster-api.sigs.k8s.io/user/quick-start#install-clusterctl) installed in your workstation to handle the lifecycle of your clusters.
+* [`kubectl`](https://kubernetes.io/docs/tasks/tools/) installed in your workstation to interact with your clusters.
+* [Kamaji](../getting-started/index.md) installed in your Management Cluster.
+
+### Initialize the Management Cluster
+
+Use `clusterctl` to initialize the Management Cluster. When executed for the first time, `clusterctl init` will fetch and install the Cluster API components in the Management Cluster
+
+```bash
+clusterctl init --control-plane kamaji
+```
+
+As result, the following Cluster API components will be installed:
+
+* Cluster API Provider in `capi-system` namespace
+* Bootstrap Provider in `capi-kubeadm-bootstrap-system` namespace
+* Kamaji Control Plane Provider in `kamaji-system` namespace
+
+In the next step, we will create a fully functional Kubernetes cluster using the Kamaji Control Plane Provider and the Infrastructure provider of choice.
+
+For a complete list of supported infrastructure providers, please refer to the [other providers](other-providers.md) page.
+

docs/content/cluster-api/external-cluster.md

@@ -0,0 +1,305 @@
+# Kamaji and `externalClusterReference` usage
+
+This document explains how to use **Kamaji's `externalClusterReference`** together with **Cluster API (CAPI)** to run Kubernetes control planes on an **external cluster**, while managing worker nodes from a management cluster.
+
+It assumes the use of the KubeVirt infrastructure provider for ease of deployment and local testing.
+
+---
+
+## High-level Architecture
+
+The following setup operates on **two Kubernetes clusters**:
+
+- **Management cluster** – runs Cluster API controllers and the Kamaji control-plane provider, and manages cluster lifecycle and topology.
+- **External cluster** - runs Kamaji, hosts the Kubernetes control plane components and receives control plane workloads via Kamaji
+
+---
+
+## Prerequisites
+
+- `docker`
+- `kind`
+- `kubectl`
+- `clusterctl`
+- `helm`
+
+---
+
+## Step 1: Create the KIND clusters
+
+Create the **management** cluster:
+
+```bash
+kind create cluster --name management
+```
+
+Create the **external** cluster that will host control planes:
+
+```bash
+kind create cluster --name external
+```
+
+Verify contexts:
+
+```bash
+kubectl config get-contexts
+```
+
+---
+
+## Step 2: Initialize Cluster API controllers
+
+Switch to the management cluster:
+
+```bash
+kubectl config use-context kind-management
+```
+
+Enable ClusterClass support and initialize Cluster API with Kamaji and KubeVirt:
+
+```bash
+export CLUSTER_TOPOLOGY=true
+clusterctl init \
+  --core cluster-api \
+  --bootstrap kubeadm \
+  --infrastructure kubevirt \
+  --control-plane kamaji
+```
+
+---
+
+## Step 3: Enable Kamaji external cluster feature gates
+
+Patch the Kamaji controller to enable `externalClusterReference`:
+
+```bash
+kubectl -n kamaji-system patch deployment capi-kamaji-controller-manager \
+  --type='json' \
+  -p='[
+    {
+      "op": "replace",
+      "path": "/spec/template/spec/containers/0/args/1",
+      "value": "--feature-gates=ExternalClusterReference=true,ExternalClusterReferenceCrossNamespace=true"
+    }
+  ]'
+```
+
+---
+
+## Step 4: Install KubeVirt
+
+Fetch the latest stable KubeVirt version and install:
+
+```bash
+export VERSION=$(curl -s "https://storage.googleapis.com/kubevirt-prow/release/kubevirt/kubevirt/stable.txt")
+kubectl apply -f "https://github.com/kubevirt/kubevirt/releases/download/${VERSION}/kubevirt-operator.yaml"
+kubectl apply -f "https://github.com/kubevirt/kubevirt/releases/download/${VERSION}/kubevirt-cr.yaml"
+```
+
+Enable emulation (optional, if virtualization is not supported):
+
+```bash
+kubectl -n kubevirt patch kubevirt kubevirt \
+  --type=merge \
+  --patch '{"spec":{"configuration":{"developerConfiguration":{"useEmulation":true}}}}'
+```
+
+---
+
+## Step 5: Prepare kubeconfig for the external cluster
+
+Retrieve the external cluster control-plane address:
+
+```bash
+EXT_CP_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "external-control-plane")
+```
+
+Export and rewrite the kubeconfig:
+
+```bash
+kubectl --context kind-external config view --raw --minify --flatten > kind-external.kubeconfig
+```
+
+Replace the API endpoint with the cluster IP, required for cross-cluster access from the management cluster:
+
+```bash
+bash -c "sed -i -E 's#https://[^:]+:[0-9]+#https://$EXT_CP_IP:6443#g' kind-external.kubeconfig"
+```
+
+Create the kubeconfig secret in the management cluster:
+
+```bash
+kubectl -n default create secret generic kind-external-kubeconfig \
+  --from-file=kubeconfig=kind-external.kubeconfig
+```
+
+---
+
+## Step 6: Install Kamaji and dependencies on the external cluster
+
+Switch context:
+
+```bash
+kubectl config use-context kind-external
+```
+
+Install cert-manager:
+
+```bash
+helm upgrade --install cert-manager jetstack/cert-manager \
+  --namespace cert-manager \
+  --create-namespace \
+  --set installCRDs=true
+```
+
+Install Kamaji:
+
+```bash
+helm upgrade --install kamaji clastix/kamaji \
+  --namespace kamaji-system \
+  --create-namespace \
+  --set 'resources=null' \
+  --version 0.0.0+latest
+```
+
+Install MetalLB:
+
+```bash
+kubectl apply -f https://raw.githubusercontent.com/metallb/metallb/v0.15.3/config/manifests/metallb-native.yaml
+```
+
+Configure MetalLB IP address pool:
+
+```bash
+SUBNET=$(docker network inspect kind | jq -r '.[0].IPAM.Config[] | select(.Subnet | test(":") | not) | .Subnet' | head -n1)
+NET_PREFIX=$(echo "$SUBNET" | cut -d/ -f1 | awk -F. '{print $1"."$2}')
+kubectl apply -f - <<EOF
+apiVersion: metallb.io/v1beta1
+kind: IPAddressPool
+metadata:
+  name: default
+  namespace: metallb-system
+spec:
+  addresses:
+  - ${NET_PREFIX}.255.200-${NET_PREFIX}.255.250
+---
+apiVersion: metallb.io/v1beta1
+kind: L2Advertisement
+metadata:
+  name: default
+  namespace: metallb-system
+EOF
+```
+
+Create tenant namespace:
+
+```bash
+kubectl create namespace kamaji-tenants
+```
+
+---
+
+## Step 7: Definition of KamajiControlPlaneTemplate
+
+The `KamajiControlPlaneTemplate` is defined in [the following manifest](https://raw.githubusercontent.com/clastix/kamaji/master/config/capi/clusterclass-kubevirt-kamaji-external.yaml) and can be applied directly.
+
+This template configures how Kamaji deploys and manages the tenant control plane on an external Kubernetes cluster using Cluster API.
+
+```bash
+apiVersion: controlplane.cluster.x-k8s.io/v1alpha1
+kind: KamajiControlPlaneTemplate
+metadata:
+  name: kamaji-controlplane-external
+  namespace: default
+spec:
+  template:
+    spec:
+      addons:
+        coreDNS: {}
+        konnectivity: {}
+        kubeProxy: {}
+      dataStoreName: "default" # reference to DataStore present on external cluster
+      deployment:
+        externalClusterReference:
+          deploymentNamespace: kamaji-tenants
+          kubeconfigSecretName: kind-external-kubeconfig
+          kubeconfigSecretKey: kubeconfig
+      network:
+        serviceType: LoadBalancer
+      kubelet:
+        cgroupfs: systemd
+        preferredAddressTypes:
+        - InternalIP
+      registry: "registry.k8s.io"
+```
+
+The `.spec.template.spec.deployment.externalClusterReference` section defines how Kamaji connects to and deploys control plane components into the external cluster:
+
+- `deploymentNamespace` - The namespace on the external cluster where `TenantControlPlane` resources and control plane components are created.
+- `kubeconfigSecretName` - The name of the Kubernetes Secret containing a kubeconfig that allows Kamaji to authenticate to the external cluster.
+- `kubeconfigSecretKey` - The key inside the secret that holds the kubeconfig data.
+
+The referenced secret must exist in the Kamaji management cluster and provide sufficient permissions to create and manage resources in the target external cluster.
+
+---
+
+## Step 8: Create the Cluster
+
+Switch context back to the management cluster:
+
+```bash
+kubectl config use-context kind-management
+```
+
+Apply the Cluster manifest:
+
+```bash
+kubectl apply -f "https://raw.githubusercontent.com/clastix/kamaji/master/config/capi/clusterclass-kubevirt-kamaji-external.yaml"
+```
+
+---
+
+## Validation
+
+Check tenant control plane pods running in the external cluster:
+
+```bash
+kubectl --context kind-external -n kamaji-tenants get pods
+```
+
+Check cluster status in the management cluster:
+
+```bash
+kubectl --context kind-management get clusters
+kubectl --context kind-management get kamajicontrolplanes
+```
+
+Get cluster kubeconfig and confirm it is working:
+
+```bash
+kubectl config use-context kind-management
+clusterctl get kubeconfig demo-external > demo-external.kubeconfig
+KUBECONFIG=./demo-external.kubeconfig kubectl get nodes
+```
+
+---
+
+## Clean up
+
+Delete Kind clusters:
+
+```bash
+kind delete cluster --name management
+kind delete cluster --name external
+```
+
+---
+
+## Summary
+
+Using `externalClusterReference` with Kamaji and Cluster API enables:
+
+- Hosted Kubernetes control planes on remote clusters
+- Strong separation of concerns
+- Multi-cluster management patterns
+- Clean integration with ClusterClass

docs/content/cluster-api/index.md

@@ -0,0 +1,14 @@
+# Cluster APIs Support
+
+The [Cluster API](https://github.com/kubernetes-sigs/cluster-api) brings declarative, Kubernetes-style APIs to the creation, configuration, and management of Kubernetes clusters. If you're not familiar with the Cluster API project, you can learn more from the [official documentation](https://cluster-api.sigs.k8s.io/).
+
+Users can utilize Kamaji in two distinct ways:
+
+* **Standalone:** Kamaji can be used as a standalone Kubernetes Operator installed in the Management Cluster to manage multiple Tenant Control Planes. Worker nodes of Tenant Clusters can join any infrastructure, whether it be cloud, data-center, or edge, using various automation tools such as _Ansible_, _Terraform_, or even manually with any script calling `kubeadm`. See [yaki](https://goyaki.clastix.io/) as an example.
+
+* **Cluster API Provider:** Kamaji can be used as a [Cluster API Control Plane Provider](https://cluster-api.sigs.k8s.io/reference/providers#control-plane) to manage multiple Tenant Control Planes across various infrastructures. Kamaji offers seamless integration with the most popular [Cluster API Infrastructure Providers](https://cluster-api.sigs.k8s.io/reference/providers#infrastructure).
+
+!!! tip "Control Plane and Infrastructure Decoupling"
+    Kamaji decouples the Control Plane from the infrastructure, allowing the Kamaji Management Cluster to reside on a different infrastructure or cloud provider than the Tenant worker machines, as long as network reachability is ensured. This flexibility enables mixing and matching infrastructure providers, such as hosting the Management Cluster on a public cloud while deploying Tenant worker machines on private data centers, edge environments, or other clouds.
+
+Check the currently supported infrastructure providers and the roadmap on the related [repository](https://github.com/clastix/cluster-api-control-plane-provider-kamaji).

docs/content/cluster-api/other-providers.md

@@ -0,0 +1,21 @@
+# Other Infra Providers
+
+Kamaji offers seamless integration with the most popular [Cluster API Infrastructure Providers](https://cluster-api.sigs.k8s.io/reference/providers#infrastructure):
+
+- AWS
+- Azure
+- Google Cloud
+- Equinix/Packet
+- Hetzner
+- KubeVirt
+- Metal³
+- Nutanix
+- OpenStack
+- Tinkerbell
+- vSphere
+- IONOS Cloud
+- Proxmox by IONOS Cloud
+
+For the most up-to-date information and technical considerations, please always check the related [repository](https://github.com/clastix/cluster-api-control-plane-provider-kamaji).
+
+