fix(docs): missing export env vars in capi proxmox sample (#971)

* fix(docs): missing export env vars in capi proxmox sample

* fix(docs): escape token in proxmox capi

---------

Co-authored-by: bsctl <bsctl@clastix.io>

.dockerignore

@@ -1,4 +0,0 @@
-# More info: https://docs.docker.com/engine/reference/builder/#dockerignore-file
-# Ignore build and test binaries.
-bin/
-testbin/

.github/dependabot.yml

@@ -8,9 +8,12 @@ updates:
     commit-message:
       prefix: "feat(deps)"
     groups:
-      arrow:
+      k8s:
         patterns:
           - "k8s.io*"
+      etcd:
+        patterns:
+          - "go.etcd.io/etcd/*"
   - package-ecosystem: github-actions
     directory: /
     schedule:

.github/release-template.md

@@ -0,0 +1,10 @@
+This edge release can be pulled from Docker Hub as follows:
+
+```
+docker pull clastix/kamaji:$TAG
+```
+
+> As from the v1.0.0 release, CLASTIX no longer provides stable release artefacts.
+>
+> Stable release artefacts are offered on a subscription basis by CLASTIX, the main Kamaji project contributor.
+> Learn more from CLASTIX's [Support](https://clastix.io/support/) section.

.github/workflows/ci.yaml

@@ -7,36 +7,45 @@ on:
     branches: [ "*" ]
 
 jobs:
+  test:
+    name: integration
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v5
+      - uses: actions/setup-go@v6
+        with:
+          go-version-file: go.mod
+      - run: make test
   golangci:
     name: lint
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-go@v5
+      - uses: actions/checkout@v5
+      - uses: actions/setup-go@v6
         with:
-          go-version: '1.22'
-          check-latest: true
+          go-version-file: go.mod
       - name: Run golangci-lint
-        uses: golangci/golangci-lint-action@v6.1.0
-        with:
-          version: v1.54.2
-          only-new-issues: false
-          args: --timeout 5m --config .golangci.yml
+        run: make golint
+# TODO(prometherion): enable back once golangci-lint is built from v1.24 rather than v1.23
+#        uses: golangci/golangci-lint-action@v6.5.2
+#        with:
+#          version: v1.62.2
+#          only-new-issues: false
+#          args: --config .golangci.yml
   diff:
     name: diff
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v5
         with:
           fetch-depth: 0
-      - uses: actions/setup-go@v5
+      - uses: actions/setup-go@v6
         with:
-          go-version: '1.22'
-          check-latest: true
-      - run: make yaml-installation-file
-      - name: Checking if YAML installer file is not aligned
+          go-version-file: go.mod
+      - run: make manifests
+      - name: Checking if generated manifests are not aligned
         run: if [[ $(git diff | wc -l) -gt 0 ]]; then echo ">>> Untracked generated files have not been committed" && git --no-pager diff && exit 1; fi
-      - name: Checking if YAML installer generated untracked files
+      - name: Checking if missing untracked files for generated manifests
         run: test -z "$(git ls-files --others --exclude-standard 2> /dev/null)"
       - name: Checking if source code is not formatted
         run: test -z "$(git diff 2> /dev/null)"

.github/workflows/docker-ci.yml

@@ -1,100 +0,0 @@
-name: docker-ci
-
-on:
-  push:
-    tags:        
-      - "v*"
-      - "edge-*"
-
-jobs:
-  docker-ci:
-    runs-on: ubuntu-22.04
-    steps:
-
-      - name: Checkout
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-
-      - name: Generate build-args
-        id: build-args
-        run: |
-          # Declare vars for internal use
-          VERSION=$(make get_version)
-          GIT_HEAD_COMMIT=$(git rev-parse --short HEAD)
-          GIT_TAG_COMMIT=$(git rev-parse --short $VERSION)
-          GIT_MODIFIED_1=$(git diff $GIT_HEAD_COMMIT $GIT_TAG_COMMIT --quiet && echo "" || echo ".dev")
-          GIT_MODIFIED_2=$(git diff --quiet && echo "" || echo ".dirty")
-          # Export to GH_ENV
-          echo "GIT_LAST_TAG=$VERSION" >> $GITHUB_ENV
-          echo "GIT_HEAD_COMMIT=$GIT_HEAD_COMMIT" >> $GITHUB_ENV
-          echo "GIT_TAG_COMMIT=$GIT_TAG_COMMIT" >> $GITHUB_ENV
-          echo "GIT_MODIFIED=$(echo "$GIT_MODIFIED_1""$GIT_MODIFIED_2")" >> $GITHUB_ENV
-          echo "GIT_REPO=$(git config --get remote.origin.url)" >> $GITHUB_ENV
-          echo "BUILD_DATE=$(git log -1 --format="%at" | xargs -I{} date -d @{} +%Y-%m-%dT%H:%M:%S)" >> $GITHUB_ENV
-
-      - name: Docker meta
-        id: meta
-        uses: docker/metadata-action@v3
-        with:
-          images: |
-             quay.io/${{ github.repository }}
-             docker.io/${{ github.repository }}
-          tags: |
-            type=semver,pattern={{raw}}
-          flavor: |
-            latest=false
-
-      - name: Set up QEMU
-        id: qemu
-        uses: docker/setup-qemu-action@v1
-        with:
-          platforms: arm64,arm
-
-      - name: Set up Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v1
-        with:
-          install: true
-
-      - name: Inspect builder
-        run: |
-          echo "Name:      ${{ steps.buildx.outputs.name }}"
-          echo "Endpoint:  ${{ steps.buildx.outputs.endpoint }}"
-          echo "Status:    ${{ steps.buildx.outputs.status }}"
-          echo "Flags:     ${{ steps.buildx.outputs.flags }}"
-          echo "Platforms: ${{ steps.buildx.outputs.platforms }}"
-
-      - name: Login to quay.io Container Registry
-        uses: docker/login-action@v1 
-        with:
-          registry: quay.io
-          username: ${{ secrets.QUAY_IO_USERNAME }}
-          password: ${{ secrets.QUAY_IO_TOKEN }}
-
-      - name: Login to docker.io Container Registry
-        uses: docker/login-action@v1 
-        with:
-          registry: docker.io
-          username: ${{ secrets.DOCKER_IO_USERNAME }}
-          password: ${{ secrets.DOCKER_IO_TOKEN }}
-
-      - name: Build and push
-        id: build-release
-        uses: docker/build-push-action@v2
-        with:
-          file: Dockerfile
-          context: .
-          platforms: linux/amd64,linux/arm64,linux/arm
-          push: true
-          tags: ${{ steps.meta.outputs.tags }}
-          build-args: |
-            GIT_LAST_TAG=${{ env.GIT_LAST_TAG }}
-            GIT_HEAD_COMMIT=${{ env.GIT_HEAD_COMMIT }}
-            GIT_TAG_COMMIT=${{ env.GIT_TAG_COMMIT }}
-            GIT_MODIFIED=${{ env.GIT_MODIFIED }}
-            GIT_REPO=${{ env.GIT_REPO }}
-            BUILD_DATE=${{ env.BUILD_DATE }}
-
-      - name: Image digest
-        run: echo ${{ steps.build-release.outputs.digest }}

.github/workflows/e2e.yaml

@@ -4,12 +4,12 @@ on:
   push:
     branches: [ "*" ]
     paths:
-      - '.github/workflows/e2e.yml'
+      - '.github/workflows/e2e.yaml'
       - 'api/**'
       - 'charts/kamaji/**'
       - 'controllers/**'
       - 'e2e/*'
-      - 'Dockerfile'
+      - '.ko.yaml'
       - 'go.*'
       - 'main.go'
       - 'Makefile'
@@ -18,12 +18,12 @@ on:
   pull_request:
     branches: [ "*" ]
     paths:
-      - '.github/workflows/e2e.yml'
+      - '.github/workflows/e2e.yaml'
       - 'api/**'
       - 'charts/kamaji/**'
       - 'controllers/**'
       - 'e2e/*'
-      - 'Dockerfile'
+      - '.ko.yaml'
       - 'go.*'
       - 'main.go'
       - 'Makefile'
@@ -33,17 +33,18 @@ on:
 jobs:
   kind:
     name: Kubernetes
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v5
         with:
           fetch-depth: 0
-      - uses: actions/setup-go@v5
+      - uses: actions/setup-go@v6
         with:
-          go-version: '1.22'
-          check-latest: true
+          go-version-file: go.mod
       - run: |
           sudo apt-get update
           sudo apt-get install -y golang-cfssl
+          sudo swapoff -a
+          sudo modprobe br_netfilter
       - name: e2e testing
         run: make e2e

.github/workflows/helm.yaml

@@ -2,8 +2,7 @@ name: Helm Chart
 
 on:
   push:
-    branches: [ "*" ]
-    tags: [ "helm-v*" ]
+    branches: [ "master" ]
   pull_request:
     branches: [ "*" ]
 
@@ -12,30 +11,36 @@ jobs:
     name: diff
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v5
         with:
           fetch-depth: 0
       - run: make -C charts/kamaji docs
-      - name: Checking if Helm docs is not aligned
+      - name: Checking if Kamaji Helm Chart docs is not aligned
+        run: if [[ $(git diff | wc -l) -gt 0 ]]; then echo ">>> Untracked changes have not been committed" && git --no-pager diff && exit 1; fi
+      - run: make -C charts/kamaji-crds docs
+      - name: Checking if Kamaji CRDs Helm Chart docs is not aligned
         run: if [[ $(git diff | wc -l) -gt 0 ]]; then echo ">>> Untracked changes have not been committed" && git --no-pager diff && exit 1; fi
   lint:
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v2
-      - uses: azure/setup-helm@v1
+      - uses: actions/checkout@v5
+      - uses: azure/setup-helm@v4
         with:
           version: 3.3.4
       - name: Building dependencies
         run: |-
           helm repo add clastix https://clastix.github.io/charts 
           helm dependency build ./charts/kamaji
-      - name: Linting Chart
+      - name: Linting Kamaji Helm Chart
         run: helm lint ./charts/kamaji
+      - name: Linting Kamaji CRDS Helm Chart
+        run: helm lint ./charts/kamaji-crds
   release:
-    if: startsWith(github.ref, 'refs/tags/helm-v')
+    if: github.event_name == 'push' && github.ref == 'refs/heads/master'
+    needs: [ "lint", "diff" ]
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v5
       - name: Publish Helm chart
         uses: stefanprodan/helm-gh-pages@master
         with:

.github/workflows/ko-build.yml

@@ -0,0 +1,37 @@
+name: Container image build
+
+on:
+  push:
+    tags:
+      - edge-*
+      - v*
+    branches:
+      - master
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: "Tag to build"
+        required: true
+        type: string
+
+jobs:
+  ko:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v5
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-go@v6
+        with:
+          go-version-file: go.mod
+      - name: "ko: install"
+        run: make ko
+      - name: "ko: login to quay.io container registry"
+        run: ./bin/ko login quay.io -u ${{ secrets.QUAY_IO_USERNAME }} -p ${{ secrets.QUAY_IO_TOKEN }}
+      - name: "ko: login to docker.io container registry"
+        run: ./bin/ko login docker.io -u ${{ secrets.DOCKER_IO_USERNAME }} -p ${{ secrets.DOCKER_IO_TOKEN }}
+      - name: "ko: build and push tag"
+        run: make VERSION=${{ github.event.inputs.tag }} KO_LOCAL=false KO_PUSH=true build
+        if: github.event_name == 'workflow_dispatch'
+      - name: "ko: build and push latest"
+        run: make VERSION=latest KO_LOCAL=false KO_PUSH=true build

.github/workflows/pr.yaml

@@ -0,0 +1,23 @@
+name: Check PR Title
+
+on:
+  pull_request:
+    types: [opened, edited, reopened, synchronize]
+
+jobs:
+  semantic-pr-title:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: amannn/action-semantic-pull-request@v6
+        with:
+          types: |
+            feat
+            fix
+            chore
+            docs
+            style
+            refactor
+            perf
+            test
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/release.yml

@@ -0,0 +1,75 @@
+name: Weekly Edge Release
+
+on:
+  schedule:
+    - cron: '0 7 * * 1'  # Every Monday at 9 AM CET
+  workflow_dispatch:
+
+permissions:
+  contents: write
+
+jobs:
+  release:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v5
+        with:
+          fetch-depth: 0
+      - name: generating date metadata
+        id: date
+        run: |
+          CURRENT_DATE=$(date -u +'%Y-%m-%d')
+          YY=$(date -u +'%y')
+          M=$(date -u +'%_m' | sed 's/ //g')
+          FIRST_OF_MONTH=$(date -u -d "$CURRENT_DATE" +%Y-%m-01)
+          WEEK_NUM=$(( (($(date -u +%s) - $(date -u -d "$FIRST_OF_MONTH" +%s)) / 86400 + $(date -u -d "$FIRST_OF_MONTH" +%u) - 1) / 7 + 1 ))
+
+          echo "yy=$YY" >> $GITHUB_OUTPUT
+          echo "month=$M" >> $GITHUB_OUTPUT
+          echo "week=$WEEK_NUM" >> $GITHUB_OUTPUT
+          echo "date=$CURRENT_DATE" >> $GITHUB_OUTPUT
+      - name: generating tag metadata
+        id: tag
+        run: |
+          TAG="edge-${{ steps.date.outputs.yy }}.${{ steps.date.outputs.month }}.${{ steps.date.outputs.week }}"
+          echo "tag=$TAG" >> $GITHUB_OUTPUT
+      - name: generate release notes from template
+        run: |
+          export TAG="${{ steps.tag.outputs.tag }}"
+          envsubst < .github/release-template.md > release-notes.md
+      - name: generate release notes from template
+        run: |
+          export TAG="${{ steps.tag.outputs.tag }}"
+          envsubst < .github/release-template.md > release-notes-header.md
+      - name: generate GitHub release notes
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          gh release --repo "$GITHUB_REPOSITORY" \
+            create "${{ steps.tag.outputs.tag }}" \
+            --generate-notes \
+            --draft \
+            --title "temp" \
+            --notes "temp" > /dev/null || true
+          
+          gh release view "${{ steps.tag.outputs.tag }}" \
+            --json body --jq .body > auto-notes.md
+          
+          gh release delete "${{ steps.tag.outputs.tag }}" --yes || true
+      - name: combine notes
+        run: |
+          cat release-notes-header.md auto-notes.md > release-notes.md
+      - name: create GitHub release
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          gh release create "${{ steps.tag.outputs.tag }}" \
+            --title "${{ steps.tag.outputs.tag }}" \
+            --notes-file release-notes.md
+      - name: trigger container build workflow
+        env:
+          GH_TOKEN: ${{ secrets.WORKFLOW_TOKEN }}
+        run: |
+          gh workflow run "Container image build" \
+            --ref master \
+            -f tag="${{ steps.tag.outputs.tag }}"

.golangci.yml

@@ -1,51 +1,76 @@
-linters-settings:
-  gci:
-    sections:
-      - standard
-      - default
-      - prefix(github.com/clastix/kamaji/)
-  goheader:
-    template: |-
-      Copyright 2022 Clastix Labs
-      SPDX-License-Identifier: Apache-2.0
-
+version: "2"
 linters:
+  default: all
   disable:
-    - depguard
-    - wrapcheck
-    - gomnd
-    - scopelint
-    - varnamelen
-    - testpackage
-    - tagliatelle
-    - paralleltest
-    - ireturn
-    - goerr113
-    - gochecknoglobals
-    - exhaustivestruct
-    - wsl
-    - exhaustive
-    - nosprintfhostport
-    - nonamedreturns
-    - interfacebloat
-    - exhaustruct
-    - lll
-    - gosec
-    - gomoddirectives
-    - godox
-    - gochecknoinits
-    - funlen
-    - dupl
     - cyclop
+    - depguard
+    - dupl
+    - err113
+    - exhaustive
+    - exhaustruct
+    - funlen
+    - gochecknoglobals
+    - gochecknoinits
     - gocognit
+    - godox
+    - gomoddirectives
+    - gosec
+    - interfacebloat
+    - ireturn
+    - lll
+    - mnd
     - nestif
-    # deprecated linters
-    - deadcode
-    - golint
-    - interfacer
-    - structcheck
-    - varcheck
-    - nosnakecase
-    - ifshort
-    - maligned
-  enable-all: true
+    - nonamedreturns
+    - nosprintfhostport
+    - paralleltest
+    - perfsprint
+    - tagliatelle
+    - testpackage
+    - varnamelen
+    - wrapcheck
+    - wsl
+  settings:
+    staticcheck:
+      checks:
+        - all
+        - -QF1008
+    goheader:
+      template: |-
+        Copyright 2022 Clastix Labs
+        SPDX-License-Identifier: Apache-2.0
+    revive:
+      rules:
+        - name: dot-imports
+          arguments:
+            - allowedPackages:
+                - github.com/onsi/ginkgo/v2
+                - github.com/onsi/gomega
+  exclusions:
+    generated: lax
+    presets:
+      - comments
+      - common-false-positives
+      - legacy
+      - std-error-handling
+    paths:
+      - third_party$
+      - builtin$
+      - examples$
+formatters:
+  enable:
+    - gci
+    - gofmt
+    - gofumpt
+    - goimports
+  settings:
+    gci:
+      sections:
+        - standard
+        - default
+        - prefix(github.com/clastix/kamaji/)
+  exclusions:
+    generated: lax
+    paths:
+      - third_party$
+      - builtin$
+      - examples$

.ko.yaml

@@ -0,0 +1,9 @@
+defaultPlatforms:
+  - linux/arm64
+  - linux/amd64
+  - linux/arm
+builds:
+  - id: kamaji
+    main: .
+    ldflags:
+      - '{{ if index .Env "LD_FLAGS" }}{{ .Env.LD_FLAGS }}{{ end }}'

ADOPTERS.md

@@ -7,15 +7,28 @@ Feel free to open a Pull-Request to get yours listed.
 
 | Type | Name | Since | Website | Use-Case |
 |:-|:-|:-|:-|:-|
-| Vendor | DCloud | 2024 | [link](https://dcloud.co.id) | DCloud is an Indonesian Cloud Provider using Kamaji to build and offer [Managed Kubernetes Service](https://dcloud.co.id/dkubes.html). |
-| End-user | Sicuro Tech Lab | 2024 | [link](https://sicurotechlab.it/) | Sicuro Tech Lab offers cloud infrastructure for Web Agencies and uses kamaji to provide managed k8s services. |
-| R&D | TIM | 2024 | [link](https://www.gruppotim.it) | TIM is an Italian telecommunications company using Kamaji for experimental research and development purposes. |
-| End-user | KINX | 2024 | [link](https://kinx.net/?lang=en) | KINX is an Internet infrastructure service provider and will use kamaji for its new [Managed Kubernetes Service](https://kinx.net/service/cloud/kubernetes/intro/?lang=en). |
-| End-user | sevensphere | 2023 | [link](https://www.sevensphere.io) | Sevensphere provides consulting services for end-user companies / cloud providers and uses Kamaji for designing cloud/on-premises Kubernetes-as-a-Service platform. |
-| Vendor | Ænix | 2023 | [link](https://aenix.io/) | Ænix provides consulting services for cloud providers and uses Kamaji for running Kubernetes-as-a-Service in free PaaS platform [Cozystack](https://cozystack.io). |
-| Vendor | Netsons | 2023 | [link](https://www.netsons.com) | Netsons is an Italian hosting and cloud provider and uses Kamaji in its [Managed Kubernetes](https://www.netsons.com/kubernetes) offering. |
 | Vendor | Aknostic | 2023 | [link](https://aknostic.com) | Aknostic is a cloud-native consultancy company using Kamaji to build a Kubernetes based PaaS. |
-
+| Vendor | Aruba | 2025 | [link](https://www.arubacloud.com/) | Aruba Cloud is an Italian Cloud Service Provider using Kamaji to build and offer [Managed Kubernetes Service](https://my.arubacloud.com). |
+| Vendor | CBWS | 2025 | [link](https://cbws.nl) | CBWS is an European Cloud Provider using Kamaji to build and offer their [Managed Kubernetes Service](https://cbws.nl/cloud/kubernetes/). |
+| Vendor | Coredge | 2025 | [link](https://coredge.io/) | Coredge uses Kamaji in its K8saaS offering to save infrastructure costs in its Sovereign Cloud & AI Infrastructure Platform for end-user organisations. |
+| Vendor | DCloud | 2024 | [link](https://dcloud.co.id) | DCloud is an Indonesian Cloud Provider using Kamaji to build and offer [Managed Kubernetes Service](https://dcloud.co.id/dkubes.html). |
+| Vendor | Dinova | 2025 | [link](https://dinova.one/) | Dinova is an Italian cloud services provider that integrates Kamaji in its datacenters to offer fully managed Kubernetes clusters. |
+| End-user | KINX | 2024 | [link](https://kinx.net/?lang=en) | KINX is an Internet infrastructure service provider and will use kamaji for its new [Managed Kubernetes Service](https://kinx.net/service/cloud/kubernetes/intro/?lang=en). |
+| Vendor | Netsons | 2023 | [link](https://www.netsons.com) | Netsons is an Italian hosting and cloud provider and uses Kamaji in its [Managed Kubernetes](https://www.netsons.com/kubernetes) offering. |
+| Vendor | NVIDIA | 2024 | [link](https://github.com/NVIDIA/doca-platform) | DOCA Platform Framework manages provisioning and service orchestration for NVIDIA Bluefield DPUs. |
+| R&D | Orange | 2024 | [link](https://gitlab.com/Orange-OpenSource/kanod) | Orange is a French telecommunications company using Kamaji for experimental research purpose, with Kanod research solution. |
+| Vendor | Platform9 | 2024 | [link](https://elasticmachinepool.com) | Platform9 uses Kamaji in its offering - Elastic Machine Pool, which is a tool for optimizing the cost of running kubernetes clusters in EKS. |
+| Vendor | Qumulus | 2024 | [link](https://www.qumulus.io) | Qumulus is a cloud provider and plans to use Kamaji for it's hosted Kubernetes service |
+| End-user | sevensphere | 2023 | [link](https://www.sevensphere.io) | Sevensphere provides consulting services for end-user companies / cloud providers and uses Kamaji for designing cloud/on-premises Kubernetes-as-a-Service platform. |
+| End-user | Sicuro Tech Lab | 2024 | [link](https://sicurotechlab.it/) | Sicuro Tech Lab offers cloud infrastructure for Web Agencies and uses kamaji to provide managed k8s services. |
+| Vendor | Sovereign Cloud Stack | 2024 | [link](https://sovereigncloudstack.org) | Sovereign Cloud Stack develops a standardized cloud platform and uses Kamaji in there Kubernetes-as-a-Service reference implementation |
+| R&D | TIM | 2024 | [link](https://www.gruppotim.it) | TIM is an Italian telecommunications company using Kamaji for experimental research and development purposes. |
+| End-user | Tinext Cloud | 2025 | [link](https://cloud.tinext.com) | Tinex Cloud is a Swiss cloud service provider using Kamaji to build their Managed Kubernetes Services. |
+| Vendor | Ænix | 2023 | [link](https://aenix.io/) | Ænix provides consulting services for cloud providers and uses Kamaji for running Kubernetes-as-a-Service in free PaaS platform [Cozystack](https://cozystack.io). |
+| End-user | Rackspace | 2024 | [link](https://spot.rackspace.com/) | Rackspace Spot uses Kamaji to manage our instances, offering fully-managed kubernetes infrastructure, auctioned in an open market. |
+| R&D | IONOS Cloud | 2024 | [link](https://cloud.ionos.com/) | IONOS Cloud is a German Cloud Provider evaluating Kamaji for its [Managed Kubernetes platform](https://cloud.ionos.com/managed/kubernetes). |
+| Vendor | OVHCloud | 2025 | [link](https://www.ovhcloud.com/) | OVHCloud is a European Cloud Provider that will use Kamaji for its Managed Kubernetes Service offer. |
+| Vendor | WOBCOM GmbH | 2024 | [link](https://www.wobcom.de/) | WOBCOM provides an [**Open Digital Platform**](https://www.wobcom.de/geschaeftskunden/odp/) solution for Smart Cities, which is provided for customers in a Managed Kubernetes provided by Kamaji. |
 
 ### Adopter Types
 

Dockerfile

@@ -1,40 +0,0 @@
-# Build the manager binary
-FROM golang:1.22 as builder
-
-WORKDIR /workspace
-# Copy the Go Modules manifests
-COPY go.mod go.mod
-COPY go.sum go.sum
-# cache deps before building and copying source so that we don't need to re-download as much
-# and so that source changes don't invalidate our downloaded layer
-RUN go mod download
-
-# Copy the go source
-COPY main.go main.go
-COPY cmd/ cmd/
-COPY api/ api/
-COPY controllers/ controllers/
-COPY internal/ internal/
-COPY indexers/ indexers/
-
-# Build
-ARG TARGETARCH
-ARG GIT_HEAD_COMMIT
-ARG GIT_TAG_COMMIT
-ARG GIT_LAST_TAG
-ARG GIT_MODIFIED
-ARG GIT_REPO
-ARG BUILD_DATE
-
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=$TARGETARCH go build \
-    -ldflags "-X github.com/clastix/kamaji/internal.GitRepo=$GIT_REPO -X github.com/clastix/kamaji/internal.GitTag=$GIT_LAST_TAG -X github.com/clastix/kamaji/internal.GitCommit=$GIT_HEAD_COMMIT -X github.com/clastix/kamaji/internal.GitDirty=$GIT_MODIFIED -X github.com/clastix/kamaji/internal.BuildTime=$BUILD_DATE" \
-    -a -o kamaji main.go
-
-# Use distroless as minimal base image to package the manager binary
-# Refer to https://github.com/GoogleContainerTools/distroless for more details
-FROM gcr.io/distroless/static:nonroot
-WORKDIR /
-COPY --from=builder /workspace/kamaji .
-USER 65532:65532
-
-ENTRYPOINT ["/kamaji"]

Makefile

@@ -3,40 +3,24 @@
 # To re-generate a bundle for another specific version without changing the standard setup, you can:
 # - use the VERSION as arg of the bundle target (e.g make bundle VERSION=0.0.2)
 # - use environment variables to overwrite this value (e.g export VERSION=0.0.2)
-VERSION ?= 1.0.0
+VERSION ?= $(or $(shell git describe --abbrev=0 --tags 2>/dev/null),$(GIT_HEAD_COMMIT))
 
-# CHANNELS define the bundle channels used in the bundle.
-# Add a new line here if you would like to change its default config. (E.g CHANNELS = "candidate,fast,stable")
-# To re-generate a bundle for other specific channels without changing the standard setup, you can:
-# - use the CHANNELS as arg of the bundle target (e.g make bundle CHANNELS=candidate,fast,stable)
-# - use environment variables to overwrite this value (e.g export CHANNELS="candidate,fast,stable")
-ifneq ($(origin CHANNELS), undefined)
-BUNDLE_CHANNELS := --channels=$(CHANNELS)
-endif
+# ENVTEST_K8S_VERSION specifies the Kubernetes version to be used 
+# during testing with the envtest environment. This ensures that 
+# the tests run against the correct API and behavior for the 
+# specific Kubernetes release being targeted (v1.31.0 in this case).
+ENVTEST_K8S_VERSION = 1.31.0
 
-# DEFAULT_CHANNEL defines the default channel used in the bundle.
-# Add a new line here if you would like to change its default config. (E.g DEFAULT_CHANNEL = "stable")
-# To re-generate a bundle for any other default channel without changing the default setup, you can:
-# - use the DEFAULT_CHANNEL as arg of the bundle target (e.g make bundle DEFAULT_CHANNEL=stable)
-# - use environment variables to overwrite this value (e.g export DEFAULT_CHANNEL="stable")
-ifneq ($(origin DEFAULT_CHANNEL), undefined)
-BUNDLE_DEFAULT_CHANNEL := --default-channel=$(DEFAULT_CHANNEL)
-endif
-BUNDLE_METADATA_OPTS ?= $(BUNDLE_CHANNELS) $(BUNDLE_DEFAULT_CHANNEL)
-
-# IMAGE_TAG_BASE defines the docker.io namespace and part of the image name for remote images.
-# This variable is used to construct full image tags for bundle and catalog images.
-#
-# For example, running 'make bundle-build bundle-push catalog-build catalog-push' will build and push both
-# clastix.io/operator-bundle:$VERSION and clastix.io/operator-catalog:$VERSION.
-IMAGE_TAG_BASE ?= clastix.io/operator
-
-# BUNDLE_IMG defines the image:tag used for the bundle.
-# You can use it as an arg. (E.g make bundle-build BUNDLE_IMG=<some-registry>/<project-name-bundle>:<tag>)
-BUNDLE_IMG ?= $(IMAGE_TAG_BASE)-bundle:v$(VERSION)
+# ENVTEST_VERSION defines the version of the setup-envtest binary 
+# used to manage and download the Kubernetes binaries (like etcd, 
+# kube-apiserver, and kubectl) required for testing. This version 
+# ensures compatibility with the selected Kubernetes version and 
+# must align closely with recent releases (release-0.19 is chosen here).
+# Mismatches between these versions could result in compatibility issues.
+ENVTEST_VERSION ?= release-0.19
 
 # Image URL to use all building/pushing image targets
-IMG ?= clastix/kamaji:v$(VERSION)
+CONTAINER_REPOSITORY ?= docker.io/clastix/kamaji
 
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
 ifeq (,$(shell go env GOBIN))
@@ -50,6 +34,22 @@ endif
 SHELL = /usr/bin/env bash -o pipefail
 .SHELLFLAGS = -ec
 
+## Location to install dependencies to
+LOCALBIN ?= $(shell pwd)/bin
+$(LOCALBIN):
+	mkdir -p $(LOCALBIN)
+
+## Tool Binaries
+APIDOCS_GEN    ?= $(LOCALBIN)/crdoc
+CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen
+GINKGO         ?= $(LOCALBIN)/ginkgo
+GOLANGCI_LINT  ?= $(LOCALBIN)/golangci-lint
+HELM           ?= $(LOCALBIN)/helm
+KIND           ?= $(LOCALBIN)/kind
+KO             ?= $(LOCALBIN)/ko
+YQ             ?= $(LOCALBIN)/yq
+ENVTEST        ?= $(LOCALBIN)/setup-envtest
+
 all: build
 
 ##@ General
@@ -70,39 +70,79 @@ help: ## Display this help.
 
 ##@ Binary
 
+.PHONY: ko
+ko: $(KO) ## Download ko locally if necessary.
+$(KO): $(LOCALBIN)
+	test -s $(LOCALBIN)/ko || GOBIN=$(LOCALBIN) CGO_ENABLED=0 go install -ldflags="-s -w" github.com/google/ko@v0.14.1
+
+.PHONY: yq
+yq: $(YQ) ## Download yq locally if necessary.
+$(YQ): $(LOCALBIN)
+	test -s $(LOCALBIN)/yq || GOBIN=$(LOCALBIN) CGO_ENABLED=0 go install -ldflags="-s -w" github.com/mikefarah/yq/v4@v4.44.2
+
 .PHONY: helm
-HELM = $(shell pwd)/bin/helm
-helm: ## Download helm locally if necessary.
-	$(call go-install-tool,$(HELM),helm.sh/helm/v3/cmd/helm@v3.9.0)
+helm: $(HELM) ## Download helm locally if necessary.
+$(HELM): $(LOCALBIN)
+	test -s $(LOCALBIN)/helm || GOBIN=$(LOCALBIN) CGO_ENABLED=0 go install -ldflags="-s -w" helm.sh/helm/v3/cmd/helm@v3.9.0
 
-GINKGO = $(shell pwd)/bin/ginkgo
-ginkgo: ## Download ginkgo locally if necessary.
-	$(call go-install-tool,$(GINKGO),github.com/onsi/ginkgo/v2/ginkgo)
+.PHONY: ginkgo
+ginkgo: $(GINKGO) ## Download ginkgo locally if necessary.
+$(GINKGO): $(LOCALBIN)
+	test -s $(LOCALBIN)/ginkgo || GOBIN=$(LOCALBIN) CGO_ENABLED=0 go install -ldflags="-s -w" github.com/onsi/ginkgo/v2/ginkgo
 
-KIND = $(shell pwd)/bin/kind
-kind: ## Download kind locally if necessary.
-	$(call go-install-tool,$(KIND),sigs.k8s.io/kind/cmd/kind@v0.14.0)
+.PHONY: kind
+kind: $(KIND) ## Download kind locally if necessary.
+$(KIND): $(LOCALBIN)
+	test -s $(LOCALBIN)/kind || GOBIN=$(LOCALBIN) CGO_ENABLED=0 go install -ldflags="-s -w" sigs.k8s.io/kind/cmd/kind@v0.14.0
 
-CONTROLLER_GEN = $(shell pwd)/bin/controller-gen
-controller-gen: ## Download controller-gen locally if necessary.
-	$(call go-install-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.14.0)
+.PHONY: controller-gen
+controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessary.
+$(CONTROLLER_GEN): $(LOCALBIN)
+	test -s $(LOCALBIN)/controller-gen || GOBIN=$(LOCALBIN) CGO_ENABLED=0 go install -ldflags="-s -w" sigs.k8s.io/controller-tools/cmd/controller-gen@v0.16.1
 
-GOLANGCI_LINT = $(shell pwd)/bin/golangci-lint
-golangci-lint: ## Download golangci-lint locally if necessary.
-	$(call go-install-tool,$(GOLANGCI_LINT),github.com/golangci/golangci-lint/cmd/golangci-lint@v1.54.2)
+.PHONY: golangci-lint
+golangci-lint: $(GOLANGCI_LINT) ## Download golangci-lint locally if necessary.
+$(GOLANGCI_LINT): $(LOCALBIN)
+	test -s $(LOCALBIN)/golangci-lint || GOBIN=$(LOCALBIN) CGO_ENABLED=0 go install -ldflags="-s -w" github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.0.2
 
-KUSTOMIZE = $(shell pwd)/bin/kustomize
-kustomize: ## Download kustomize locally if necessary.
-	$(call install-kustomize,$(KUSTOMIZE),3.8.7)
+.PHONY: apidocs-gen
+apidocs-gen: $(APIDOCS_GEN)  ## Download crdoc locally if necessary.
+$(APIDOCS_GEN): $(LOCALBIN)
+	test -s $(LOCALBIN)/crdoc || GOBIN=$(LOCALBIN) CGO_ENABLED=0 go install -ldflags="-s -w" fybrik.io/crdoc@latest
 
-APIDOCS_GEN = $(shell pwd)/bin/crdoc
-apidocs-gen: ## Download crdoc locally if necessary.
-	$(call go-install-tool,$(APIDOCS_GEN),fybrik.io/crdoc@latest)
+.PHONY: envtest
+envtest: $(ENVTEST) ## Download envtest-setup locally if necessary.
+$(ENVTEST): $(LOCALBIN)
+	test -s $(LOCALBIN)/setup-envtest || GOBIN=$(LOCALBIN) CGO_ENABLED=0 go install -ldflags="-s -w" sigs.k8s.io/controller-runtime/tools/setup-envtest@$(ENVTEST_VERSION)
 
 ##@ Development
 
-manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects.
-	$(CONTROLLER_GEN) rbac:roleName=manager-role crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases
+rbac: controller-gen yq
+	$(CONTROLLER_GEN) rbac:roleName=manager-role paths="./..." output:stdout | $(YQ) '.rules' > ./charts/kamaji/controller-gen/clusterrole.yaml
+
+webhook: controller-gen yq
+	$(CONTROLLER_GEN) webhook paths="./..." output:stdout | $(YQ) 'select(documentIndex == 0) | .webhooks' > ./charts/kamaji/controller-gen/mutating-webhook.yaml
+	$(CONTROLLER_GEN) webhook paths="./..." output:stdout | $(YQ) 'select(documentIndex == 1) | .webhooks' > ./charts/kamaji/controller-gen/validating-webhook.yaml
+	$(YQ) -i 'map(.clientConfig.service.name |= "{{ include \"kamaji.webhookServiceName\" . }}")' ./charts/kamaji/controller-gen/mutating-webhook.yaml
+	$(YQ) -i 'map(.clientConfig.service.namespace |= "{{ .Release.Namespace }}")' ./charts/kamaji/controller-gen/mutating-webhook.yaml
+	$(YQ) -i 'map(.clientConfig.service.name |= "{{ include \"kamaji.webhookServiceName\" . }}")' ./charts/kamaji/controller-gen/validating-webhook.yaml
+	$(YQ) -i 'map(.clientConfig.service.namespace |= "{{ .Release.Namespace }}")' ./charts/kamaji/controller-gen/validating-webhook.yaml
+
+crds: controller-gen yq
+	# kamaji chart
+	$(CONTROLLER_GEN) crd webhook paths="./..." output:stdout | $(YQ) 'select(documentIndex == 0)' > ./charts/kamaji/crds/kamaji.clastix.io_datastores.yaml
+	$(CONTROLLER_GEN) crd webhook paths="./..." output:stdout | $(YQ) 'select(documentIndex == 1)' > ./charts/kamaji/crds/kamaji.clastix.io_kubeconfiggenerators.yaml
+	$(CONTROLLER_GEN) crd webhook paths="./..." output:stdout | $(YQ) 'select(documentIndex == 2)' > ./charts/kamaji/crds/kamaji.clastix.io_tenantcontrolplanes.yaml
+	$(YQ) -i '. *n load("./charts/kamaji/controller-gen/crd-conversion.yaml")' ./charts/kamaji/crds/kamaji.clastix.io_tenantcontrolplanes.yaml
+	# kamaji-crds chart
+	cp ./charts/kamaji/controller-gen/crd-conversion.yaml ./charts/kamaji-crds/hack/crd-conversion.yaml
+	$(YQ) '.spec' ./charts/kamaji/crds/kamaji.clastix.io_datastores.yaml > ./charts/kamaji-crds/hack/kamaji.clastix.io_datastores_spec.yaml
+	$(YQ) '.spec' ./charts/kamaji/crds/kamaji.clastix.io_tenantcontrolplanes.yaml > ./charts/kamaji-crds/hack/kamaji.clastix.io_tenantcontrolplanes_spec.yaml
+	$(YQ) '.spec' ./charts/kamaji/crds/kamaji.clastix.io_kubeconfiggenerators.yaml > ./charts/kamaji-crds/hack/kamaji.clastix.io_kubeconfiggenerators_spec.yaml
+	$(YQ) -i '.conversion.webhook.clientConfig.service.name = "{{ .Values.kamajiService }}"' ./charts/kamaji-crds/hack/kamaji.clastix.io_tenantcontrolplanes_spec.yaml
+	$(YQ) -i '.conversion.webhook.clientConfig.service.namespace = "{{ .Values.kamajiNamespace }}"' ./charts/kamaji-crds/hack/kamaji.clastix.io_tenantcontrolplanes_spec.yaml
+
+manifests: rbac webhook crds ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects.
 
 generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations.
 	$(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..."
@@ -110,8 +150,13 @@ generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and
 golint: golangci-lint ## Linting the code according to the styling guide.
 	$(GOLANGCI_LINT) run -c .golangci.yml
 
-test:
-	go test ./... -coverprofile cover.out
+## Run unit tests (all tests except E2E).
+.PHONY: test
+test: envtest ginkgo
+	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" $(GINKGO) -r -v -coverprofile cover.out --trace \
+		./api/... \
+		./cmd/... \
+		./internal/... \
 
 _datastore-mysql:
 	$(MAKE) NAME=$(NAME) -C deploy/kine/mysql mariadb
@@ -159,161 +204,71 @@ datastores: datastore-mysql datastore-etcd datastore-postgres datastore-nats ##
 
 # Get information about git current status
 GIT_HEAD_COMMIT ?= $$(git rev-parse --short HEAD)
-GIT_TAG_COMMIT  ?= $$(git rev-parse --short v$(VERSION))
+GIT_TAG_COMMIT  ?= $$(git rev-parse --short $(VERSION))
 GIT_MODIFIED_1  ?= $$(git diff $(GIT_HEAD_COMMIT) $(GIT_TAG_COMMIT) --quiet && echo "" || echo ".dev")
 GIT_MODIFIED_2  ?= $$(git diff --quiet && echo "" || echo ".dirty")
 GIT_MODIFIED    ?= $$(echo "$(GIT_MODIFIED_1)$(GIT_MODIFIED_2)")
 GIT_REPO        ?= $$(git config --get remote.origin.url)
 BUILD_DATE      ?= $$(git log -1 --format="%at" | xargs -I{} date -d @{} +%Y-%m-%dT%H:%M:%S)
 
-.PHONY: get_version
-get_version:
-	@echo -n v$(VERSION)
+LD_FLAGS ?= "-X github.com/clastix/kamaji/internal.GitCommit=$(GIT_HEAD_COMMIT) \
+             -X github.com/clastix/kamaji/internal.GitTag=$(VERSION) \
+             -X github.com/clastix/kamaji/internal.GitDirty=$(GIT_MODIFIED) \
+             -X github.com/clastix/kamaji/internal.BuildTime=$(BUILD_DATE) \
+             -X github.com/clastix/kamaji/internal.GitRepo=$(GIT_REPO)"
 
-build: generate fmt vet ## Build manager binary.
-	go build -o bin/manager main.go
+KO_PUSH ?= false
+KO_LOCAL ?= true
 
-run: manifests generate fmt vet ## Run a controller from your host.
+run: manifests generate ## Run a controller from your host.
 	go run ./main.go
 
-docker-build: ## Build docker image with the manager.
-	docker build -t ${IMG} . --build-arg GIT_HEAD_COMMIT=$(GIT_HEAD_COMMIT) \
-		--build-arg GIT_TAG_COMMIT=$(GIT_TAG_COMMIT) \
-		--build-arg GIT_MODIFIED=$(GIT_MODIFIED) \
-		--build-arg GIT_REPO=$(GIT_REPO) \
-		--build-arg GIT_LAST_TAG=v$(VERSION) \
-		--build-arg BUILD_DATE=$(BUILD_DATE)
+build: $(KO)
+	LD_FLAGS=$(LD_FLAGS) \
+	KOCACHE=/tmp/ko-cache KO_DOCKER_REPO=${CONTAINER_REPOSITORY} \
+	$(KO) build ./ --bare --tags=$(VERSION) --local=$(KO_LOCAL) --push=$(KO_PUSH)
 
-docker-push: ## Push docker image with the manager.
-	docker push ${IMG}
-
-##@ Deployment
+##@ Development
 
 metallb:
-	kubectl apply -f https://raw.githubusercontent.com/metallb/metallb/v0.13.7/config/manifests/metallb-native.yaml
-	kubectl apply -f https://kind.sigs.k8s.io/examples/loadbalancer/metallb-config.yaml
-	echo ""
-	docker network inspect -f '{{.IPAM.Config}}' kind
+	kubectl apply -f "https://raw.githubusercontent.com/metallb/metallb/$$(curl "https://api.github.com/repos/metallb/metallb/releases/latest" | jq -r ".tag_name")/config/manifests/metallb-native.yaml"
+	kubectl wait pods -n metallb-system -l app=metallb,component=controller --for=condition=Ready --timeout=10m
+	kubectl wait pods -n metallb-system -l app=metallb,component=speaker --for=condition=Ready --timeout=2m
+	cat hack/metallb.yaml | sed -E "s|172.19|$$(docker network inspect -f '{{range .IPAM.Config}}{{.Gateway}}{{end}}' kind | sed -E 's|^([0-9]+\.[0-9]+)\..*$$|\1|g')|g" | kubectl apply -f -
 
 cert-manager:
-	kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.10.1/cert-manager.yaml
+	$(HELM) repo add jetstack https://charts.jetstack.io
+	$(HELM) upgrade --install cert-manager jetstack/cert-manager --namespace certmanager-system --create-namespace --set "installCRDs=true"
 
-dev: generate manifests uninstall install rbac ## Full installation for development purposes
-	go fmt ./...
-
-load: docker-build kind
-	$(KIND) load docker-image --name kamaji ${IMG}
-
-rbac: manifests kustomize ## Install RBAC into the K8s cluster specified in ~/.kube/config.
-	$(KUSTOMIZE) build config/rbac | kubectl apply -f -
-
-install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config.
-	$(KUSTOMIZE) build config/crd | kubectl apply -f -
-
-uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config.
-	$(KUSTOMIZE) build config/crd | kubectl delete -f -
-
-deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config.
-	cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
-	$(KUSTOMIZE) build config/default | kubectl apply -f -
-
-undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config.
-	$(KUSTOMIZE) build config/default | kubectl delete -f -
-
-yaml-installation-file: manifests kustomize ## Create yaml installation file
-	$(KUSTOMIZE) build config/default > config/install.yaml
-
-.PHONY: bundle
-bundle: manifests kustomize ## Generate bundle manifests and metadata, then validate generated files.
-	operator-sdk generate kustomize manifests -q
-	cd config/manager && $(KUSTOMIZE) edit set image controller=$(IMG)
-	$(KUSTOMIZE) build config/manifests | operator-sdk generate bundle -q --overwrite --version $(VERSION) $(BUNDLE_METADATA_OPTS)
-	operator-sdk bundle validate ./bundle
-
-.PHONY: bundle-build
-bundle-build: ## Build the bundle image.
-	docker build -f bundle.Dockerfile -t $(BUNDLE_IMG) .
-
-.PHONY: bundle-push
-bundle-push: ## Push the bundle image.
-	$(MAKE) docker-push IMG=$(BUNDLE_IMG)
-
-.PHONY: opm
-OPM = ./bin/opm
-opm: ## Download opm locally if necessary.
-ifeq (,$(wildcard $(OPM)))
-ifeq (,$(shell which opm 2>/dev/null))
-	@{ \
-	set -e ;\
-	mkdir -p $(dir $(OPM)) ;\
-	OS=$(shell go env GOOS) && ARCH=$(shell go env GOARCH) && \
-	curl -sSLo $(OPM) https://github.com/operator-framework/operator-registry/releases/download/v1.15.1/$${OS}-$${ARCH}-opm ;\
-	chmod +x $(OPM) ;\
-	}
-else
-OPM = $(shell which opm)
-endif
-endif
-
-# A comma-separated list of bundle images (e.g. make catalog-build BUNDLE_IMGS=example.com/operator-bundle:v0.1.0,example.com/operator-bundle:v0.2.0).
-# These images MUST exist in a registry and be pull-able.
-BUNDLE_IMGS ?= $(BUNDLE_IMG)
-
-# The image tag given to the resulting catalog image (e.g. make catalog-build CATALOG_IMG=example.com/operator-catalog:v0.2.0).
-CATALOG_IMG ?= $(IMAGE_TAG_BASE)-catalog:v$(VERSION)
-
-# Set CATALOG_BASE_IMG to an existing catalog image tag to add $BUNDLE_IMGS to that image.
-ifneq ($(origin CATALOG_BASE_IMG), undefined)
-FROM_INDEX_OPT := --from-index $(CATALOG_BASE_IMG)
-endif
-
-# Build a catalog image by adding bundle images to an empty catalog using the operator package manager tool, 'opm'.
-# This recipe invokes 'opm' in 'semver' bundle add mode. For more information on add modes, see:
-# https://github.com/operator-framework/community-operators/blob/7f1438c/docs/packaging-operator.md#updating-your-existing-operator
-.PHONY: catalog-build
-catalog-build: opm ## Build a catalog image.
-	$(OPM) index add --container-tool docker --mode semver --tag $(CATALOG_IMG) --bundles $(BUNDLE_IMGS) $(FROM_INDEX_OPT)
-
-# Push the catalog image.
-.PHONY: catalog-push
-catalog-push: ## Push a catalog image.
-	$(MAKE) docker-push IMG=$(CATALOG_IMG)
-
-define install-kustomize
-@[ -f $(1) ] || { \
-set -e ;\
-echo "Installing v$(2)" ;\
-cd bin ;\
-wget "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" ;\
-bash ./install_kustomize.sh $(2) ;\
-}
-endef
-
-# go-install-tool will 'go install' any package $2 and install it to $1.
-PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST))))
-define go-install-tool
-@[ -f $(1) ] || { \
-set -e ;\
-echo "Installing $(2)" ;\
-GOBIN=$(PROJECT_DIR)/bin go install $(2) ;\
-}
-endef
-
-.PHONY: env
-env:
-	@make -C deploy/kind kind ingress-nginx
+load: kind
+	$(KIND) load docker-image --name kamaji ${CONTAINER_REPOSITORY}:${VERSION}
 
 ##@ e2e
 
+.PHONY: env
+env: kind
+	$(KIND) create cluster --name kamaji
+
 .PHONY: e2e
-e2e: env load helm ginkgo cert-manager ## Create a KinD cluster, install Kamaji on it and run the test suite.
+e2e: env build load helm ginkgo cert-manager ## Create a KinD cluster, install Kamaji on it and run the test suite.
+	$(HELM) upgrade --debug --install kamaji-crds ./charts/kamaji-crds --create-namespace --namespace kamaji-system
 	$(HELM) repo add clastix https://clastix.github.io/charts
 	$(HELM) dependency build ./charts/kamaji
-	$(HELM) upgrade --debug --install kamaji ./charts/kamaji --create-namespace --namespace kamaji-system --set "image.pullPolicy=Never" --set "telemetry.disabled=true"
+	$(HELM) upgrade --debug --install kamaji ./charts/kamaji --create-namespace --namespace kamaji-system --set "image.tag=$(VERSION)" --set "image.pullPolicy=Never" --set "telemetry.disabled=true"
 	$(MAKE) datastores
 	$(GINKGO) -v ./e2e
 
 ##@ Document
 
+CAPI_URL = https://github.com/clastix/cluster-api-control-plane-provider-kamaji.git
+CAPI_DIR := $(shell mktemp -d)
+CRDS_DIR := $(shell mktemp -d)
+
+.PHONY: apidoc
 apidoc: apidocs-gen
-	$(APIDOCS_GEN) crdoc --resources config/crd/bases --output docs/content/reference/api.md --template docs/templates/reference-cr.tmpl
+	@cp charts/kamaji/crds/*.yaml $(CRDS_DIR)
+	@git clone $(CAPI_URL) $(CAPI_DIR)
+	@cp $(CAPI_DIR)/config/crd/bases/*.yaml $(CRDS_DIR)
+	@rm -rf $(CAPI_DIR)
+	$(APIDOCS_GEN) crdoc --resources $(CRDS_DIR) --output docs/content/reference/api.md --template docs/templates/reference-cr.tmpl
+	@rm -rf $(CRDS_DIR)

NOTICE

@@ -0,0 +1,15 @@
+Kamaji — The Kubernetes Control Plane Manager: copyright 2022 Clastix Labs
+Licensed under the Apache License, Version 2.0: https://kamaji.clastix.io
+
+This product includes software developed by Clastix Labs and the Kamaji open-source community under the Apache License, Version 2.0.
+
+Kamaji powers Kubernetes Control Planes at scale for companies worldwide.
+
+We encourage all commercial products and services using Kamaji to acknowledge this publicly and join our growing ecosystem of adopters.
+
+You can support the Kamaji community by:
+- Listing Kamaji in your product's "Open Source Credits" or similar section
+- Adding your organization to the Adopters list on GitHub: https://github.com/clastix/kamaji/blob/master/ADOPTERS.md
+- Mentioning Kamaji on your company or product website
+
+Public acknowledgement strengthens the open-source ecosystem and helps ensure the sustainability of the project you rely on.

PROJECT

@@ -7,6 +7,15 @@ plugins:
 projectName: operator
 repo: github.com/clastix/kamaji
 resources:
+- api:
+    crdVersion: v1
+    namespaced: false
+  controller: true
+  domain: clastix.io
+  group: kamaji
+  kind: KubeconfigGenerator
+  path: github.com/clastix/kamaji/api/v1alpha1
+  version: v1alpha1
 - api:
     crdVersion: v1
     namespaced: true

README.md

@@ -13,7 +13,7 @@
 
 ### 🤔 What is Kamaji?
 
-**Kamaji** is a **Kubernetes Control Plane Manager** leveraging on the concept of [**Hosted Control Plane**](https://clastix.io/post/the-raise-of-hosted-control-plane-in-kubernetes/).
+**Kamaji** is the **Kubernetes Control Plane Manager** leveraging on the concept of [**Hosted Control Plane**](https://clastix.io/post/the-raise-of-hosted-control-plane-in-kubernetes/).
 
 Kamaji's approach is based on running the Kubernetes Control Plane components in Pods instead of dedicated machines.
 This allows operating Kubernetes clusters at scale, with a fraction of the operational burden.
@@ -148,7 +148,7 @@ In case of **✨ Feature Requests** please use the [Discussion's Feature Request
 
 ### 📝 License
 
-The Kamaji Cluster API Control Plane provider is licensed under Apache 2.0.
+Kamaji is licensed under Apache 2.0.
 The code is provided as-is with no warranties.
 
 ### 🛟 Commercial Support

api/v1alpha1/datastore_types.go

@@ -8,7 +8,8 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
-// +kubebuilder:validation:Enum=etcd;MySQL;PostgreSQL;NATS
+//+kubebuilder:validation:Enum=etcd;MySQL;PostgreSQL;NATS
+//+kubebuilder:validation:XValidation:rule="self == oldSelf",message="Datastore driver is immutable"
 
 type Driver string
 
@@ -19,11 +20,18 @@ var (
 	KineNatsDriver       Driver = "NATS"
 )
 
-// +kubebuilder:validation:MinItems=1
+//+kubebuilder:validation:MinItems=1
 
 type Endpoints []string
 
 // DataStoreSpec defines the desired state of DataStore.
+// +kubebuilder:validation:XValidation:rule="(self.driver == \"etcd\") ? (self.tlsConfig != null && (has(self.tlsConfig.certificateAuthority.privateKey.secretReference) || has(self.tlsConfig.certificateAuthority.privateKey.content))) : true", message="certificateAuthority privateKey must have secretReference or content when driver is etcd"
+// +kubebuilder:validation:XValidation:rule="(self.driver == \"etcd\") ? (self.tlsConfig != null && (has(self.tlsConfig.clientCertificate.certificate.secretReference) || has(self.tlsConfig.clientCertificate.certificate.content))) : true", message="clientCertificate must have secretReference or content when driver is etcd"
+// +kubebuilder:validation:XValidation:rule="(self.driver == \"etcd\") ? (self.tlsConfig != null && (has(self.tlsConfig.clientCertificate.privateKey.secretReference) || has(self.tlsConfig.clientCertificate.privateKey.content))) : true", message="clientCertificate privateKey must have secretReference or content when driver is etcd"
+// +kubebuilder:validation:XValidation:rule="(self.driver != \"etcd\" && has(self.tlsConfig) && has(self.tlsConfig.clientCertificate)) ? (((has(self.tlsConfig.clientCertificate.certificate.secretReference) || has(self.tlsConfig.clientCertificate.certificate.content)))) : true", message="When driver is not etcd and tlsConfig exists, clientCertificate must be null or contain valid content"
+// +kubebuilder:validation:XValidation:rule="(self.driver != \"etcd\" && has(self.basicAuth)) ? ((has(self.basicAuth.username.secretReference) || has(self.basicAuth.username.content))) : true", message="When driver is not etcd and basicAuth exists, username must have secretReference or content"
+// +kubebuilder:validation:XValidation:rule="(self.driver != \"etcd\" && has(self.basicAuth)) ? ((has(self.basicAuth.password.secretReference) || has(self.basicAuth.password.content))) : true", message="When driver is not etcd and basicAuth exists, password must have secretReference or content"
+// +kubebuilder:validation:XValidation:rule="(self.driver != \"etcd\") ? (has(self.tlsConfig) || has(self.basicAuth)) : true", message="When driver is not etcd, either tlsConfig or basicAuth must be provided"
 type DataStoreSpec struct {
 	// The driver to use to connect to the shared datastore.
 	Driver Driver `json:"driver"`
@@ -91,6 +99,7 @@ type DataStoreStatus struct {
 //+kubebuilder:resource:scope=Cluster
 //+kubebuilder:printcolumn:name="Driver",type="string",JSONPath=".spec.driver",description="Kamaji data store driver"
 //+kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="Age"
+//+kubebuilder:metadata:annotations={"cert-manager.io/inject-ca-from=kamaji-system/kamaji-serving-cert"}
 
 // DataStore is the Schema for the datastores API.
 type DataStore struct {

api/v1alpha1/indexer_datastore_usedsecret.go

@@ -52,12 +52,14 @@ func (d *DatastoreUsedSecret) ExtractValue() client.IndexerFunc {
 				res = append(res, d.namespacedName(*ds.Spec.TLSConfig.CertificateAuthority.PrivateKey.SecretRef))
 			}
 
-			if ds.Spec.TLSConfig.ClientCertificate.Certificate.SecretRef != nil {
-				res = append(res, d.namespacedName(*ds.Spec.TLSConfig.ClientCertificate.Certificate.SecretRef))
-			}
+			if ds.Spec.TLSConfig.ClientCertificate != nil {
+				if ds.Spec.TLSConfig.ClientCertificate.Certificate.SecretRef != nil {
+					res = append(res, d.namespacedName(*ds.Spec.TLSConfig.ClientCertificate.Certificate.SecretRef))
+				}
 
-			if ds.Spec.TLSConfig.ClientCertificate.PrivateKey.SecretRef != nil {
-				res = append(res, d.namespacedName(*ds.Spec.TLSConfig.ClientCertificate.PrivateKey.SecretRef))
+				if ds.Spec.TLSConfig.ClientCertificate.PrivateKey.SecretRef != nil {
+					res = append(res, d.namespacedName(*ds.Spec.TLSConfig.ClientCertificate.PrivateKey.SecretRef))
+				}
 			}
 		}
 

api/v1alpha1/kubeconfiggenerator_types.go

@@ -0,0 +1,91 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package v1alpha1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+var (
+	ManagedByLabel  = "kamaji.clastix.io/managed-by"
+	ManagedForLabel = "kamaji.clastix.io/managed-for"
+)
+
+//+kubebuilder:object:root=true
+//+kubebuilder:subresource:status
+//+kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="Age"
+//+kubebuilder:metadata:annotations={"cert-manager.io/inject-ca-from=kamaji-system/kamaji-serving-cert"}
+//+kubebuilder:resource:scope=Cluster,shortName=kc,categories=kamaji
+
+// KubeconfigGenerator is the Schema for the kubeconfiggenerators API.
+type KubeconfigGenerator struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec   KubeconfigGeneratorSpec   `json:"spec,omitempty"`
+	Status KubeconfigGeneratorStatus `json:"status,omitempty"`
+}
+
+// CompoundValue allows defining a static, or a dynamic value.
+// Options are mutually exclusive, just one should be picked up.
+// +kubebuilder:validation:XValidation:rule="(has(self.stringValue) || has(self.fromDefinition)) && !(has(self.stringValue) && has(self.fromDefinition))",message="Either stringValue or fromDefinition must be set, but not both."
+type CompoundValue struct {
+	// StringValue is a static string value.
+	StringValue string `json:"stringValue,omitempty"`
+	// FromDefinition is used to generate a dynamic value,
+	// it uses the dot notation to access fields from the referenced TenantControlPlane object:
+	// e.g.: metadata.name
+	FromDefinition string `json:"fromDefinition,omitempty"`
+}
+
+type KubeconfigGeneratorSpec struct {
+	// NamespaceSelector is used to filter Namespaces from which the generator should extract TenantControlPlane objects.
+	NamespaceSelector metav1.LabelSelector `json:"namespaceSelector,omitempty"`
+	// TenantControlPlaneSelector is used to filter the TenantControlPlane objects that should be address by the generator.
+	TenantControlPlaneSelector metav1.LabelSelector `json:"tenantControlPlaneSelector,omitempty"`
+	// Groups is resolved a set of strings used to assign the x509 organisations field.
+	// It will be recognised by Kubernetes as user groups.
+	Groups []CompoundValue `json:"groups,omitempty"`
+	// User resolves to a string to identify the client, assigned to the x509 Common Name field.
+	User CompoundValue `json:"user"`
+	// ControlPlaneEndpointFrom is the key used to extract the Tenant Control Plane endpoint that must be used by the generator.
+	// The targeted Secret is the `${TCP}-admin-kubeconfig` one, default to `admin.svc`.
+	//+kubebuilder:default="admin.svc"
+	ControlPlaneEndpointFrom string `json:"controlPlaneEndpointFrom,omitempty"`
+}
+
+type KubeconfigGeneratorStatusError struct {
+	// Resource is the Namespaced name of the errored resource.
+	//+kubebuilder:validation:Required
+	Resource string `json:"resource"`
+	// Message is the error message recorded upon the last generator run.
+	//+kubebuilder:validation:Required
+	Message string `json:"message"`
+}
+
+// KubeconfigGeneratorStatus defines the observed state of KubeconfigGenerator.
+type KubeconfigGeneratorStatus struct {
+	// Resources is the sum of targeted TenantControlPlane objects.
+	//+kubebuilder:default=0
+	Resources int `json:"resources"`
+	// AvailableResources is the sum of successfully generated resources.
+	// In case of a different value compared to Resources, check the field errors.
+	//+kubebuilder:default=0
+	AvailableResources int `json:"availableResources"`
+	// Errors is the list of failed kubeconfig generations.
+	Errors []KubeconfigGeneratorStatusError `json:"errors,omitempty"`
+}
+
+//+kubebuilder:object:root=true
+
+// KubeconfigGeneratorList contains a list of TenantControlPlane.
+type KubeconfigGeneratorList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []KubeconfigGenerator `json:"items"`
+}
+
+func init() {
+	SchemeBuilder.Register(&KubeconfigGenerator{}, &KubeconfigGeneratorList{})
+}

api/v1alpha1/suite_test.go

@@ -0,0 +1,55 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package v1alpha1
+
+import (
+	"path/filepath"
+	"testing"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/rest"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/envtest"
+)
+
+var (
+	cfg       *rest.Config
+	k8sClient client.Client
+	testEnv   *envtest.Environment
+)
+
+func TestAPIs(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "TenantControlPlane Suite")
+}
+
+var _ = BeforeSuite(func() {
+	By("bootstrapping test environment")
+	testEnv = &envtest.Environment{
+		CRDDirectoryPaths: []string{
+			filepath.Join("..", "..", "charts", "kamaji", "crds"),
+			// filepath.Join("../..", "chart", "kamaji", "crds"),
+		},
+	}
+
+	var err error
+	cfg, err = testEnv.Start()
+	Expect(err).ToNot(HaveOccurred())
+	Expect(cfg).ToNot(BeNil())
+
+	err = AddToScheme(scheme.Scheme)
+	Expect(err).NotTo(HaveOccurred())
+
+	k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme})
+	Expect(err).ToNot(HaveOccurred())
+	Expect(k8sClient).ToNot(BeNil())
+})
+
+var _ = AfterSuite(func() {
+	By("tearing down the test environment")
+	err := testEnv.Stop()
+	Expect(err).ToNot(HaveOccurred())
+})

api/v1alpha1/tenantcontrolplane_const.go

@@ -0,0 +1,10 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package v1alpha1
+
+const (
+	// PausedReconciliationAnnotation is an annotation that can be applied to
+	// Tenant Control Plane objects to prevent the controller from processing such a resource.
+	PausedReconciliationAnnotation = "kamaji.clastix.io/paused"
+)

api/v1alpha1/tenantcontrolplane_funcs.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"net"
 	"strconv"
+	"strings"
 
 	"github.com/pkg/errors"
 	corev1 "k8s.io/api/core/v1"
@@ -61,12 +62,51 @@ func (in *TenantControlPlane) DeclaredControlPlaneAddress(ctx context.Context, c
 			return "", kamajierrors.NonExposedLoadBalancerError{}
 		}
 
-		for _, lb := range loadBalancerStatus.Ingress {
-			if ip := lb.IP; len(ip) > 0 {
-				return ip, nil
-			}
+		return getLoadBalancerAddress(loadBalancerStatus.Ingress)
+	}
+
+	return "", kamajierrors.MissingValidIPError{}
+}
+
+// getLoadBalancerAddress extracts the IP address from LoadBalancer ingress.
+// It also checks and rejects hostname usage for LoadBalancer ingress.
+//
+// Reasons for not supporting hostnames:
+// - DNS resolution can differ across environments, leading to inconsistent behavior.
+// - It may cause connectivity problems between Kubernetes components.
+// - The DNS resolution could change over time, potentially breaking cluster-to-API-server connections.
+//
+// Recommended solutions:
+// - Use a static IP address to ensure stable and predictable communication within the cluster.
+// - If a hostname is necessary, consider setting up a Virtual IP (VIP) for the given hostname.
+// - Alternatively, use an external load balancer that can provide a stable IP address.
+//
+// Note: Implementing L7 routing with the API Server requires a deep understanding of the implications.
+// Users should be aware of the complexities involved, including potential issues with TLS passthrough
+// for client-based certificate authentication in Ingress expositions.
+func getLoadBalancerAddress(ingress []corev1.LoadBalancerIngress) (string, error) {
+	for _, lb := range ingress {
+		if ip := lb.IP; len(ip) > 0 {
+			return ip, nil
+		}
+		if hostname := lb.Hostname; len(hostname) > 0 {
+			return "", fmt.Errorf("hostname not supported for LoadBalancer ingress: use static IP instead")
 		}
 	}
 
 	return "", kamajierrors.MissingValidIPError{}
 }
+
+func (in *TenantControlPlane) normalizeNamespaceName() string {
+	// The dash character (-) must be replaced with an underscore, PostgreSQL is complaining about it:
+	// https://github.com/clastix/kamaji/issues/328
+	return strings.ReplaceAll(fmt.Sprintf("%s_%s", in.GetNamespace(), in.GetName()), "-", "_")
+}
+
+func (in *TenantControlPlane) GetDefaultDatastoreUsername() string {
+	return in.normalizeNamespaceName()
+}
+
+func (in *TenantControlPlane) GetDefaultDatastoreSchema() string {
+	return in.normalizeNamespaceName()
+}

api/v1alpha1/tenantcontrolplane_interfaces.go

@@ -8,5 +8,5 @@ package v1alpha1
 // +kubebuilder:object:generate=false
 type KubeadmConfigChecksumDependant interface {
 	GetChecksum() string
-	SetChecksum(string)
+	SetChecksum(checksum string)
 }

api/v1alpha1/tenantcontrolplane_kubeadmphase_funcs.go

@@ -7,7 +7,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
-func (in KubeadmPhaseStatus) GetChecksum() string {
+func (in *KubeadmPhaseStatus) GetChecksum() string {
 	return in.Checksum
 }
 

api/v1alpha1/tenantcontrolplane_registrysettings.go

@@ -4,15 +4,15 @@
 package v1alpha1
 
 type RegistrySettings struct {
-	// +kubebuilder:default="registry.k8s.io"
+	//+kubebuilder:default="registry.k8s.io"
 	Registry string `json:"registry,omitempty"`
 	// The tag to append to all the Control Plane container images.
 	// Optional.
 	TagSuffix string `json:"tagSuffix,omitempty"`
-	// +kubebuilder:default="kube-apiserver"
+	//+kubebuilder:default="kube-apiserver"
 	APIServerImage string `json:"apiServerImage,omitempty"`
-	// +kubebuilder:default="kube-controller-manager"
+	//+kubebuilder:default="kube-controller-manager"
 	ControllerManagerImage string `json:"controllerManagerImage,omitempty"`
-	// +kubebuilder:default="kube-scheduler"
+	//+kubebuilder:default="kube-scheduler"
 	SchedulerImage string `json:"schedulerImage,omitempty"`
 }

api/v1alpha1/tenantcontrolplane_status.go

@@ -122,6 +122,12 @@ type ExternalKubernetesObjectStatus struct {
 	LastUpdate metav1.Time `json:"lastUpdate,omitempty"`
 }
 
+type KonnectivityAgentStatus struct {
+	ExternalKubernetesObjectStatus `json:",inline"`
+
+	Mode KonnectivityAgentMode `json:"mode,omitempty"`
+}
+
 // KonnectivityStatus defines the status of Konnectivity as Addon.
 type KonnectivityStatus struct {
 	Enabled            bool                            `json:"enabled"`
@@ -130,7 +136,7 @@ type KonnectivityStatus struct {
 	Kubeconfig         KubeconfigStatus                `json:"kubeconfig,omitempty"`
 	ServiceAccount     ExternalKubernetesObjectStatus  `json:"sa,omitempty"`
 	ClusterRoleBinding ExternalKubernetesObjectStatus  `json:"clusterrolebinding,omitempty"`
-	Agent              ExternalKubernetesObjectStatus  `json:"agent,omitempty"`
+	Agent              KonnectivityAgentStatus         `json:"agent,omitempty"`
 	Service            KubernetesServiceStatus         `json:"service,omitempty"`
 }
 
@@ -183,11 +189,12 @@ type KubernetesStatus struct {
 	Ingress    *KubernetesIngressStatus   `json:"ingress,omitempty"`
 }
 
-// +kubebuilder:validation:Enum=Provisioning;CertificateAuthorityRotating;Upgrading;Migrating;Ready;NotReady
+// +kubebuilder:validation:Enum=Provisioning;CertificateAuthorityRotating;Upgrading;Migrating;Ready;NotReady;Sleeping
 type KubernetesVersionStatus string
 
 var (
 	VersionProvisioning KubernetesVersionStatus = "Provisioning"
+	VersionSleeping     KubernetesVersionStatus = "Sleeping"
 	VersionCARotating   KubernetesVersionStatus = "CertificateAuthorityRotating"
 	VersionUpgrading    KubernetesVersionStatus = "Upgrading"
 	VersionMigrating    KubernetesVersionStatus = "Migrating"
@@ -198,7 +205,7 @@ var (
 type KubernetesVersion struct {
 	// Version is the running Kubernetes version of the Tenant Control Plane.
 	Version string `json:"version,omitempty"`
-	// +kubebuilder:default=Provisioning
+	//+kubebuilder:default=Provisioning
 	// Status returns the current status of the Kubernetes version, such as its provisioning state, or completed upgrade.
 	Status *KubernetesVersionStatus `json:"status,omitempty"`
 }

api/v1alpha1/tenantcontrolplane_types.go

@@ -11,25 +11,46 @@ import (
 
 // NetworkProfileSpec defines the desired state of NetworkProfile.
 type NetworkProfileSpec struct {
+	// LoadBalancerSourceRanges restricts the IP ranges that can access
+	// the LoadBalancer type Service. This field defines a list of IP
+	// address ranges (in CIDR format) that are allowed to access the service.
+	// If left empty, the service will allow traffic from all IP ranges (0.0.0.0/0).
+	// This feature is useful for restricting access to API servers or services
+	// to specific networks for security purposes.
+	// Example: {"192.168.1.0/24", "10.0.0.0/8"}
+	LoadBalancerSourceRanges []string `json:"loadBalancerSourceRanges,omitempty"`
+	// Specify the LoadBalancer class in case of multiple load balancer implementations.
+	// Field supported only for Tenant Control Plane instances exposed using a LoadBalancer Service.
+	// +kubebuilder:validation:MinLength=1
+	// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="LoadBalancerClass is immutable"
+	LoadBalancerClass *string `json:"loadBalancerClass,omitempty"`
 	// Address where API server of will be exposed.
 	// In case of LoadBalancer Service, this can be empty in order to use the exposed IP provided by the cloud controller manager.
 	Address string `json:"address,omitempty"`
+	// The default domain name used for DNS resolution within the cluster.
+	//+kubebuilder:default="cluster.local"
+	//+kubebuilder:validation:XValidation:rule="self == oldSelf",message="changing the cluster domain is not supported"
+	//+kubebuilder:validation:Pattern=.*\..*
+	ClusterDomain string `json:"clusterDomain,omitempty"`
 	// AllowAddressAsExternalIP will include tenantControlPlane.Spec.NetworkProfile.Address in the section of
 	// ExternalIPs of the Kubernetes Service (only ClusterIP or NodePort)
 	AllowAddressAsExternalIP bool `json:"allowAddressAsExternalIP,omitempty"`
 	// Port where API server of will be exposed
-	// +kubebuilder:default=6443
+	//+kubebuilder:default=6443
 	Port int32 `json:"port,omitempty"`
 	// CertSANs sets extra Subject Alternative Names (SANs) for the API Server signing certificate.
 	// Use this field to add additional hostnames when exposing the Tenant Control Plane with third solutions.
 	CertSANs []string `json:"certSANs,omitempty"`
-	// Kubernetes Service
-	// +kubebuilder:default="10.96.0.0/16"
+	// CIDR for Kubernetes Services: if empty, defaulted to 10.96.0.0/16.
+	//+kubebuilder:default="10.96.0.0/16"
 	ServiceCIDR string `json:"serviceCidr,omitempty"`
-	// CIDR for Kubernetes Pods
-	// +kubebuilder:default="10.244.0.0/16"
+	// CIDR for Kubernetes Pods: if empty, defaulted to 10.244.0.0/16.
+	//+kubebuilder:default="10.244.0.0/16"
 	PodCIDR string `json:"podCidr,omitempty"`
-	// +kubebuilder:default={"10.96.0.10"}
+	// The DNS Service for internal resolution, it must match the Service CIDR.
+	// In case of an empty value, it is automatically computed according to the Service CIDR, e.g.:
+	// Service CIDR 10.96.0.0/16, the resulting DNS Service IP will be 10.96.0.10 for IPv4,
+	// for IPv6 from the CIDR 2001:db8:abcd::/64 the resulting DNS Service IP will be 2001:db8:abcd::10.
 	DNSServiceIPs []string `json:"dnsServiceIPs,omitempty"`
 }
 
@@ -46,11 +67,12 @@ const (
 
 type KubeletSpec struct {
 	// Ordered list of the preferred NodeAddressTypes to use for kubelet connections.
-	// Default to Hostname, InternalIP, ExternalIP.
-	// +kubebuilder:default={"Hostname","InternalIP","ExternalIP"}
-	// +kubebuilder:validation:MinItems=1
+	// Default to InternalIP, ExternalIP, Hostname.
+	//+kubebuilder:default={"InternalIP","ExternalIP","Hostname"}
+	//+kubebuilder:validation:MinItems=1
+	//+listType=set
 	PreferredAddressTypes []KubeletPreferredAddressType `json:"preferredAddressTypes,omitempty"`
-	// CGroupFS defines the  cgroup driver for Kubelet
+	// CGroupFS defines the cgroup driver for Kubelet
 	// https://kubernetes.io/docs/tasks/administer-cluster/kubeadm/configure-cgroup-driver/
 	CGroupFS CGroupDriver `json:"cgroupfs,omitempty"`
 }
@@ -63,7 +85,7 @@ type KubernetesSpec struct {
 
 	// List of enabled Admission Controllers for the Tenant cluster.
 	// Full reference available here: https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers
-	// +kubebuilder:default=CertificateApproval;CertificateSigning;CertificateSubjectRestriction;DefaultIngressClass;DefaultStorageClass;DefaultTolerationSeconds;LimitRanger;MutatingAdmissionWebhook;NamespaceLifecycle;PersistentVolumeClaimResize;Priority;ResourceQuota;RuntimeClass;ServiceAccount;StorageObjectInUseProtection;TaintNodesByCondition;ValidatingAdmissionWebhook
+	//+kubebuilder:default=CertificateApproval;CertificateSigning;CertificateSubjectRestriction;DefaultIngressClass;DefaultStorageClass;DefaultTolerationSeconds;LimitRanger;MutatingAdmissionWebhook;NamespaceLifecycle;PersistentVolumeClaimResize;Priority;ResourceQuota;RuntimeClass;ServiceAccount;StorageObjectInUseProtection;TaintNodesByCondition;ValidatingAdmissionWebhook
 	AdmissionControllers AdmissionControllers `json:"admissionControllers,omitempty"`
 }
 
@@ -105,9 +127,9 @@ type ControlPlaneComponentsResources struct {
 type DeploymentSpec struct {
 	// RegistrySettings allows to override the default images for the given Tenant Control Plane instance.
 	// It could be used to point to a different container registry rather than the public one.
-	// +kubebuilder:default={registry:"registry.k8s.io",apiServerImage:"kube-apiserver",controllerManagerImage:"kube-controller-manager",schedulerImage:"kube-scheduler"}
+	//+kubebuilder:default={registry:"registry.k8s.io",apiServerImage:"kube-apiserver",controllerManagerImage:"kube-controller-manager",schedulerImage:"kube-scheduler"}
 	RegistrySettings RegistrySettings `json:"registrySettings,omitempty"`
-	// +kubebuilder:default=2
+	//+kubebuilder:default=2
 	Replicas *int32 `json:"replicas,omitempty"`
 	// NodeSelector is a selector which must be true for the pod to fit on a node.
 	// Selector which must match a node's labels for the pod to be scheduled on that node.
@@ -121,7 +143,7 @@ type DeploymentSpec struct {
 	RuntimeClassName string `json:"runtimeClassName,omitempty"`
 	// Strategy describes how to replace existing pods with new ones for the given Tenant Control Plane.
 	// Default value is set to Rolling Update, with a blue/green strategy.
-	// +kubebuilder:default={type:"RollingUpdate",rollingUpdate:{maxUnavailable:0,maxSurge:"100%"}}
+	//+kubebuilder:default={type:"RollingUpdate",rollingUpdate:{maxUnavailable:0,maxSurge:"100%"}}
 	Strategy appsv1.DeploymentStrategy `json:"strategy,omitempty"`
 	// If specified, the Tenant Control Plane pod's tolerations.
 	// More info: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/
@@ -153,7 +175,7 @@ type DeploymentSpec struct {
 	// AdditionalVolumeMounts allows to mount an additional volume into each component of the Control Plane
 	// (kube-apiserver, controller-manager, and scheduler).
 	AdditionalVolumeMounts *AdditionalVolumeMounts `json:"additionalVolumeMounts,omitempty"`
-	// +kubebuilder:default="default"
+	//+kubebuilder:default="default"
 	// ServiceAccountName allows to specify the service account to be mounted to the pods of the Control plane deployment
 	ServiceAccountName string `json:"serviceAccountName,omitempty"`
 }
@@ -204,35 +226,62 @@ type KonnectivityServerSpec struct {
 	// The port which Konnectivity server is listening to.
 	Port int32 `json:"port"`
 	// Container image version of the Konnectivity server.
-	// +kubebuilder:default=v0.28.6
+	// If left empty, Kamaji will automatically inflect the version from the deployed Tenant Control Plane.
+	//
+	// WARNING: for last cut-off releases, the container image could be not available.
 	Version string `json:"version,omitempty"`
 	// Container image used by the Konnectivity server.
-	// +kubebuilder:default=registry.k8s.io/kas-network-proxy/proxy-server
+	//+kubebuilder:default=registry.k8s.io/kas-network-proxy/proxy-server
 	Image string `json:"image,omitempty"`
 	// Resources define the amount of CPU and memory to allocate to the Konnectivity server.
 	Resources *corev1.ResourceRequirements `json:"resources,omitempty"`
 	ExtraArgs ExtraArgs                    `json:"extraArgs,omitempty"`
 }
 
+type KonnectivityAgentMode string
+
+var (
+	KonnectivityAgentModeDaemonSet  KonnectivityAgentMode = "DaemonSet"
+	KonnectivityAgentModeDeployment KonnectivityAgentMode = "Deployment"
+)
+
+//+kubebuilder:validation:XValidation:rule="!(self.mode == 'DaemonSet' && has(self.replicas) && self.replicas != 0) && !(self.mode == 'Deployment' && self.replicas == 0)",message="replicas must be 0 when mode is DaemonSet, and greater than 0 when mode is Deployment"
+
 type KonnectivityAgentSpec struct {
 	// AgentImage defines the container image for Konnectivity's agent.
-	// +kubebuilder:default=registry.k8s.io/kas-network-proxy/proxy-agent
+	//+kubebuilder:default=registry.k8s.io/kas-network-proxy/proxy-agent
 	Image string `json:"image,omitempty"`
 	// Version for Konnectivity agent.
-	// +kubebuilder:default=v0.28.6
+	// If left empty, Kamaji will automatically inflect the version from the deployed Tenant Control Plane.
+	//
+	// WARNING: for last cut-off releases, the container image could be not available.
 	Version string `json:"version,omitempty"`
 	// Tolerations for the deployed agent.
 	// Can be customized to start the konnectivity-agent even if the nodes are not ready or tainted.
-	// +kubebuilder:default={{key: "CriticalAddonsOnly", operator: "Exists"}}
+	//+kubebuilder:default={{key: "CriticalAddonsOnly", operator: "Exists"}}
 	Tolerations []corev1.Toleration `json:"tolerations,omitempty"`
 	ExtraArgs   ExtraArgs           `json:"extraArgs,omitempty"`
+	// HostNetwork enables the konnectivity agent to use the Host network namespace.
+	// By enabling this mode, the Agent doesn't need to wait for the CNI initialisation,
+	// enabling a sort of out-of-band access to nodes for troubleshooting scenarios,
+	// or when the agent needs direct access to the host network.
+	//+kubebuilder:default=false
+	HostNetwork bool `json:"hostNetwork,omitempty"`
+	// Mode allows specifying the Agent deployment mode: Deployment, or DaemonSet (default).
+	//+kubebuilder:default="DaemonSet"
+	//+kubebuilder:validation:Enum=DaemonSet;Deployment
+	Mode KonnectivityAgentMode `json:"mode,omitempty"`
+	// Replicas defines the number of replicas when Mode is Deployment.
+	// Must be 0 if Mode is DaemonSet.
+	//+kubebuilder:validation:Optional
+	Replicas int32 `json:"replicas,omitempty"`
 }
 
 // KonnectivitySpec defines the spec for Konnectivity.
 type KonnectivitySpec struct {
-	// +kubebuilder:default={version:"v0.28.6",image:"registry.k8s.io/kas-network-proxy/proxy-server",port:8132}
+	//+kubebuilder:default={image:"registry.k8s.io/kas-network-proxy/proxy-server",port:8132}
 	KonnectivityServerSpec KonnectivityServerSpec `json:"server,omitempty"`
-	// +kubebuilder:default={version:"v0.28.6",image:"registry.k8s.io/kas-network-proxy/proxy-agent"}
+	//+kubebuilder:default={image:"registry.k8s.io/kas-network-proxy/proxy-agent",mode:"DaemonSet"}
 	KonnectivityAgentSpec KonnectivityAgentSpec `json:"agent,omitempty"`
 }
 
@@ -249,12 +298,34 @@ type AddonsSpec struct {
 }
 
 // TenantControlPlaneSpec defines the desired state of TenantControlPlane.
+// +kubebuilder:validation:XValidation:rule="!has(oldSelf.dataStore) || has(self.dataStore)", message="unsetting the dataStore is not supported"
+// +kubebuilder:validation:XValidation:rule="!has(oldSelf.dataStoreSchema) || has(self.dataStoreSchema)", message="unsetting the dataStoreSchema is not supported"
+// +kubebuilder:validation:XValidation:rule="!has(oldSelf.dataStoreUsername) || has(self.dataStoreUsername)", message="unsetting the dataStoreUsername is not supported"
+// +kubebuilder:validation:XValidation:rule="!has(self.networkProfile.loadBalancerSourceRanges) || (size(self.networkProfile.loadBalancerSourceRanges) == 0 || self.controlPlane.service.serviceType == 'LoadBalancer')", message="LoadBalancer source ranges are supported only with LoadBalancer service type"
+// +kubebuilder:validation:XValidation:rule="!has(self.networkProfile.loadBalancerClass) || self.controlPlane.service.serviceType == 'LoadBalancer'", message="LoadBalancerClass is supported only with LoadBalancer service type"
+// +kubebuilder:validation:XValidation:rule="self.controlPlane.service.serviceType != 'LoadBalancer' || (oldSelf.controlPlane.service.serviceType != 'LoadBalancer' && self.controlPlane.service.serviceType == 'LoadBalancer') || has(self.networkProfile.loadBalancerClass) == has(oldSelf.networkProfile.loadBalancerClass)",message="LoadBalancerClass cannot be set or unset at runtime"
+
 type TenantControlPlaneSpec struct {
-	// DataStore allows to specify a DataStore that should be used to store the Kubernetes data for the given Tenant Control Plane.
-	// This parameter is optional and acts as an override over the default one which is used by the Kamaji Operator.
-	// Migration from a different DataStore to another one is not yet supported and the reconciliation will be blocked.
-	DataStore    string       `json:"dataStore,omitempty"`
-	ControlPlane ControlPlane `json:"controlPlane"`
+	// DataStore specifies the DataStore that should be used to store the Kubernetes data for the given Tenant Control Plane.
+	// When Kamaji runs with the default DataStore flag, all empty values will inherit the default value.
+	// By leaving it empty and running Kamaji with no default DataStore flag, it is possible to achieve automatic assignment to a specific DataStore object.
+	//
+	// Migration from one DataStore to another backed by the same Driver is possible. See: https://kamaji.clastix.io/guides/datastore-migration/
+	// Migration from one DataStore to another backed by a different Driver is not supported.
+	DataStore string `json:"dataStore,omitempty"`
+	// DataStoreSchema allows to specify the name of the database (for relational DataStores) or the key prefix (for etcd). This
+	// value is optional and immutable. Note that Kamaji currently doesn't ensure that DataStoreSchema values are unique. It's up
+	// to the user to avoid clashes between different TenantControlPlanes. If not set upon creation, Kamaji will default the
+	// DataStoreSchema by concatenating the namespace and name of the TenantControlPlane.
+	// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="changing the dataStoreSchema is not supported"
+	DataStoreSchema string `json:"dataStoreSchema,omitempty"`
+	// DataStoreUsername allows to specify the username of the database (for relational DataStores). This
+	// value is optional and immutable. Note that Kamaji currently doesn't ensure that DataStoreUsername values are unique. It's up
+	// to the user to avoid clashes between different TenantControlPlanes. If not set upon creation, Kamaji will default the
+	// DataStoreUsername by concatenating the namespace and name of the TenantControlPlane.
+	// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="changing the dataStoreUsername is not supported"
+	DataStoreUsername string       `json:"dataStoreUsername,omitempty"`
+	ControlPlane      ControlPlane `json:"controlPlane"`
 	// Kubernetes specification for tenant control plane
 	Kubernetes KubernetesSpec `json:"kubernetes"`
 	// NetworkProfile specifies how the network is
@@ -263,16 +334,18 @@ type TenantControlPlaneSpec struct {
 	Addons AddonsSpec `json:"addons,omitempty"`
 }
 
-// +kubebuilder:object:root=true
-// +kubebuilder:subresource:status
-// +kubebuilder:subresource:scale:specpath=.spec.controlPlane.deployment.replicas,statuspath=.status.kubernetesResources.deployment.replicas,selectorpath=.status.kubernetesResources.deployment.selector
-// +kubebuilder:resource:categories=kamaji,shortName=tcp
-// +kubebuilder:printcolumn:name="Version",type="string",JSONPath=".spec.kubernetes.version",description="Kubernetes version"
-// +kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.kubernetesResources.version.status",description="Status"
-// +kubebuilder:printcolumn:name="Control-Plane endpoint",type="string",JSONPath=".status.controlPlaneEndpoint",description="Tenant Control Plane Endpoint (API server)"
-// +kubebuilder:printcolumn:name="Kubeconfig",type="string",JSONPath=".status.kubeconfig.admin.secretName",description="Secret which contains admin kubeconfig"
+//+kubebuilder:object:root=true
+//+kubebuilder:subresource:status
+//+kubebuilder:subresource:scale:specpath=.spec.controlPlane.deployment.replicas,statuspath=.status.kubernetesResources.deployment.replicas,selectorpath=.status.kubernetesResources.deployment.selector
+//+kubebuilder:resource:categories=kamaji,shortName=tcp
+//+kubebuilder:printcolumn:name="Version",type="string",JSONPath=".spec.kubernetes.version",description="Kubernetes version"
+//+kubebuilder:printcolumn:name="Installed Version",type="string",JSONPath=".status.kubernetesResources.version.version",description="The actual installed Kubernetes version from status"
+//+kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.kubernetesResources.version.status",description="Status"
+//+kubebuilder:printcolumn:name="Control-Plane endpoint",type="string",JSONPath=".status.controlPlaneEndpoint",description="Tenant Control Plane Endpoint (API server)"
+//+kubebuilder:printcolumn:name="Kubeconfig",type="string",JSONPath=".status.kubeconfig.admin.secretName",description="Secret which contains admin kubeconfig"
 //+kubebuilder:printcolumn:name="Datastore",type="string",JSONPath=".status.storage.dataStoreName",description="DataStore actually used"
-// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="Age"
+//+kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="Age"
+//+kubebuilder:metadata:annotations={"cert-manager.io/inject-ca-from=kamaji-system/kamaji-serving-cert"}
 
 // TenantControlPlane is the Schema for the tenantcontrolplanes API.
 type TenantControlPlane struct {

api/v1alpha1/tenantcontrolplane_types_test.go

@@ -0,0 +1,78 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package v1alpha1
+
+import (
+	"context"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+var _ = Describe("Cluster controller", func() {
+	var (
+		ctx context.Context
+		tcp *TenantControlPlane
+	)
+
+	BeforeEach(func() {
+		ctx = context.Background()
+		tcp = &TenantControlPlane{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "tcp",
+				Namespace: "default",
+			},
+			Spec: TenantControlPlaneSpec{},
+		}
+	})
+
+	AfterEach(func() {
+		if err := k8sClient.Delete(ctx, tcp); err != nil && !apierrors.IsNotFound(err) {
+			Expect(err).NotTo(HaveOccurred())
+		}
+	})
+
+	Context("LoadBalancer Source Ranges", func() {
+		It("allows creation when no CIDR ranges are provided", func() {
+			tcp.Spec.ControlPlane.Service.ServiceType = ServiceTypeLoadBalancer
+
+			err := k8sClient.Create(ctx, tcp)
+			Expect(err).NotTo(HaveOccurred())
+		})
+
+		It("allows creation with an explicitly empty CIDR list", func() {
+			tcp.Spec.ControlPlane.Service.ServiceType = ServiceTypeLoadBalancer
+			tcp.Spec.NetworkProfile.LoadBalancerSourceRanges = []string{}
+
+			err := k8sClient.Create(ctx, tcp)
+			Expect(err).NotTo(HaveOccurred())
+		})
+
+		It("allows creation when service type is not LoadBalancer and it has an empty CIDR list", func() {
+			tcp.Spec.ControlPlane.Service.ServiceType = ServiceTypeNodePort
+
+			err := k8sClient.Create(ctx, tcp)
+			Expect(err).NotTo(HaveOccurred())
+		})
+
+		It("allows CIDR ranges when service type is LoadBalancer", func() {
+			tcp.Spec.ControlPlane.Service.ServiceType = ServiceTypeLoadBalancer
+			tcp.Spec.NetworkProfile.LoadBalancerSourceRanges = []string{"192.168.0.0/24"}
+
+			err := k8sClient.Create(ctx, tcp)
+			Expect(err).NotTo(HaveOccurred())
+		})
+
+		It("denies CIDR ranges when service type is not LoadBalancer", func() {
+			tcp.Spec.ControlPlane.Service.ServiceType = ServiceTypeNodePort
+			tcp.Spec.NetworkProfile.LoadBalancerSourceRanges = []string{"192.168.0.0/24"}
+
+			err := k8sClient.Create(ctx, tcp)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("LoadBalancer source ranges are supported only with LoadBalancer service type"))
+		})
+	})
+})

api/v1alpha1/validations_test.go

@@ -0,0 +1,177 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package v1alpha1
+
+import (
+	"context"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+var _ = Describe("Datastores validation test", func() {
+	var (
+		ctx context.Context
+		ds  *DataStore
+	)
+
+	BeforeEach(func() {
+		ctx = context.Background()
+		ds = &DataStore{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "ds",
+				Namespace: "default",
+			},
+			Spec: DataStoreSpec{},
+		}
+	})
+
+	AfterEach(func() {
+		if err := k8sClient.Delete(ctx, ds); err != nil && !apierrors.IsNotFound(err) {
+			Expect(err).NotTo(HaveOccurred())
+		}
+	})
+
+	Context("DataStores fields", func() {
+		It("datastores of type ETCD must have their TLS configurations set correctly", func() {
+			ds = &DataStore{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "bad-etcd",
+				},
+				Spec: DataStoreSpec{
+					Driver:    "etcd",
+					Endpoints: []string{"etcd-server:2379"},
+					TLSConfig: &TLSConfig{
+						CertificateAuthority: CertKeyPair{},
+						ClientCertificate:    &ClientCertificate{},
+					},
+				},
+			}
+
+			err := k8sClient.Create(ctx, ds)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("certificateAuthority privateKey must have secretReference or content when driver is etcd"))
+		})
+
+		It("valid ETCD DataStore should be created", func() {
+			var (
+				cert = []byte("cert")
+				key  = []byte("privkey")
+			)
+
+			ds = &DataStore{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "good-etcd",
+				},
+				Spec: DataStoreSpec{
+					Driver:    "etcd",
+					Endpoints: []string{"etcd-server:2379"},
+					TLSConfig: &TLSConfig{
+						CertificateAuthority: CertKeyPair{
+							Certificate: ContentRef{
+								Content: cert,
+							},
+							PrivateKey: &ContentRef{
+								Content: key,
+							},
+						},
+						ClientCertificate: &ClientCertificate{
+							Certificate: ContentRef{
+								Content: cert,
+							},
+							PrivateKey: ContentRef{
+								Content: key,
+							},
+						},
+					},
+				},
+			}
+
+			err := k8sClient.Create(ctx, ds)
+			Expect(err).To(Not(HaveOccurred()))
+		})
+
+		It("datastores of type PostgreSQL must have either basicAuth or tlsConfig", func() {
+			ds = &DataStore{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "bad-pg",
+				},
+				Spec: DataStoreSpec{
+					Driver:    "PostgreSQL",
+					Endpoints: []string{"pg-server:5432"},
+				},
+			}
+
+			err := k8sClient.Create(ctx, ds)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("When driver is not etcd, either tlsConfig or basicAuth must be provided"))
+		})
+
+		It("datastores of type PostgreSQL can have basicAuth", func() {
+			ds = &DataStore{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "good-pg",
+				},
+				Spec: DataStoreSpec{
+					Driver:    "PostgreSQL",
+					Endpoints: []string{"pg-server:5432"},
+					BasicAuth: &BasicAuth{
+						Username: ContentRef{
+							Content: []byte("postgres"),
+						},
+						Password: ContentRef{
+							Content: []byte("postgres"),
+						},
+					},
+				},
+			}
+
+			err := k8sClient.Create(ctx, ds)
+			Expect(err).To(Not(HaveOccurred()))
+		})
+
+		It("datastores of type PostgreSQL must have tlsConfig with proper content", func() {
+			ds = &DataStore{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "bad-pg",
+				},
+				Spec: DataStoreSpec{
+					Driver:    "PostgreSQL",
+					Endpoints: []string{"pg-server:5432"},
+					TLSConfig: &TLSConfig{
+						ClientCertificate: &ClientCertificate{},
+					},
+				},
+			}
+
+			err := k8sClient.Create(context.Background(), ds)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("When driver is not etcd and tlsConfig exists, clientCertificate must be null or contain valid content"))
+		})
+
+		It("datastores of type PostgreSQL need a proper clientCertificate", func() {
+			ds = &DataStore{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "good-pg",
+				},
+				Spec: DataStoreSpec{
+					Driver:    "PostgreSQL",
+					Endpoints: []string{"pg-server:5432"},
+					TLSConfig: &TLSConfig{
+						ClientCertificate: &ClientCertificate{
+							Certificate: ContentRef{
+								Content: []byte("cert"),
+							},
+						},
+					},
+				},
+			}
+
+			err := k8sClient.Create(context.Background(), ds)
+			Expect(err).ToNot(HaveOccurred())
+		})
+	})
+})

api/v1alpha1/zz_generated.deepcopy.go

@@ -289,6 +289,21 @@ func (in *ClientCertificate) DeepCopy() *ClientCertificate {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *CompoundValue) DeepCopyInto(out *CompoundValue) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CompoundValue.
+func (in *CompoundValue) DeepCopy() *CompoundValue {
+	if in == nil {
+		return nil
+	}
+	out := new(CompoundValue)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ContentRef) DeepCopyInto(out *ContentRef) {
 	*out = *in
@@ -808,6 +823,22 @@ func (in *KonnectivityAgentSpec) DeepCopy() *KonnectivityAgentSpec {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *KonnectivityAgentStatus) DeepCopyInto(out *KonnectivityAgentStatus) {
+	*out = *in
+	in.ExternalKubernetesObjectStatus.DeepCopyInto(&out.ExternalKubernetesObjectStatus)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KonnectivityAgentStatus.
+func (in *KonnectivityAgentStatus) DeepCopy() *KonnectivityAgentStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(KonnectivityAgentStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *KonnectivityConfigMap) DeepCopyInto(out *KonnectivityConfigMap) {
 	*out = *in
@@ -935,6 +966,123 @@ func (in *KubeadmPhasesStatus) DeepCopy() *KubeadmPhasesStatus {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *KubeconfigGenerator) DeepCopyInto(out *KubeconfigGenerator) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
+	in.Spec.DeepCopyInto(&out.Spec)
+	in.Status.DeepCopyInto(&out.Status)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeconfigGenerator.
+func (in *KubeconfigGenerator) DeepCopy() *KubeconfigGenerator {
+	if in == nil {
+		return nil
+	}
+	out := new(KubeconfigGenerator)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *KubeconfigGenerator) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *KubeconfigGeneratorList) DeepCopyInto(out *KubeconfigGeneratorList) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ListMeta.DeepCopyInto(&out.ListMeta)
+	if in.Items != nil {
+		in, out := &in.Items, &out.Items
+		*out = make([]KubeconfigGenerator, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeconfigGeneratorList.
+func (in *KubeconfigGeneratorList) DeepCopy() *KubeconfigGeneratorList {
+	if in == nil {
+		return nil
+	}
+	out := new(KubeconfigGeneratorList)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *KubeconfigGeneratorList) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *KubeconfigGeneratorSpec) DeepCopyInto(out *KubeconfigGeneratorSpec) {
+	*out = *in
+	in.NamespaceSelector.DeepCopyInto(&out.NamespaceSelector)
+	in.TenantControlPlaneSelector.DeepCopyInto(&out.TenantControlPlaneSelector)
+	if in.Groups != nil {
+		in, out := &in.Groups, &out.Groups
+		*out = make([]CompoundValue, len(*in))
+		copy(*out, *in)
+	}
+	out.User = in.User
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeconfigGeneratorSpec.
+func (in *KubeconfigGeneratorSpec) DeepCopy() *KubeconfigGeneratorSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(KubeconfigGeneratorSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *KubeconfigGeneratorStatus) DeepCopyInto(out *KubeconfigGeneratorStatus) {
+	*out = *in
+	if in.Errors != nil {
+		in, out := &in.Errors, &out.Errors
+		*out = make([]KubeconfigGeneratorStatusError, len(*in))
+		copy(*out, *in)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeconfigGeneratorStatus.
+func (in *KubeconfigGeneratorStatus) DeepCopy() *KubeconfigGeneratorStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(KubeconfigGeneratorStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *KubeconfigGeneratorStatusError) DeepCopyInto(out *KubeconfigGeneratorStatusError) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeconfigGeneratorStatusError.
+func (in *KubeconfigGeneratorStatusError) DeepCopy() *KubeconfigGeneratorStatusError {
+	if in == nil {
+		return nil
+	}
+	out := new(KubeconfigGeneratorStatusError)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *KubeconfigStatus) DeepCopyInto(out *KubeconfigStatus) {
 	*out = *in
@@ -1105,6 +1253,16 @@ func (in *KubernetesVersion) DeepCopy() *KubernetesVersion {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *NetworkProfileSpec) DeepCopyInto(out *NetworkProfileSpec) {
 	*out = *in
+	if in.LoadBalancerSourceRanges != nil {
+		in, out := &in.LoadBalancerSourceRanges, &out.LoadBalancerSourceRanges
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
+	if in.LoadBalancerClass != nil {
+		in, out := &in.LoadBalancerClass, &out.LoadBalancerClass
+		*out = new(string)
+		**out = **in
+	}
 	if in.CertSANs != nil {
 		in, out := &in.CertSANs, &out.CertSANs
 		*out = make([]string, len(*in))

charts/kamaji-crds/.helmignore

@@ -0,0 +1,28 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/
+# Helm source files
+README.md.gotmpl
+.helmignore
+# Build tools
+Makefile

charts/kamaji-crds/Chart.yaml

@@ -0,0 +1,37 @@
+apiVersion: v2
+appVersion: latest
+description: Kamaji is the Hosted Control Plane Manager for Kubernetes.
+home: https://github.com/clastix/kamaji
+icon: https://github.com/clastix/kamaji/raw/master/assets/logo-colored.png
+maintainers:
+  - email: dario@tranchitella.eu
+    name: Dario Tranchitella
+    url: https://clastix.io
+  - email: me@bsctl.io
+    name: Adriano Pezzuto
+    url: https://clastix.io
+name: kamaji-crds
+sources:
+  - https://github.com/clastix/kamaji
+type: application
+version: 0.0.0+edge
+annotations:
+  artifacthub.io/crds: |
+    - kind: TenantControlPlane
+      version: v1alpha1
+      name: tenantcontrolplanes.kamaji.clastix.io
+      displayName: TenantControlPlane
+      description: TenantControlPlane defines the desired state for a Control Plane backed by Kamaji.
+    - kind: DataStore
+      version: v1alpha1
+      name: datastores.kamaji.clastix.io
+      displayName: DataStore
+      description: DataStores is holding all the required details to communicate with a Datastore, such as etcd, MySQL, PostgreSQL, and NATS.
+  artifacthub.io/links: |
+    - name: CLASTIX
+      url: https://clastix.io
+    - name: support
+      url: https://clastix.io/support
+  artifacthub.io/changes: |
+    - kind: added
+      description: First commit

charts/kamaji-crds/Makefile

@@ -0,0 +1,9 @@
+docs: HELMDOCS_VERSION := v1.8.1
+docs: docker
+	@docker run --rm -v "$$(pwd):/helm-docs" -u $$(id -u) jnorwood/helm-docs:$(HELMDOCS_VERSION)
+
+docker:
+	@hash docker 2>/dev/null || {\
+		echo "You need docker" &&\
+		exit 1;\
+	}

charts/kamaji-crds/NOTES.txt

@@ -0,0 +1,2 @@
+Kamaji Custom Resource Definitions have been installed properly:
+you can proceed to upgrade your Kamaji operator instance.

charts/kamaji-crds/README.md

@@ -0,0 +1,66 @@
+# kamaji-crds
+
+![Version: 0.0.0+edge](https://img.shields.io/badge/Version-0.0.0+edge-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: latest](https://img.shields.io/badge/AppVersion-latest-informational?style=flat-square)
+
+Kamaji is the Hosted Control Plane Manager for Kubernetes.
+
+## Maintainers
+
+| Name | Email | Url |
+| ---- | ------ | --- |
+| Dario Tranchitella | <dario@tranchitella.eu> | <https://clastix.io> |
+| Adriano Pezzuto | <me@bsctl.io> | <https://clastix.io> |
+
+## Source Code
+
+* <https://github.com/clastix/kamaji>
+
+[Kamaji](https://github.com/clastix/kamaji) Custom Resource Definitions packaged as Helm Charts.
+
+## How to use this chart
+
+Add `clastix` Helm repository:
+
+    helm repo add clastix https://clastix.github.io/charts
+
+Install the Chart with the release name `kamaji-crds`:
+
+    helm upgrade --install --namespace kamaji-system --create-namespace kamaji-crds clastix/kamaji-crds
+
+Show the status:
+
+    helm status kamaji-crds -n kamaji-system
+
+Upgrade the Chart
+
+    helm upgrade kamaji-crds -n kamaji-system clastix/kamaji-crds
+
+Uninstall the Chart
+
+    helm uninstall kamaji-crds -n kamaji-system
+
+## Customize the installation
+
+There are two methods for specifying overrides of values during Chart installation: `--values` and `--set`.
+
+The `--values` option is the preferred method because it allows you to keep your overrides in a YAML file, rather than specifying them all on the command line. Create a copy of the YAML file `values.yaml` and add your overrides to it.
+
+Specify your overrides file when you install the Chart:
+
+    helm upgrade kamaji-crds --install --namespace kamaji-system --create-namespace clastix/kamaji-crds --values myvalues.yaml
+
+The values in your overrides file `myvalues.yaml` will override their counterparts in the Chart's values.yaml file. Any values in `values.yaml` that weren’t overridden will keep their defaults.
+
+If you only need to make minor customizations, you can specify them on the command line by using the `--set` option. For example:
+
+    helm upgrade kamaji-crds --install --namespace kamaji-system --create-namespace clastix/kamaji-crds --set kamajiCertificateName=kamaji
+
+## Values
+
+| Key | Type | Default | Description |
+|-----|------|---------|-------------|
+| fullnameOverride | string | `""` | Overrides the full name of the resources created by the chart. |
+| kamajiCertificateName | string | `"kamaji-serving-cert"` | The cert-manager Certificate resource name, holding the Certificate Authority for webhooks. |
+| kamajiNamespace | string | `"kamaji-system"` | The namespace where Kamaji has been installed: required to inject the Certificate Authority for cert-manager. |
+| kamajiService | string | `"kamaji-webhook-service"` | The Kamaji webhook Service name. |
+| nameOverride | string | `""` | Overrides the name of the chart for resource naming purposes. |

charts/kamaji-crds/README.md.gotmpl

@@ -0,0 +1,54 @@
+{{ template "chart.header" . }}
+{{ template "chart.deprecationWarning" . }}
+
+{{ template "chart.badgesSection" . }}
+
+{{ template "chart.description" . }}
+
+{{ template "chart.maintainersSection" . }}
+
+{{ template "chart.sourcesSection" . }}
+
+{{ template "chart.requirementsSection" . }}
+
+[Kamaji](https://github.com/clastix/kamaji) Custom Resource Definitions packaged as Helm Charts.
+
+## How to use this chart
+
+Add `clastix` Helm repository:
+
+    helm repo add clastix https://clastix.github.io/charts
+
+Install the Chart with the release name `kamaji-crds`:
+
+    helm upgrade --install --namespace kamaji-system --create-namespace kamaji-crds clastix/kamaji-crds
+
+Show the status:
+
+    helm status kamaji-crds -n kamaji-system
+
+Upgrade the Chart
+
+    helm upgrade kamaji-crds -n kamaji-system clastix/kamaji-crds
+
+Uninstall the Chart
+
+    helm uninstall kamaji-crds -n kamaji-system
+
+## Customize the installation
+
+There are two methods for specifying overrides of values during Chart installation: `--values` and `--set`.
+
+The `--values` option is the preferred method because it allows you to keep your overrides in a YAML file, rather than specifying them all on the command line. Create a copy of the YAML file `values.yaml` and add your overrides to it.
+
+Specify your overrides file when you install the Chart:
+
+    helm upgrade kamaji-crds --install --namespace kamaji-system --create-namespace clastix/kamaji-crds --values myvalues.yaml
+
+The values in your overrides file `myvalues.yaml` will override their counterparts in the Chart's values.yaml file. Any values in `values.yaml` that weren’t overridden will keep their defaults.
+
+If you only need to make minor customizations, you can specify them on the command line by using the `--set` option. For example:
+
+    helm upgrade kamaji-crds --install --namespace kamaji-system --create-namespace clastix/kamaji-crds --set kamajiCertificateName=kamaji
+
+{{ template "chart.valuesSection" . }}

charts/kamaji-crds/hack/crd-conversion.yaml

@@ -0,0 +1,11 @@
+spec:
+  conversion:
+    strategy: Webhook
+    webhook:
+      clientConfig:
+        service:
+          name: kamaji-webhook-service
+          namespace: kamaji-system
+          path: /convert
+      conversionReviewVersions:
+        - v1

charts/kamaji-crds/hack/kamaji.clastix.io_datastores_spec.yaml

@@ -1,28 +1,20 @@
----
-apiVersion: apiextensions.k8s.io/v1
-kind: CustomResourceDefinition
-metadata:
-  annotations:
-    controller-gen.kubebuilder.io/version: v0.14.0
-  name: datastores.kamaji.clastix.io
-spec:
-  group: kamaji.clastix.io
-  names:
-    kind: DataStore
-    listKind: DataStoreList
-    plural: datastores
-    singular: datastore
-  scope: Cluster
-  versions:
+group: kamaji.clastix.io
+names:
+  kind: DataStore
+  listKind: DataStoreList
+  plural: datastores
+  singular: datastore
+scope: Cluster
+versions:
   - additionalPrinterColumns:
-    - description: Kamaji data store driver
-      jsonPath: .spec.driver
-      name: Driver
-      type: string
-    - description: Age
-      jsonPath: .metadata.creationTimestamp
-      name: Age
-      type: date
+      - description: Kamaji data store driver
+        jsonPath: .spec.driver
+        name: Driver
+        type: string
+      - description: Age
+        jsonPath: .metadata.creationTimestamp
+        name: Age
+        type: date
     name: v1alpha1
     schema:
       openAPIV3Schema:
@@ -70,15 +62,13 @@ spec:
                             minLength: 1
                             type: string
                           name:
-                            description: name is unique within a namespace to reference
-                              a secret resource.
+                            description: name is unique within a namespace to reference a secret resource.
                             type: string
                           namespace:
-                            description: namespace defines the space within which
-                              the secret name must be unique.
+                            description: namespace defines the space within which the secret name must be unique.
                             type: string
                         required:
-                        - keyPath
+                          - keyPath
                         type: object
                         x-kubernetes-map-type: atomic
                     type: object
@@ -99,30 +89,31 @@ spec:
                             minLength: 1
                             type: string
                           name:
-                            description: name is unique within a namespace to reference
-                              a secret resource.
+                            description: name is unique within a namespace to reference a secret resource.
                             type: string
                           namespace:
-                            description: namespace defines the space within which
-                              the secret name must be unique.
+                            description: namespace defines the space within which the secret name must be unique.
                             type: string
                         required:
-                        - keyPath
+                          - keyPath
                         type: object
                         x-kubernetes-map-type: atomic
                     type: object
                 required:
-                - password
-                - username
+                  - password
+                  - username
                 type: object
               driver:
                 description: The driver to use to connect to the shared datastore.
                 enum:
-                - etcd
-                - MySQL
-                - PostgreSQL
-                - NATS
+                  - etcd
+                  - MySQL
+                  - PostgreSQL
+                  - NATS
                 type: string
+                x-kubernetes-validations:
+                  - message: Datastore driver is immutable
+                    rule: self == oldSelf
               endpoints:
                 description: |-
                   List of the endpoints to connect to the shared datastore.
@@ -158,15 +149,13 @@ spec:
                                 minLength: 1
                                 type: string
                               name:
-                                description: name is unique within a namespace to
-                                  reference a secret resource.
+                                description: name is unique within a namespace to reference a secret resource.
                                 type: string
                               namespace:
-                                description: namespace defines the space within which
-                                  the secret name must be unique.
+                                description: namespace defines the space within which the secret name must be unique.
                                 type: string
                             required:
-                            - keyPath
+                              - keyPath
                             type: object
                             x-kubernetes-map-type: atomic
                         type: object
@@ -187,24 +176,21 @@ spec:
                                 minLength: 1
                                 type: string
                               name:
-                                description: name is unique within a namespace to
-                                  reference a secret resource.
+                                description: name is unique within a namespace to reference a secret resource.
                                 type: string
                               namespace:
-                                description: namespace defines the space within which
-                                  the secret name must be unique.
+                                description: namespace defines the space within which the secret name must be unique.
                                 type: string
                             required:
-                            - keyPath
+                              - keyPath
                             type: object
                             x-kubernetes-map-type: atomic
                         type: object
                     required:
-                    - certificate
+                      - certificate
                     type: object
                   clientCertificate:
-                    description: Specifies the SSL/TLS key and private key pair used
-                      to connect to the data store.
+                    description: Specifies the SSL/TLS key and private key pair used to connect to the data store.
                     properties:
                       certificate:
                         properties:
@@ -223,15 +209,13 @@ spec:
                                 minLength: 1
                                 type: string
                               name:
-                                description: name is unique within a namespace to
-                                  reference a secret resource.
+                                description: name is unique within a namespace to reference a secret resource.
                                 type: string
                               namespace:
-                                description: namespace defines the space within which
-                                  the secret name must be unique.
+                                description: namespace defines the space within which the secret name must be unique.
                                 type: string
                             required:
-                            - keyPath
+                              - keyPath
                             type: object
                             x-kubernetes-map-type: atomic
                         type: object
@@ -252,35 +236,47 @@ spec:
                                 minLength: 1
                                 type: string
                               name:
-                                description: name is unique within a namespace to
-                                  reference a secret resource.
+                                description: name is unique within a namespace to reference a secret resource.
                                 type: string
                               namespace:
-                                description: namespace defines the space within which
-                                  the secret name must be unique.
+                                description: namespace defines the space within which the secret name must be unique.
                                 type: string
                             required:
-                            - keyPath
+                              - keyPath
                             type: object
                             x-kubernetes-map-type: atomic
                         type: object
                     required:
-                    - certificate
-                    - privateKey
+                      - certificate
+                      - privateKey
                     type: object
                 required:
-                - certificateAuthority
+                  - certificateAuthority
                 type: object
             required:
-            - driver
-            - endpoints
+              - driver
+              - endpoints
             type: object
+            x-kubernetes-validations:
+              - message: certificateAuthority privateKey must have secretReference or content when driver is etcd
+                rule: '(self.driver == "etcd") ? (self.tlsConfig != null && (has(self.tlsConfig.certificateAuthority.privateKey.secretReference) || has(self.tlsConfig.certificateAuthority.privateKey.content))) : true'
+              - message: clientCertificate must have secretReference or content when driver is etcd
+                rule: '(self.driver == "etcd") ? (self.tlsConfig != null && (has(self.tlsConfig.clientCertificate.certificate.secretReference) || has(self.tlsConfig.clientCertificate.certificate.content))) : true'
+              - message: clientCertificate privateKey must have secretReference or content when driver is etcd
+                rule: '(self.driver == "etcd") ? (self.tlsConfig != null && (has(self.tlsConfig.clientCertificate.privateKey.secretReference) || has(self.tlsConfig.clientCertificate.privateKey.content))) : true'
+              - message: When driver is not etcd and tlsConfig exists, clientCertificate must be null or contain valid content
+                rule: '(self.driver != "etcd" && has(self.tlsConfig) && has(self.tlsConfig.clientCertificate)) ? (((has(self.tlsConfig.clientCertificate.certificate.secretReference) || has(self.tlsConfig.clientCertificate.certificate.content)))) : true'
+              - message: When driver is not etcd and basicAuth exists, username must have secretReference or content
+                rule: '(self.driver != "etcd" && has(self.basicAuth)) ? ((has(self.basicAuth.username.secretReference) || has(self.basicAuth.username.content))) : true'
+              - message: When driver is not etcd and basicAuth exists, password must have secretReference or content
+                rule: '(self.driver != "etcd" && has(self.basicAuth)) ? ((has(self.basicAuth.password.secretReference) || has(self.basicAuth.password.content))) : true'
+              - message: When driver is not etcd, either tlsConfig or basicAuth must be provided
+                rule: '(self.driver != "etcd") ? (has(self.tlsConfig) || has(self.basicAuth)) : true'
           status:
             description: DataStoreStatus defines the observed state of DataStore.
             properties:
               usedBy:
-                description: List of the Tenant Control Planes, namespaced named,
-                  using this data store.
+                description: List of the Tenant Control Planes, namespaced named, using this data store.
                 items:
                   type: string
                 type: array

charts/kamaji-crds/hack/kamaji.clastix.io_kubeconfiggenerators_spec.yaml

@@ -0,0 +1,214 @@
+group: kamaji.clastix.io
+names:
+  categories:
+    - kamaji
+  kind: KubeconfigGenerator
+  listKind: KubeconfigGeneratorList
+  plural: kubeconfiggenerators
+  shortNames:
+    - kc
+  singular: kubeconfiggenerator
+scope: Cluster
+versions:
+  - additionalPrinterColumns:
+      - description: Age
+        jsonPath: .metadata.creationTimestamp
+        name: Age
+        type: date
+    name: v1alpha1
+    schema:
+      openAPIV3Schema:
+        description: KubeconfigGenerator is the Schema for the kubeconfiggenerators API.
+        properties:
+          apiVersion:
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            type: string
+          kind:
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            type: string
+          metadata:
+            type: object
+          spec:
+            properties:
+              controlPlaneEndpointFrom:
+                default: admin.svc
+                description: |-
+                  ControlPlaneEndpointFrom is the key used to extract the Tenant Control Plane endpoint that must be used by the generator.
+                  The targeted Secret is the `${TCP}-admin-kubeconfig` one, default to `admin.svc`.
+                type: string
+              groups:
+                description: |-
+                  Groups is resolved a set of strings used to assign the x509 organisations field.
+                  It will be recognised by Kubernetes as user groups.
+                items:
+                  description: |-
+                    CompoundValue allows defining a static, or a dynamic value.
+                    Options are mutually exclusive, just one should be picked up.
+                  properties:
+                    fromDefinition:
+                      description: |-
+                        FromDefinition is used to generate a dynamic value,
+                        it uses the dot notation to access fields from the referenced TenantControlPlane object:
+                        e.g.: metadata.name
+                      type: string
+                    stringValue:
+                      description: StringValue is a static string value.
+                      type: string
+                  type: object
+                  x-kubernetes-validations:
+                    - message: Either stringValue or fromDefinition must be set, but not both.
+                      rule: (has(self.stringValue) || has(self.fromDefinition)) && !(has(self.stringValue) && has(self.fromDefinition))
+                type: array
+              namespaceSelector:
+                description: NamespaceSelector is used to filter Namespaces from which the generator should extract TenantControlPlane objects.
+                properties:
+                  matchExpressions:
+                    description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
+                    items:
+                      description: |-
+                        A label selector requirement is a selector that contains values, a key, and an operator that
+                        relates the key and values.
+                      properties:
+                        key:
+                          description: key is the label key that the selector applies to.
+                          type: string
+                        operator:
+                          description: |-
+                            operator represents a key's relationship to a set of values.
+                            Valid operators are In, NotIn, Exists and DoesNotExist.
+                          type: string
+                        values:
+                          description: |-
+                            values is an array of string values. If the operator is In or NotIn,
+                            the values array must be non-empty. If the operator is Exists or DoesNotExist,
+                            the values array must be empty. This array is replaced during a strategic
+                            merge patch.
+                          items:
+                            type: string
+                          type: array
+                          x-kubernetes-list-type: atomic
+                      required:
+                        - key
+                        - operator
+                      type: object
+                    type: array
+                    x-kubernetes-list-type: atomic
+                  matchLabels:
+                    additionalProperties:
+                      type: string
+                    description: |-
+                      matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels
+                      map is equivalent to an element of matchExpressions, whose key field is "key", the
+                      operator is "In", and the values array contains only "value". The requirements are ANDed.
+                    type: object
+                type: object
+                x-kubernetes-map-type: atomic
+              tenantControlPlaneSelector:
+                description: TenantControlPlaneSelector is used to filter the TenantControlPlane objects that should be address by the generator.
+                properties:
+                  matchExpressions:
+                    description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
+                    items:
+                      description: |-
+                        A label selector requirement is a selector that contains values, a key, and an operator that
+                        relates the key and values.
+                      properties:
+                        key:
+                          description: key is the label key that the selector applies to.
+                          type: string
+                        operator:
+                          description: |-
+                            operator represents a key's relationship to a set of values.
+                            Valid operators are In, NotIn, Exists and DoesNotExist.
+                          type: string
+                        values:
+                          description: |-
+                            values is an array of string values. If the operator is In or NotIn,
+                            the values array must be non-empty. If the operator is Exists or DoesNotExist,
+                            the values array must be empty. This array is replaced during a strategic
+                            merge patch.
+                          items:
+                            type: string
+                          type: array
+                          x-kubernetes-list-type: atomic
+                      required:
+                        - key
+                        - operator
+                      type: object
+                    type: array
+                    x-kubernetes-list-type: atomic
+                  matchLabels:
+                    additionalProperties:
+                      type: string
+                    description: |-
+                      matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels
+                      map is equivalent to an element of matchExpressions, whose key field is "key", the
+                      operator is "In", and the values array contains only "value". The requirements are ANDed.
+                    type: object
+                type: object
+                x-kubernetes-map-type: atomic
+              user:
+                description: User resolves to a string to identify the client, assigned to the x509 Common Name field.
+                properties:
+                  fromDefinition:
+                    description: |-
+                      FromDefinition is used to generate a dynamic value,
+                      it uses the dot notation to access fields from the referenced TenantControlPlane object:
+                      e.g.: metadata.name
+                    type: string
+                  stringValue:
+                    description: StringValue is a static string value.
+                    type: string
+                type: object
+                x-kubernetes-validations:
+                  - message: Either stringValue or fromDefinition must be set, but not both.
+                    rule: (has(self.stringValue) || has(self.fromDefinition)) && !(has(self.stringValue) && has(self.fromDefinition))
+            required:
+              - user
+            type: object
+          status:
+            description: KubeconfigGeneratorStatus defines the observed state of KubeconfigGenerator.
+            properties:
+              availableResources:
+                default: 0
+                description: |-
+                  AvailableResources is the sum of successfully generated resources.
+                  In case of a different value compared to Resources, check the field errors.
+                type: integer
+              errors:
+                description: Errors is the list of failed kubeconfig generations.
+                items:
+                  properties:
+                    message:
+                      description: Message is the error message recorded upon the last generator run.
+                      type: string
+                    resource:
+                      description: Resource is the Namespaced name of the errored resource.
+                      type: string
+                  required:
+                    - message
+                    - resource
+                  type: object
+                type: array
+              resources:
+                default: 0
+                description: Resources is the sum of targeted TenantControlPlane objects.
+                type: integer
+            required:
+              - availableResources
+              - resources
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}

charts/kamaji-crds/templates/_helpers.tpl

@@ -0,0 +1,49 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "kamaji-crds.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "kamaji.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "kamaji-crds.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Create the cert-manager annotation to inject Certificate CA.
+*/}}
+{{- define "kamaji-crds.certManagerAnnotation" -}}
+{{- printf "%s/%s" (required "A valid .Values.kamajiNamespace is required" .Values.kamajiNamespace) (required "A valid .Values.kamajiCertificateName is required" .Values.kamajiCertificateName) }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "kamaji-crds.labels" -}}
+helm.sh/chart: {{ include "kamaji-crds.chart" . }}
+app.kubernetes.io/name: {{ include "kamaji-crds.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+app.kubernetes.io/component: "crds"
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}

charts/kamaji-crds/templates/kamaji.clastix.io_datastores.yaml

@@ -0,0 +1,10 @@
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    cert-manager.io/inject-ca-from: {{ include "kamaji-crds.certManagerAnnotation" . }}
+  labels:
+    {{- include "kamaji-crds.labels" . | nindent 4 }}
+  name: datastores.kamaji.clastix.io
+spec:
+  {{ tpl (.Files.Get "hack/kamaji.clastix.io_datastores_spec.yaml") . | nindent 2}}

charts/kamaji-crds/templates/kamaji.clastix.io_kubeconfiggenerators.yaml

@@ -0,0 +1,10 @@
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    cert-manager.io/inject-ca-from: {{ include "kamaji-crds.certManagerAnnotation" . }}
+  labels:
+    {{- include "kamaji-crds.labels" . | nindent 4 }}
+  name: kubeconfiggenerators.kamaji.clastix.io
+spec:
+  {{ tpl (.Files.Get "hack/kamaji.clastix.io_kubeconfiggenerators_spec.yaml") . | nindent 2 }}

charts/kamaji-crds/templates/kamaji.clastix.io_tenantcontrolplanes.yaml

@@ -0,0 +1,10 @@
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    cert-manager.io/inject-ca-from: {{ include "kamaji-crds.certManagerAnnotation" . }}
+  labels:
+    {{- include "kamaji-crds.labels" . | nindent 4 }}
+  name: tenantcontrolplanes.kamaji.clastix.io
+spec:
+  {{ tpl (.Files.Get "hack/kamaji.clastix.io_tenantcontrolplanes_spec.yaml") . | nindent 2 }}

charts/kamaji-crds/values.yaml

@@ -0,0 +1,15 @@
+# Default values for kamaji-crds.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+# -- Overrides the name of the chart for resource naming purposes.
+nameOverride: ""
+# -- Overrides the full name of the resources created by the chart.
+fullnameOverride: ""
+
+# -- The namespace where Kamaji has been installed: required to inject the Certificate Authority for cert-manager.
+kamajiNamespace: kamaji-system
+# -- The Kamaji webhook Service name.
+kamajiService: kamaji-webhook-service
+# -- The cert-manager Certificate resource name, holding the Certificate Authority for webhooks.
+kamajiCertificateName: kamaji-serving-cert

charts/kamaji/.helmignore

@@ -21,3 +21,8 @@
 .idea/
 *.tmproj
 .vscode/
+# Helm source files
+README.md.gotmpl
+.helmignore
+# Build tools
+Makefile

charts/kamaji/Chart.lock

@@ -1,6 +1,6 @@
 dependencies:
 - name: kamaji-etcd
   repository: https://clastix.github.io/charts
-  version: 0.7.0
-digest: sha256:2e23a11b06da2aa3a935a56a93d56876c6c8ef0a7800ee44070f9a269dfbca71
-generated: "2024-08-10T23:03:35.382486587+02:00"
+  version: 0.11.0
+digest: sha256:96b4115b8c02f771f809ec1bed3be3a3903e7e8315d6966aa54b0f73230ea421
+generated: "2025-07-03T09:19:19.835421461+02:00"

charts/kamaji/Chart.yaml

@@ -1,5 +1,5 @@
 apiVersion: v2
-appVersion: v1.0.0
+appVersion: latest
 description: Kamaji is the Hosted Control Plane Manager for Kubernetes.
 home: https://github.com/clastix/kamaji
 icon: https://github.com/clastix/kamaji/raw/master/assets/logo-colored.png
@@ -17,11 +17,11 @@ name: kamaji
 sources:
 - https://github.com/clastix/kamaji
 type: application
-version: 2.0.0
+version: 0.0.0+latest
 dependencies:
 - name: kamaji-etcd
   repository: https://clastix.github.io/charts
-  version: ">=0.7.0"
+  version: ">=0.11.0"
   condition: kamaji-etcd.deploy
 annotations:
   catalog.cattle.io/certified: partner
@@ -46,4 +46,5 @@ annotations:
   artifacthub.io/operator: "true"
   artifacthub.io/operatorCapabilities: "full lifecycle"
   artifacthub.io/changes: |
-    - Using dependency chart `kamaji-etcd` as a default DataStore.
+    - kind: added
+      description: Releasing latest chart at every push

charts/kamaji/README.md

@@ -1,6 +1,6 @@
 # kamaji
 
-![Version: 2.0.0](https://img.shields.io/badge/Version-2.0.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v1.0.0](https://img.shields.io/badge/AppVersion-v1.0.0-informational?style=flat-square)
+![Version: 0.0.0+latest](https://img.shields.io/badge/Version-0.0.0+latest-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: latest](https://img.shields.io/badge/AppVersion-latest-informational?style=flat-square)
 
 Kamaji is the Hosted Control Plane Manager for Kubernetes.
 
@@ -22,7 +22,7 @@ Kubernetes: `>=1.21.0-0`
 
 | Repository | Name | Version |
 |------------|------|---------|
-| https://clastix.github.io/charts | kamaji-etcd | >=0.7.0 |
+| https://clastix.github.io/charts | kamaji-etcd | >=0.11.0 |
 
 [Kamaji](https://github.com/clastix/kamaji) requires a [multi-tenant `etcd`](https://github.com/clastix/kamaji-internal/blob/master/deploy/getting-started-with-kamaji.md#setup-internal-multi-tenant-etcd) cluster.
 This Helm Chart starting from v0.1.1 provides the installation of an internal `etcd` in order to streamline the local test. If you'd like to use an externally managed etcd instance, you can specify the overrides and by setting the value `etcd.deploy=false`.
@@ -31,9 +31,13 @@ This Helm Chart starting from v0.1.1 provides the installation of an internal `e
 
 ## Install Kamaji
 
+To add clastix helm repository:
+
+        helm repo add clastix https://clastix.github.io/charts
+
 To install the Chart with the release name `kamaji`:
 
-        helm upgrade --install --namespace kamaji-system --create-namespace clastix/kamaji
+        helm upgrade --install --namespace kamaji-system --create-namespace kamaji clastix/kamaji
 
 Show the status:
 
@@ -70,7 +74,7 @@ Here the values you can override:
 | Key | Type | Default | Description |
 |-----|------|---------|-------------|
 | affinity | object | `{}` | Kubernetes affinity rules to apply to Kamaji controller pods |
-| defaultDatastoreName | string | `"default"` | Specify the default DataStore name for the Kamaji instance. |
+| defaultDatastoreName | string | `"default"` | If specified, all the Kamaji instances with an unassigned DataStore will inherit this default value. |
 | extraArgs | list | `[]` | A list of extra arguments to add to the kamaji controller default ones |
 | fullnameOverride | string | `""` |  |
 | healthProbeBindAddress | string | `":8081"` | The address the probe endpoint binds to. (default ":8081") |
@@ -78,10 +82,25 @@ Here the values you can override:
 | image.repository | string | `"clastix/kamaji"` | The container image of the Kamaji controller. |
 | image.tag | string | `nil` | Overrides the image tag whose default is the chart appVersion. |
 | imagePullSecrets | list | `[]` |  |
-| kamaji-etcd.datastore.enabled | bool | `true` |  |
-| kamaji-etcd.datastore.name | string | `"default"` |  |
-| kamaji-etcd.deploy | bool | `true` |  |
-| kamaji-etcd.fullnameOverride | string | `"kamaji-etcd"` |  |
+| kamaji-etcd | object | `{"clusterDomain":"cluster.local","datastore":{"enabled":true,"name":"default"},"deploy":true,"fullnameOverride":"kamaji-etcd"}` | Subchart: See https://github.com/clastix/kamaji-etcd/blob/master/charts/kamaji-etcd/values.yaml |
+| kubeconfigGenerator.affinity | object | `{}` | Kubernetes affinity rules to apply to Kubeconfig Generator controller pods |
+| kubeconfigGenerator.enableLeaderElect | bool | `true` | Enables the leader election. |
+| kubeconfigGenerator.enabled | bool | `false` | Toggle to deploy the Kubeconfig Generator Deployment. |
+| kubeconfigGenerator.extraArgs | list | `[]` | A list of extra arguments to add to the Kubeconfig Generator controller default ones. |
+| kubeconfigGenerator.fullnameOverride | string | `""` |  |
+| kubeconfigGenerator.healthProbeBindAddress | string | `":8081"` | The address the probe endpoint binds to. |
+| kubeconfigGenerator.loggingDevel.enable | bool | `false` | Development Mode defaults(encoder=consoleEncoder,logLevel=Debug,stackTraceLevel=Warn). Production Mode defaults(encoder=jsonEncoder,logLevel=Info,stackTraceLevel=Error) |
+| kubeconfigGenerator.nodeSelector | object | `{}` | Kubernetes node selector rules to schedule Kubeconfig Generator controller |
+| kubeconfigGenerator.podAnnotations | object | `{}` | The annotations to apply to the Kubeconfig Generator controller pods. |
+| kubeconfigGenerator.podSecurityContext | object | `{"runAsNonRoot":true}` | The securityContext to apply to the Kubeconfig Generator controller pods. |
+| kubeconfigGenerator.replicaCount | int | `2` | The number of the pod replicas for the Kubeconfig Generator controller. |
+| kubeconfigGenerator.resources.limits.cpu | string | `"200m"` |  |
+| kubeconfigGenerator.resources.limits.memory | string | `"512Mi"` |  |
+| kubeconfigGenerator.resources.requests.cpu | string | `"200m"` |  |
+| kubeconfigGenerator.resources.requests.memory | string | `"512Mi"` |  |
+| kubeconfigGenerator.securityContext | object | `{"allowPrivilegeEscalation":false}` | The securityContext to apply to the Kubeconfig Generator controller container only. |
+| kubeconfigGenerator.serviceAccountOverride | string | `""` | The name of the service account to use. If not set, the root Kamaji one will be used. |
+| kubeconfigGenerator.tolerations | list | `[]` | Kubernetes node taints that the Kubeconfig Generator controller pods would tolerate |
 | livenessProbe | object | `{"httpGet":{"path":"/healthz","port":"healthcheck"},"initialDelaySeconds":15,"periodSeconds":20}` | The livenessProbe for the controller container |
 | loggingDevel.enable | bool | `false` | Development Mode defaults(encoder=consoleEncoder,logLevel=Debug,stackTraceLevel=Warn). Production Mode defaults(encoder=jsonEncoder,logLevel=Info,stackTraceLevel=Error) (default false) |
 | metricsBindAddress | string | `":8080"` | The address the metric endpoint binds to. (default ":8080") |

charts/kamaji/README.md.gotmpl

@@ -18,10 +18,15 @@ This Helm Chart starting from v0.1.1 provides the installation of an internal `e
 
 ## Install Kamaji
 
+To add clastix helm repository:
+
+
+        helm repo add clastix https://clastix.github.io/charts
+
 To install the Chart with the release name `kamaji`:
 
 
-        helm upgrade --install --namespace kamaji-system --create-namespace clastix/kamaji
+        helm upgrade --install --namespace kamaji-system --create-namespace kamaji clastix/kamaji
 
 Show the status:
 

charts/kamaji/app-readme.md

@@ -1,12 +0,0 @@
-# Kamaji
-
-Kamaji deploys and operates Kubernetes at scale with a fraction of the operational burden.
-
-Useful links:
-- [Kamaji Github repository](https://github.com/clastix/kamaji)
-- [Kamaji Documentation](https://kamaji.clastix.io)
-
-## Requirements
-
-* Kubernetes v1.22+
-* Helm v3

charts/kamaji/controller-gen/clusterrole.yaml

@@ -0,0 +1,97 @@
+- apiGroups:
+    - ""
+  resources:
+    - namespaces
+  verbs:
+    - get
+    - list
+    - watch
+- apiGroups:
+    - apps
+  resources:
+    - deployments
+  verbs:
+    - create
+    - delete
+    - get
+    - list
+    - patch
+    - update
+    - watch
+- apiGroups:
+    - batch
+  resources:
+    - jobs
+  verbs:
+    - create
+    - delete
+    - get
+    - list
+    - watch
+- apiGroups:
+    - ""
+  resources:
+    - configmaps
+    - secrets
+    - services
+  verbs:
+    - create
+    - delete
+    - get
+    - list
+    - patch
+    - update
+    - watch
+- apiGroups:
+    - kamaji.clastix.io
+  resources:
+    - datastores
+    - tenantcontrolplanes
+  verbs:
+    - create
+    - delete
+    - get
+    - list
+    - patch
+    - update
+    - watch
+- apiGroups:
+    - kamaji.clastix.io
+  resources:
+    - datastores/status
+    - kubeconfiggenerators/status
+    - tenantcontrolplanes/status
+  verbs:
+    - get
+    - patch
+    - update
+- apiGroups:
+    - kamaji.clastix.io
+  resources:
+    - kubeconfiggenerators
+  verbs:
+    - create
+    - get
+    - list
+    - patch
+    - update
+    - watch
+- apiGroups:
+    - kamaji.clastix.io
+  resources:
+    - kubeconfiggenerators/finalizers
+    - tenantcontrolplanes/finalizers
+  verbs:
+    - update
+- apiGroups:
+    - networking.k8s.io
+  resources:
+    - ingresses
+  verbs:
+    - create
+    - delete
+    - get
+    - list
+    - patch
+    - update
+    - watch

charts/kamaji/controller-gen/crd-conversion.yaml

@@ -0,0 +1,11 @@
+spec:
+  conversion:
+    strategy: Webhook
+    webhook:
+      clientConfig:
+        service:
+          name: kamaji-webhook-service
+          namespace: kamaji-system
+          path: /convert
+      conversionReviewVersions:
+        - v1

charts/kamaji/controller-gen/mutating-webhook.yaml

@@ -0,0 +1,20 @@
+- admissionReviewVersions:
+    - v1
+  clientConfig:
+    service:
+      name: '{{ include "kamaji.webhookServiceName" . }}'
+      namespace: '{{ .Release.Namespace }}'
+      path: /mutate-kamaji-clastix-io-v1alpha1-tenantcontrolplane
+  failurePolicy: Fail
+  name: mtenantcontrolplane.kb.io
+  rules:
+    - apiGroups:
+        - kamaji.clastix.io
+      apiVersions:
+        - v1alpha1
+      operations:
+        - CREATE
+        - UPDATE
+      resources:
+        - tenantcontrolplanes
+  sideEffects: None

charts/kamaji/controller-gen/validating-webhook.yaml

@@ -0,0 +1,81 @@
+- admissionReviewVersions:
+    - v1
+  clientConfig:
+    service:
+      name: '{{ include "kamaji.webhookServiceName" . }}'
+      namespace: '{{ .Release.Namespace }}'
+      path: /telemetry
+  failurePolicy: Ignore
+  name: telemetry.kamaji.clastix.io
+  rules:
+    - apiGroups:
+        - kamaji.clastix.io
+      apiVersions:
+        - v1alpha1
+      operations:
+        - CREATE
+        - UPDATE
+        - DELETE
+      resources:
+        - tenantcontrolplanes
+  sideEffects: None
+- admissionReviewVersions:
+    - v1
+  clientConfig:
+    service:
+      name: '{{ include "kamaji.webhookServiceName" . }}'
+      namespace: '{{ .Release.Namespace }}'
+      path: /validate-kamaji-clastix-io-v1alpha1-datastore
+  failurePolicy: Fail
+  name: vdatastore.kb.io
+  rules:
+    - apiGroups:
+        - kamaji.clastix.io
+      apiVersions:
+        - v1alpha1
+      operations:
+        - CREATE
+        - UPDATE
+        - DELETE
+      resources:
+        - datastores
+  sideEffects: None
+- admissionReviewVersions:
+    - v1
+  clientConfig:
+    service:
+      name: '{{ include "kamaji.webhookServiceName" . }}'
+      namespace: '{{ .Release.Namespace }}'
+      path: /validate--v1-secret
+  failurePolicy: Ignore
+  name: vdatastoresecrets.kb.io
+  rules:
+    - apiGroups:
+        - ""
+      apiVersions:
+        - v1
+      operations:
+        - DELETE
+      resources:
+        - secrets
+  sideEffects: None
+- admissionReviewVersions:
+    - v1
+  clientConfig:
+    service:
+      name: '{{ include "kamaji.webhookServiceName" . }}'
+      namespace: '{{ .Release.Namespace }}'
+      path: /validate-kamaji-clastix-io-v1alpha1-tenantcontrolplane
+  failurePolicy: Fail
+  name: vtenantcontrolplane.kb.io
+  rules:
+    - apiGroups:
+        - kamaji.clastix.io
+      apiVersions:
+        - v1alpha1
+      operations:
+        - CREATE
+        - UPDATE
+      resources:
+        - tenantcontrolplanes
+  sideEffects: None

charts/kamaji/crds/kamaji.clastix.io_datastores.yaml

@@ -4,7 +4,7 @@ kind: CustomResourceDefinition
 metadata:
   annotations:
     cert-manager.io/inject-ca-from: kamaji-system/kamaji-serving-cert
-    controller-gen.kubebuilder.io/version: v0.11.4
+    controller-gen.kubebuilder.io/version: v0.16.1
   name: datastores.kamaji.clastix.io
 spec:
   group: kamaji.clastix.io
@@ -71,12 +71,10 @@ spec:
                               minLength: 1
                               type: string
                             name:
-                              description: name is unique within a namespace to reference
-                                a secret resource.
+                              description: name is unique within a namespace to reference a secret resource.
                               type: string
                             namespace:
-                              description: namespace defines the space within which
-                                the secret name must be unique.
+                              description: namespace defines the space within which the secret name must be unique.
                               type: string
                           required:
                             - keyPath
@@ -100,12 +98,10 @@ spec:
                               minLength: 1
                               type: string
                             name:
-                              description: name is unique within a namespace to reference
-                                a secret resource.
+                              description: name is unique within a namespace to reference a secret resource.
                               type: string
                             namespace:
-                              description: namespace defines the space within which
-                                the secret name must be unique.
+                              description: namespace defines the space within which the secret name must be unique.
                               type: string
                           required:
                             - keyPath
@@ -124,6 +120,9 @@ spec:
                     - PostgreSQL
                     - NATS
                   type: string
+                  x-kubernetes-validations:
+                    - message: Datastore driver is immutable
+                      rule: self == oldSelf
                 endpoints:
                   description: |-
                     List of the endpoints to connect to the shared datastore.
@@ -159,12 +158,10 @@ spec:
                                   minLength: 1
                                   type: string
                                 name:
-                                  description: name is unique within a namespace to
-                                    reference a secret resource.
+                                  description: name is unique within a namespace to reference a secret resource.
                                   type: string
                                 namespace:
-                                  description: namespace defines the space within which
-                                    the secret name must be unique.
+                                  description: namespace defines the space within which the secret name must be unique.
                                   type: string
                               required:
                                 - keyPath
@@ -188,12 +185,10 @@ spec:
                                   minLength: 1
                                   type: string
                                 name:
-                                  description: name is unique within a namespace to
-                                    reference a secret resource.
+                                  description: name is unique within a namespace to reference a secret resource.
                                   type: string
                                 namespace:
-                                  description: namespace defines the space within which
-                                    the secret name must be unique.
+                                  description: namespace defines the space within which the secret name must be unique.
                                   type: string
                               required:
                                 - keyPath
@@ -204,8 +199,7 @@ spec:
                         - certificate
                       type: object
                     clientCertificate:
-                      description: Specifies the SSL/TLS key and private key pair used
-                        to connect to the data store.
+                      description: Specifies the SSL/TLS key and private key pair used to connect to the data store.
                       properties:
                         certificate:
                           properties:
@@ -224,12 +218,10 @@ spec:
                                   minLength: 1
                                   type: string
                                 name:
-                                  description: name is unique within a namespace to
-                                    reference a secret resource.
+                                  description: name is unique within a namespace to reference a secret resource.
                                   type: string
                                 namespace:
-                                  description: namespace defines the space within which
-                                    the secret name must be unique.
+                                  description: namespace defines the space within which the secret name must be unique.
                                   type: string
                               required:
                                 - keyPath
@@ -253,12 +245,10 @@ spec:
                                   minLength: 1
                                   type: string
                                 name:
-                                  description: name is unique within a namespace to
-                                    reference a secret resource.
+                                  description: name is unique within a namespace to reference a secret resource.
                                   type: string
                                 namespace:
-                                  description: namespace defines the space within which
-                                    the secret name must be unique.
+                                  description: namespace defines the space within which the secret name must be unique.
                                   type: string
                               required:
                                 - keyPath
@@ -276,12 +266,26 @@ spec:
                 - driver
                 - endpoints
               type: object
+              x-kubernetes-validations:
+                - message: certificateAuthority privateKey must have secretReference or content when driver is etcd
+                  rule: '(self.driver == "etcd") ? (self.tlsConfig != null && (has(self.tlsConfig.certificateAuthority.privateKey.secretReference) || has(self.tlsConfig.certificateAuthority.privateKey.content))) : true'
+                - message: clientCertificate must have secretReference or content when driver is etcd
+                  rule: '(self.driver == "etcd") ? (self.tlsConfig != null && (has(self.tlsConfig.clientCertificate.certificate.secretReference) || has(self.tlsConfig.clientCertificate.certificate.content))) : true'
+                - message: clientCertificate privateKey must have secretReference or content when driver is etcd
+                  rule: '(self.driver == "etcd") ? (self.tlsConfig != null && (has(self.tlsConfig.clientCertificate.privateKey.secretReference) || has(self.tlsConfig.clientCertificate.privateKey.content))) : true'
+                - message: When driver is not etcd and tlsConfig exists, clientCertificate must be null or contain valid content
+                  rule: '(self.driver != "etcd" && has(self.tlsConfig) && has(self.tlsConfig.clientCertificate)) ? (((has(self.tlsConfig.clientCertificate.certificate.secretReference) || has(self.tlsConfig.clientCertificate.certificate.content)))) : true'
+                - message: When driver is not etcd and basicAuth exists, username must have secretReference or content
+                  rule: '(self.driver != "etcd" && has(self.basicAuth)) ? ((has(self.basicAuth.username.secretReference) || has(self.basicAuth.username.content))) : true'
+                - message: When driver is not etcd and basicAuth exists, password must have secretReference or content
+                  rule: '(self.driver != "etcd" && has(self.basicAuth)) ? ((has(self.basicAuth.password.secretReference) || has(self.basicAuth.password.content))) : true'
+                - message: When driver is not etcd, either tlsConfig or basicAuth must be provided
+                  rule: '(self.driver != "etcd") ? (has(self.tlsConfig) || has(self.basicAuth)) : true'
             status:
               description: DataStoreStatus defines the observed state of DataStore.
               properties:
                 usedBy:
-                  description: List of the Tenant Control Planes, namespaced named,
-                    using this data store.
+                  description: List of the Tenant Control Planes, namespaced named, using this data store.
                   items:
                     type: string
                   type: array

charts/kamaji/crds/kamaji.clastix.io_kubeconfiggenerators.yaml

@@ -0,0 +1,222 @@
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    cert-manager.io/inject-ca-from: kamaji-system/kamaji-serving-cert
+    controller-gen.kubebuilder.io/version: v0.16.1
+  name: kubeconfiggenerators.kamaji.clastix.io
+spec:
+  group: kamaji.clastix.io
+  names:
+    categories:
+      - kamaji
+    kind: KubeconfigGenerator
+    listKind: KubeconfigGeneratorList
+    plural: kubeconfiggenerators
+    shortNames:
+      - kc
+    singular: kubeconfiggenerator
+  scope: Cluster
+  versions:
+    - additionalPrinterColumns:
+        - description: Age
+          jsonPath: .metadata.creationTimestamp
+          name: Age
+          type: date
+      name: v1alpha1
+      schema:
+        openAPIV3Schema:
+          description: KubeconfigGenerator is the Schema for the kubeconfiggenerators API.
+          properties:
+            apiVersion:
+              description: |-
+                APIVersion defines the versioned schema of this representation of an object.
+                Servers should convert recognized schemas to the latest internal value, and
+                may reject unrecognized values.
+                More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+              type: string
+            kind:
+              description: |-
+                Kind is a string value representing the REST resource this object represents.
+                Servers may infer this from the endpoint the client submits requests to.
+                Cannot be updated.
+                In CamelCase.
+                More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+              type: string
+            metadata:
+              type: object
+            spec:
+              properties:
+                controlPlaneEndpointFrom:
+                  default: admin.svc
+                  description: |-
+                    ControlPlaneEndpointFrom is the key used to extract the Tenant Control Plane endpoint that must be used by the generator.
+                    The targeted Secret is the `${TCP}-admin-kubeconfig` one, default to `admin.svc`.
+                  type: string
+                groups:
+                  description: |-
+                    Groups is resolved a set of strings used to assign the x509 organisations field.
+                    It will be recognised by Kubernetes as user groups.
+                  items:
+                    description: |-
+                      CompoundValue allows defining a static, or a dynamic value.
+                      Options are mutually exclusive, just one should be picked up.
+                    properties:
+                      fromDefinition:
+                        description: |-
+                          FromDefinition is used to generate a dynamic value,
+                          it uses the dot notation to access fields from the referenced TenantControlPlane object:
+                          e.g.: metadata.name
+                        type: string
+                      stringValue:
+                        description: StringValue is a static string value.
+                        type: string
+                    type: object
+                    x-kubernetes-validations:
+                      - message: Either stringValue or fromDefinition must be set, but not both.
+                        rule: (has(self.stringValue) || has(self.fromDefinition)) && !(has(self.stringValue) && has(self.fromDefinition))
+                  type: array
+                namespaceSelector:
+                  description: NamespaceSelector is used to filter Namespaces from which the generator should extract TenantControlPlane objects.
+                  properties:
+                    matchExpressions:
+                      description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
+                      items:
+                        description: |-
+                          A label selector requirement is a selector that contains values, a key, and an operator that
+                          relates the key and values.
+                        properties:
+                          key:
+                            description: key is the label key that the selector applies to.
+                            type: string
+                          operator:
+                            description: |-
+                              operator represents a key's relationship to a set of values.
+                              Valid operators are In, NotIn, Exists and DoesNotExist.
+                            type: string
+                          values:
+                            description: |-
+                              values is an array of string values. If the operator is In or NotIn,
+                              the values array must be non-empty. If the operator is Exists or DoesNotExist,
+                              the values array must be empty. This array is replaced during a strategic
+                              merge patch.
+                            items:
+                              type: string
+                            type: array
+                            x-kubernetes-list-type: atomic
+                        required:
+                          - key
+                          - operator
+                        type: object
+                      type: array
+                      x-kubernetes-list-type: atomic
+                    matchLabels:
+                      additionalProperties:
+                        type: string
+                      description: |-
+                        matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels
+                        map is equivalent to an element of matchExpressions, whose key field is "key", the
+                        operator is "In", and the values array contains only "value". The requirements are ANDed.
+                      type: object
+                  type: object
+                  x-kubernetes-map-type: atomic
+                tenantControlPlaneSelector:
+                  description: TenantControlPlaneSelector is used to filter the TenantControlPlane objects that should be address by the generator.
+                  properties:
+                    matchExpressions:
+                      description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
+                      items:
+                        description: |-
+                          A label selector requirement is a selector that contains values, a key, and an operator that
+                          relates the key and values.
+                        properties:
+                          key:
+                            description: key is the label key that the selector applies to.
+                            type: string
+                          operator:
+                            description: |-
+                              operator represents a key's relationship to a set of values.
+                              Valid operators are In, NotIn, Exists and DoesNotExist.
+                            type: string
+                          values:
+                            description: |-
+                              values is an array of string values. If the operator is In or NotIn,
+                              the values array must be non-empty. If the operator is Exists or DoesNotExist,
+                              the values array must be empty. This array is replaced during a strategic
+                              merge patch.
+                            items:
+                              type: string
+                            type: array
+                            x-kubernetes-list-type: atomic
+                        required:
+                          - key
+                          - operator
+                        type: object
+                      type: array
+                      x-kubernetes-list-type: atomic
+                    matchLabels:
+                      additionalProperties:
+                        type: string
+                      description: |-
+                        matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels
+                        map is equivalent to an element of matchExpressions, whose key field is "key", the
+                        operator is "In", and the values array contains only "value". The requirements are ANDed.
+                      type: object
+                  type: object
+                  x-kubernetes-map-type: atomic
+                user:
+                  description: User resolves to a string to identify the client, assigned to the x509 Common Name field.
+                  properties:
+                    fromDefinition:
+                      description: |-
+                        FromDefinition is used to generate a dynamic value,
+                        it uses the dot notation to access fields from the referenced TenantControlPlane object:
+                        e.g.: metadata.name
+                      type: string
+                    stringValue:
+                      description: StringValue is a static string value.
+                      type: string
+                  type: object
+                  x-kubernetes-validations:
+                    - message: Either stringValue or fromDefinition must be set, but not both.
+                      rule: (has(self.stringValue) || has(self.fromDefinition)) && !(has(self.stringValue) && has(self.fromDefinition))
+              required:
+                - user
+              type: object
+            status:
+              description: KubeconfigGeneratorStatus defines the observed state of KubeconfigGenerator.
+              properties:
+                availableResources:
+                  default: 0
+                  description: |-
+                    AvailableResources is the sum of successfully generated resources.
+                    In case of a different value compared to Resources, check the field errors.
+                  type: integer
+                errors:
+                  description: Errors is the list of failed kubeconfig generations.
+                  items:
+                    properties:
+                      message:
+                        description: Message is the error message recorded upon the last generator run.
+                        type: string
+                      resource:
+                        description: Resource is the Namespaced name of the errored resource.
+                        type: string
+                    required:
+                      - message
+                      - resource
+                    type: object
+                  type: array
+                resources:
+                  default: 0
+                  description: Resources is the sum of targeted TenantControlPlane objects.
+                  type: integer
+              required:
+                - availableResources
+                - resources
+              type: object
+          type: object
+      served: true
+      storage: true
+      subresources:
+        status: {}

charts/kamaji/templates/_helpers.tpl

@@ -89,3 +89,15 @@ Create the name of the cert-manager Certificate
 {{- define "kamaji.certificateName" -}}
 {{- printf "%s-serving-cert" (include "kamaji.fullname" .) }}
 {{- end }}
+
+
+{{/*
+Kubeconfig Generator Deployment name.
+*/}}
+{{- define "kamaji.kubeconfigGeneratorName" -}}
+{{- if .Values.kubeconfigGenerator.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name "kubeconfig-generator" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}

charts/kamaji/templates/controller.yaml

@@ -19,10 +19,6 @@ spec:
       labels:
         {{- include "kamaji.selectorLabels" . | nindent 8 }}
     spec:
-      {{- with .Values.imagePullSecrets }}
-      imagePullSecrets:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
       securityContext:
         {{- toYaml .Values.podSecurityContext | nindent 8 }}
       serviceAccountName: {{ include "kamaji.serviceAccountName" . }}
@@ -33,8 +29,9 @@ spec:
         - --leader-elect
         - --metrics-bind-address={{ .Values.metricsBindAddress }}
         - --tmp-directory={{ .Values.temporaryDirectoryPath }}
-        {{- $datastoreName := .Values.defaultDatastoreName | required ".Values.defaultDatastoreName is required!" }}
-        - --datastore={{ $datastoreName }}
+        {{- if not (eq .Values.defaultDatastoreName "") }}
+        - --datastore={{ .Values.defaultDatastoreName }}
+        {{- end }}
         {{- if .Values.telemetry.disabled }}
         - --disable-telemetry
         {{- end }}
@@ -44,8 +41,6 @@ spec:
         {{- with .Values.extraArgs }}
         {{- toYaml . | nindent 8 }}
         {{- end }}
-        command:
-        - /kamaji
         env:
           - name: POD_NAMESPACE
             valueFrom:

charts/kamaji/templates/kubeconfiggenerator-deployment.yaml

@@ -0,0 +1,54 @@
+{{- if .Values.kubeconfigGenerator.enabled }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    {{- include "kamaji.labels" . | nindent 4 }}
+  name: {{ include "kamaji.kubeconfigGeneratorName" . }}
+  namespace: {{ .Release.Namespace }}
+spec:
+  replicas: {{ .Values.kubeconfigGenerator.replicaCount }}
+  selector:
+    matchLabels:
+      {{- include "kamaji.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      {{- with .Values.kubeconfigGenerator.podAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      labels:
+        {{- include "kamaji.selectorLabels" . | nindent 8 }}
+    spec:
+      securityContext:
+        {{- toYaml .Values.kubeconfigGenerator.podSecurityContext | nindent 8 }}
+      serviceAccountName: {{ default .Values.kubeconfigGenerator.serviceAccountOverride (include "kamaji.serviceAccountName" .) }}
+      containers:
+        - args:
+          - kubeconfig-generator
+          - --health-probe-bind-address={{ .Values.kubeconfigGenerator.healthProbeBindAddress }}
+          - --leader-elect={{ .Values.kubeconfigGenerator.enableLeaderElect }}
+          {{- if .Values.kubeconfigGenerator.loggingDevel.enable }}- --zap-devel{{- end }}
+          {{- with .Values.kubeconfigGenerator.extraArgs }}
+          {{- toYaml . | nindent 10 }}
+          {{- end }}
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          name: controller
+          resources:
+            {{- toYaml .Values.kubeconfigGenerator.resources | nindent 12 }}
+          securityContext:
+            {{- toYaml .Values.kubeconfigGenerator.securityContext | nindent 12 }}
+      {{- with .Values.kubeconfigGenerator.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.kubeconfigGenerator.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.kubeconfigGenerator.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+{{- end }}

charts/kamaji/templates/mutatingwebhookconfiguration.yaml

@@ -8,23 +8,4 @@ metadata:
     {{- include "kamaji.labels" $data | nindent 4 }}
   name: kamaji-mutating-webhook-configuration
 webhooks:
-  - admissionReviewVersions:
-      - v1
-    clientConfig:
-      service:
-        name: {{ include "kamaji.webhookServiceName" . }}
-        namespace: {{ .Release.Namespace }}
-        path: /mutate-kamaji-clastix-io-v1alpha1-tenantcontrolplane
-    failurePolicy: Fail
-    name: mtenantcontrolplane.kb.io
-    rules:
-      - apiGroups:
-          - kamaji.clastix.io
-        apiVersions:
-          - v1alpha1
-        operations:
-          - CREATE
-          - UPDATE
-        resources:
-          - tenantcontrolplanes
-    sideEffects: None
+{{ tpl (.Files.Get "controller-gen/mutating-webhook.yaml") . }}

charts/kamaji/templates/rbac.yaml

@@ -9,6 +9,10 @@ metadata:
     {{- toYaml . | nindent 4 }}
   {{- end }}
   namespace: {{ .Release.Namespace }}
+{{- with .Values.imagePullSecrets }}
+imagePullSecrets:
+  {{- toYaml . | nindent 2 }}
+{{- end }}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
@@ -54,122 +58,7 @@ metadata:
   creationTimestamp: null
   name: kamaji-manager-role
 rules:
-- apiGroups:
-  - apps
-  resources:
-  - deployments
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch
-- apiGroups:
-    - batch
-  resources:
-    - jobs
-  verbs:
-    - create
-    - delete
-    - get
-    - list
-    - watch
-- apiGroups:
-  - ""
-  resources:
-  - configmaps
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch
-- apiGroups:
-  - ""
-  resources:
-  - secrets
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch
-- apiGroups:
-  - ""
-  resources:
-  - services
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch
-- apiGroups:
-    - kamaji.clastix.io
-  resources:
-    - datastores
-  verbs:
-    - create
-    - delete
-    - get
-    - list
-    - patch
-    - update
-    - watch
-- apiGroups:
-    - kamaji.clastix.io
-  resources:
-    - datastores/status
-  verbs:
-    - get
-    - patch
-    - update
-- apiGroups:
-  - kamaji.clastix.io
-  resources:
-  - tenantcontrolplanes
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch
-- apiGroups:
-  - kamaji.clastix.io
-  resources:
-  - tenantcontrolplanes/finalizers
-  verbs:
-  - update
-- apiGroups:
-  - kamaji.clastix.io
-  resources:
-  - tenantcontrolplanes/status
-  verbs:
-  - get
-  - patch
-  - update
-- apiGroups:
-  - networking.k8s.io
-  resources:
-  - ingresses
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch
+{{ tpl (.Files.Get "controller-gen/clusterrole.yaml") . }}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole

charts/kamaji/templates/validatingwebhookconfiguration.yaml

@@ -8,84 +8,4 @@ metadata:
     {{- include "kamaji.labels" $data | nindent 4 }}
   name: kamaji-validating-webhook-configuration
 webhooks:
-  - admissionReviewVersions:
-      - v1
-    clientConfig:
-      service:
-        name: {{ include "kamaji.webhookServiceName" . }}
-        namespace: {{ .Release.Namespace }}
-        path: /telemetry
-    failurePolicy: Ignore
-    name: telemetry.kamaji.clastix.io
-    rules:
-      - apiGroups:
-          - kamaji.clastix.io
-        apiVersions:
-          - v1alpha1
-        operations:
-          - CREATE
-          - UPDATE
-          - DELETE
-        resources:
-          - tenantcontrolplanes
-    sideEffects: None
-  - admissionReviewVersions:
-      - v1
-    clientConfig:
-      service:
-        name: {{ include "kamaji.webhookServiceName" . }}
-        namespace: {{ .Release.Namespace }}
-        path: /validate--v1-secret
-    failurePolicy: Ignore
-    name: vdatastoresecrets.kb.io
-    rules:
-      - apiGroups:
-          - ""
-        apiVersions:
-          - v1
-        operations:
-          - DELETE
-        resources:
-          - secrets
-    sideEffects: None
-  - admissionReviewVersions:
-      - v1
-    clientConfig:
-      service:
-        name: {{ include "kamaji.webhookServiceName" . }}
-        namespace: {{ .Release.Namespace }}
-        path: /validate-kamaji-clastix-io-v1alpha1-datastore
-    failurePolicy: Fail
-    name: vdatastore.kb.io
-    rules:
-      - apiGroups:
-          - kamaji.clastix.io
-        apiVersions:
-          - v1alpha1
-        operations:
-          - CREATE
-          - UPDATE
-          - DELETE
-        resources:
-          - datastores
-    sideEffects: None
-  - admissionReviewVersions:
-      - v1
-    clientConfig:
-      service:
-        name: {{ include "kamaji.webhookServiceName" . }}
-        namespace: {{ .Release.Namespace }}
-        path: /validate-kamaji-clastix-io-v1alpha1-tenantcontrolplane
-    failurePolicy: Fail
-    name: vtenantcontrolplane.kb.io
-    rules:
-      - apiGroups:
-          - kamaji.clastix.io
-        apiVersions:
-          - v1alpha1
-        operations:
-          - CREATE
-          - UPDATE
-        resources:
-          - tenantcontrolplanes
-    sideEffects: None
+{{ tpl (.Files.Get "controller-gen/validating-webhook.yaml") . }}

charts/kamaji/values.yaml

@@ -95,12 +95,15 @@ loggingDevel:
   # -- Development Mode defaults(encoder=consoleEncoder,logLevel=Debug,stackTraceLevel=Warn). Production Mode defaults(encoder=jsonEncoder,logLevel=Info,stackTraceLevel=Error) (default false)
   enable: false
 
-# -- Specify the default DataStore name for the Kamaji instance.
+# -- If specified, all the Kamaji instances with an unassigned DataStore will inherit this default value.
 defaultDatastoreName: default
 
+# -- Subchart: See https://github.com/clastix/kamaji-etcd/blob/master/charts/kamaji-etcd/values.yaml
 kamaji-etcd:
   deploy: true
   fullnameOverride: kamaji-etcd
+  ## -- Important, this must match your management cluster's clusterDomain, otherwise the init jobs will fail
+  clusterDomain: "cluster.local"
   datastore:
     enabled: true
     name: default
@@ -108,4 +111,48 @@ kamaji-etcd:
 # -- Disable the analytics traces collection
 telemetry:
   disabled: false
-  
+
+kubeconfigGenerator:
+  # -- Toggle to deploy the Kubeconfig Generator Deployment.
+  enabled: false
+  fullnameOverride: ""
+  # -- The number of the pod replicas for the Kubeconfig Generator controller.
+  replicaCount: 2
+  # -- The annotations to apply to the Kubeconfig Generator controller pods.
+  podAnnotations: {}
+  # -- The securityContext to apply to the Kubeconfig Generator controller pods.
+  podSecurityContext:
+    runAsNonRoot: true
+  # -- The name of the service account to use. If not set, the root Kamaji one will be used.
+  serviceAccountOverride: ""
+  # -- The address the probe endpoint binds to.
+  healthProbeBindAddress: ":8081"
+  # -- Enables the leader election.
+  enableLeaderElect: true
+  loggingDevel:
+    # -- Development Mode defaults(encoder=consoleEncoder,logLevel=Debug,stackTraceLevel=Warn). Production Mode defaults(encoder=jsonEncoder,logLevel=Info,stackTraceLevel=Error)
+    enable: false
+  # -- A list of extra arguments to add to the Kubeconfig Generator controller default ones.
+  extraArgs: []
+  resources:
+    limits:
+      cpu: 200m
+      memory: 512Mi
+    requests:
+      cpu: 200m
+      memory: 512Mi
+  # -- The securityContext to apply to the Kubeconfig Generator controller container only.
+  securityContext:
+    allowPrivilegeEscalation: false
+    # capabilities:
+    #   drop:
+    #   - ALL
+    # readOnlyRootFilesystem: true
+    # runAsNonRoot: true
+    # runAsUser: 1000
+  # -- Kubernetes node selector rules to schedule Kubeconfig Generator controller
+  nodeSelector: {}
+  # -- Kubernetes node taints that the Kubeconfig Generator controller pods would tolerate
+  tolerations: []
+  # -- Kubernetes affinity rules to apply to Kubeconfig Generator controller pods
+  affinity: {}

cmd/kubeconfig-generator/cmd.go

@@ -0,0 +1,167 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package kubeconfiggenerator
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	goRuntime "runtime"
+	"time"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/rest"
+	"k8s.io/klog/v2"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/cache"
+	"sigs.k8s.io/controller-runtime/pkg/event"
+	"sigs.k8s.io/controller-runtime/pkg/healthz"
+	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
+
+	"github.com/clastix/kamaji/controllers"
+	"github.com/clastix/kamaji/internal"
+)
+
+func NewCmd(scheme *runtime.Scheme) *cobra.Command {
+	// CLI flags
+	var (
+		metricsBindAddress            string
+		healthProbeBindAddress        string
+		leaderElect                   bool
+		controllerReconcileTimeout    time.Duration
+		cacheResyncPeriod             time.Duration
+		managerNamespace              string
+		certificateExpirationDeadline time.Duration
+	)
+
+	cmd := &cobra.Command{
+		Use:           "kubeconfig-generator",
+		Short:         "Start the Kubeconfig Generator manager",
+		SilenceErrors: false,
+		SilenceUsage:  true,
+		PreRunE: func(*cobra.Command, []string) error {
+			// Avoid polluting stdout with useless details by the underlying klog implementations
+			klog.SetOutput(io.Discard)
+			klog.LogToStderr(false)
+
+			if certificateExpirationDeadline < 24*time.Hour {
+				return fmt.Errorf("certificate expiration deadline must be at least 24 hours")
+			}
+
+			return nil
+		},
+		RunE: func(*cobra.Command, []string) error {
+			ctx := ctrl.SetupSignalHandler()
+
+			setupLog := ctrl.Log.WithName("kubeconfig-generator")
+
+			setupLog.Info(fmt.Sprintf("Kamaji version %s %s%s", internal.GitTag, internal.GitCommit, internal.GitDirty))
+			setupLog.Info(fmt.Sprintf("Build from: %s", internal.GitRepo))
+			setupLog.Info(fmt.Sprintf("Build date: %s", internal.BuildTime))
+			setupLog.Info(fmt.Sprintf("Go Version: %s", goRuntime.Version()))
+			setupLog.Info(fmt.Sprintf("Go OS/Arch: %s/%s", goRuntime.GOOS, goRuntime.GOARCH))
+
+			ctrlOpts := ctrl.Options{
+				Scheme: scheme,
+				Metrics: metricsserver.Options{
+					BindAddress: metricsBindAddress,
+				},
+				HealthProbeBindAddress:  healthProbeBindAddress,
+				LeaderElection:          leaderElect,
+				LeaderElectionNamespace: managerNamespace,
+				LeaderElectionID:        "kubeconfiggenerator.kamaji.clastix.io",
+				NewCache: func(config *rest.Config, opts cache.Options) (cache.Cache, error) {
+					opts.SyncPeriod = &cacheResyncPeriod
+
+					return cache.New(config, opts)
+				},
+			}
+
+			triggerChan := make(chan event.GenericEvent)
+
+			mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrlOpts)
+			if err != nil {
+				setupLog.Error(err, "unable to start manager")
+
+				return err
+			}
+
+			setupLog.Info("setting probes")
+			{
+				if err = mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
+					setupLog.Error(err, "unable to set up health check")
+
+					return err
+				}
+				if err = mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil {
+					setupLog.Error(err, "unable to set up ready check")
+
+					return err
+				}
+			}
+
+			certController := &controllers.CertificateLifecycle{Channel: triggerChan, Deadline: certificateExpirationDeadline}
+			certController.EnqueueFn = certController.EnqueueForKubeconfigGenerator
+			if err = certController.SetupWithManager(mgr); err != nil {
+				setupLog.Error(err, "unable to create controller", "controller", "CertificateLifecycle")
+
+				return err
+			}
+
+			if err = (&controllers.KubeconfigGeneratorWatcher{
+				Client:        mgr.GetClient(),
+				GeneratorChan: triggerChan,
+			}).SetupWithManager(mgr); err != nil {
+				setupLog.Error(err, "unable to create controller", "controller", "KubeconfigGeneratorWatcher")
+
+				return err
+			}
+
+			if err = (&controllers.KubeconfigGeneratorReconciler{
+				Client:            mgr.GetClient(),
+				NotValidThreshold: certificateExpirationDeadline,
+				CertificateChan:   triggerChan,
+			}).SetupWithManager(mgr); err != nil {
+				setupLog.Error(err, "unable to create controller", "controller", "KubeconfigGenerator")
+
+				return err
+			}
+
+			setupLog.Info("starting manager")
+			if err = mgr.Start(ctx); err != nil {
+				setupLog.Error(err, "problem running manager")
+
+				return err
+			}
+
+			return nil
+		},
+	}
+	// Setting zap logger
+	zapfs := flag.NewFlagSet("zap", flag.ExitOnError)
+	opts := zap.Options{
+		Development: true,
+	}
+	opts.BindFlags(zapfs)
+	cmd.Flags().AddGoFlagSet(zapfs)
+	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
+	// Setting CLI flags
+	cmd.Flags().StringVar(&metricsBindAddress, "metrics-bind-address", ":8090", "The address the metric endpoint binds to.")
+	cmd.Flags().StringVar(&healthProbeBindAddress, "health-probe-bind-address", ":8091", "The address the probe endpoint binds to.")
+	cmd.Flags().BoolVar(&leaderElect, "leader-elect", true, "Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.")
+	cmd.Flags().DurationVar(&controllerReconcileTimeout, "controller-reconcile-timeout", 30*time.Second, "The reconciliation request timeout before the controller withdraw the external resource calls, such as dealing with the Datastore, or the Tenant Control Plane API endpoint.")
+	cmd.Flags().DurationVar(&cacheResyncPeriod, "cache-resync-period", 10*time.Hour, "The controller-runtime.Manager cache resync period.")
+	cmd.Flags().StringVar(&managerNamespace, "pod-namespace", os.Getenv("POD_NAMESPACE"), "The Kubernetes Namespace on which the Operator is running in, required for the TenantControlPlane migration jobs.")
+	cmd.Flags().DurationVar(&certificateExpirationDeadline, "certificate-expiration-deadline", 24*time.Hour, "Define the deadline upon certificate expiration to start the renewal process, cannot be less than a 24 hours.")
+
+	cobra.OnInitialize(func() {
+		viper.AutomaticEnv()
+	})
+
+	return cmd
+}

cmd/manager/cmd.go

@@ -4,6 +4,7 @@
 package manager
 
 import (
+	"context"
 	"flag"
 	"fmt"
 	"io"
@@ -20,6 +21,7 @@ import (
 	"k8s.io/klog/v2"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/cache"
+	"sigs.k8s.io/controller-runtime/pkg/event"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
@@ -41,46 +43,49 @@ import (
 func NewCmd(scheme *runtime.Scheme) *cobra.Command {
 	// CLI flags
 	var (
-		metricsBindAddress         string
-		healthProbeBindAddress     string
-		leaderElect                bool
-		tmpDirectory               string
-		kineImage                  string
-		controllerReconcileTimeout time.Duration
-		cacheResyncPeriod          time.Duration
-		datastore                  string
-		managerNamespace           string
-		managerServiceAccountName  string
-		managerServiceName         string
-		webhookCABundle            []byte
-		migrateJobImage            string
-		maxConcurrentReconciles    int
-		disableTelemetry           bool
+		metricsBindAddress            string
+		healthProbeBindAddress        string
+		leaderElect                   bool
+		tmpDirectory                  string
+		kineImage                     string
+		controllerReconcileTimeout    time.Duration
+		cacheResyncPeriod             time.Duration
+		datastore                     string
+		managerNamespace              string
+		managerServiceAccountName     string
+		managerServiceName            string
+		webhookCABundle               []byte
+		migrateJobImage               string
+		maxConcurrentReconciles       int
+		disableTelemetry              bool
+		certificateExpirationDeadline time.Duration
 
 		webhookCAPath string
 	)
 
-	ctx := ctrl.SetupSignalHandler()
-
 	cmd := &cobra.Command{
 		Use:           "manager",
 		Short:         "Start the Kamaji Kubernetes Operator",
 		SilenceErrors: false,
 		SilenceUsage:  true,
-		PreRunE: func(cmd *cobra.Command, args []string) (err error) {
+		PreRunE: func(cmd *cobra.Command, _ []string) (err error) {
 			// Avoid to pollute Kamaji stdout with useless details by the underlying klog implementations
 			klog.SetOutput(io.Discard)
 			klog.LogToStderr(false)
 
-			if err = cmdutils.CheckFlags(cmd.Flags(), []string{"kine-image", "datastore", "migrate-image", "tmp-directory", "pod-namespace", "webhook-service-name", "serviceaccount-name", "webhook-ca-path"}...); err != nil {
+			if err = cmdutils.CheckFlags(cmd.Flags(), []string{"kine-image", "migrate-image", "tmp-directory", "pod-namespace", "webhook-service-name", "serviceaccount-name", "webhook-ca-path"}...); err != nil {
 				return err
 			}
 
+			if certificateExpirationDeadline < 24*time.Hour {
+				return fmt.Errorf("certificate expiration deadline must be at least 24 hours")
+			}
+
 			if webhookCABundle, err = os.ReadFile(webhookCAPath); err != nil {
 				return fmt.Errorf("unable to read webhook CA: %w", err)
 			}
 
-			if err = datastoreutils.CheckExists(ctx, scheme, datastore); err != nil {
+			if err = datastoreutils.CheckExists(context.Background(), scheme, datastore); err != nil {
 				return err
 			}
 
@@ -90,7 +95,9 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command {
 
 			return nil
 		},
-		RunE: func(cmd *cobra.Command, args []string) error {
+		RunE: func(*cobra.Command, []string) error {
+			ctx := ctrl.SetupSignalHandler()
+
 			setupLog := ctrl.Log.WithName("setup")
 
 			setupLog.Info(fmt.Sprintf("Kamaji version %s %s%s", internal.GitTag, internal.GitCommit, internal.GitDirty))
@@ -131,7 +138,7 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command {
 				return err
 			}
 
-			tcpChannel, certChannel := make(controllers.TenantControlPlaneChannel), make(controllers.CertificateChannel)
+			tcpChannel, certChannel := make(chan event.GenericEvent), make(chan event.GenericEvent)
 
 			if err = (&controllers.DataStore{Client: mgr.GetClient(), TenantControlPlaneTrigger: tcpChannel}).SetupWithManager(mgr); err != nil {
 				setupLog.Error(err, "unable to create controller", "controller", "DataStore")
@@ -143,11 +150,12 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command {
 				Client:    mgr.GetClient(),
 				APIReader: mgr.GetAPIReader(),
 				Config: controllers.TenantControlPlaneReconcilerConfig{
-					ReconcileTimeout:     controllerReconcileTimeout,
-					DefaultDataStoreName: datastore,
-					KineContainerImage:   kineImage,
-					TmpBaseDirectory:     tmpDirectory,
+					DefaultDataStoreName:    datastore,
+					KineContainerImage:      kineImage,
+					TmpBaseDirectory:        tmpDirectory,
+					CertExpirationThreshold: certificateExpirationDeadline,
 				},
+				ReconcileTimeout:        controllerReconcileTimeout,
 				CertificateChan:         certChannel,
 				TriggerChan:             tcpChannel,
 				KamajiNamespace:         managerNamespace,
@@ -186,7 +194,10 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command {
 				}
 			}
 
-			if err = (&controllers.CertificateLifecycle{Channel: certChannel}).SetupWithManager(mgr); err != nil {
+			certController := &controllers.CertificateLifecycle{Channel: certChannel, Deadline: certificateExpirationDeadline}
+			certController.EnqueueFn = certController.EnqueueForTenantControlPlane
+
+			if err = certController.SetupWithManager(mgr); err != nil {
 				setupLog.Error(err, "unable to create controller", "controller", "CertificateLifecycle")
 
 				return err
@@ -214,9 +225,9 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command {
 					},
 				},
 				routes.TenantControlPlaneValidate{}: {
+					handlers.TenantControlPlaneCertSANs{},
 					handlers.TenantControlPlaneName{},
 					handlers.TenantControlPlaneVersion{},
-					handlers.TenantControlPlaneKubeletAddresses{},
 					handlers.TenantControlPlaneDataStore{Client: mgr.GetClient()},
 					handlers.TenantControlPlaneDeployment{
 						Client: mgr.GetClient(),
@@ -229,6 +240,7 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command {
 						},
 					},
 					handlers.TenantControlPlaneServiceCIDR{},
+					handlers.TenantControlPlaneLoadBalancerSourceRanges{},
 				},
 				routes.TenantControlPlaneTelemetry{}: {
 					handlers.TenantControlPlaneTelemetry{
@@ -298,8 +310,8 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command {
 	cmd.Flags().BoolVar(&leaderElect, "leader-elect", true, "Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.")
 	cmd.Flags().StringVar(&tmpDirectory, "tmp-directory", "/tmp/kamaji", "Directory which will be used to work with temporary files.")
 	cmd.Flags().StringVar(&kineImage, "kine-image", "rancher/kine:v0.11.10-amd64", "Container image along with tag to use for the Kine sidecar container (used only if etcd-storage-type is set to one of kine strategies).")
-	cmd.Flags().StringVar(&datastore, "datastore", "etcd", "The default DataStore that should be used by Kamaji to setup the required storage.")
-	cmd.Flags().StringVar(&migrateJobImage, "migrate-image", fmt.Sprintf("clastix/kamaji:%s", internal.GitTag), "Specify the container image to launch when a TenantControlPlane is migrated to a new datastore.")
+	cmd.Flags().StringVar(&datastore, "datastore", "", "Optional, the default DataStore that should be used by Kamaji to setup the required storage of Tenant Control Planes with undeclared DataStore.")
+	cmd.Flags().StringVar(&migrateJobImage, "migrate-image", fmt.Sprintf("%s/clastix/kamaji:%s", internal.ContainerRepository, internal.GitTag), "Specify the container image to launch when a TenantControlPlane is migrated to a new datastore.")
 	cmd.Flags().IntVar(&maxConcurrentReconciles, "max-concurrent-tcp-reconciles", 1, "Specify the number of workers for the Tenant Control Plane controller (beware of CPU consumption)")
 	cmd.Flags().StringVar(&managerNamespace, "pod-namespace", os.Getenv("POD_NAMESPACE"), "The Kubernetes Namespace on which the Operator is running in, required for the TenantControlPlane migration jobs.")
 	cmd.Flags().StringVar(&managerServiceName, "webhook-service-name", "kamaji-webhook-service", "The Kamaji webhook server Service name which is used to get validation webhooks, required for the TenantControlPlane migration jobs.")
@@ -308,6 +320,7 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command {
 	cmd.Flags().DurationVar(&controllerReconcileTimeout, "controller-reconcile-timeout", 30*time.Second, "The reconciliation request timeout before the controller withdraw the external resource calls, such as dealing with the Datastore, or the Tenant Control Plane API endpoint.")
 	cmd.Flags().DurationVar(&cacheResyncPeriod, "cache-resync-period", 10*time.Hour, "The controller-runtime.Manager cache resync period.")
 	cmd.Flags().BoolVar(&disableTelemetry, "disable-telemetry", false, "Disable the analytics traces collection.")
+	cmd.Flags().DurationVar(&certificateExpirationDeadline, "certificate-expiration-deadline", 24*time.Hour, "Define the deadline upon certificate expiration to start the renewal process, cannot be less than a 24 hours.")
 
 	cobra.OnInitialize(func() {
 		viper.AutomaticEnv()

cmd/migrate/cmd.go

@@ -22,16 +22,17 @@ import (
 func NewCmd(scheme *runtime.Scheme) *cobra.Command {
 	// CLI flags
 	var (
-		tenantControlPlane string
-		targetDataStore    string
-		timeout            time.Duration
+		tenantControlPlane    string
+		targetDataStore       string
+		cleanupPriorMigration bool
+		timeout               time.Duration
 	)
 
 	cmd := &cobra.Command{
 		Use:          "migrate",
 		Short:        "Migrate the data of a TenantControlPlane to another compatible DataStore",
 		SilenceUsage: true,
-		RunE: func(cmd *cobra.Command, args []string) error {
+		RunE: func(*cobra.Command, []string) error {
 			ctx, cancelFn := context.WithTimeout(context.Background(), timeout)
 			defer cancelFn()
 
@@ -95,6 +96,20 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command {
 				return err
 			}
 			defer targetConnection.Close()
+
+			if cleanupPriorMigration {
+				log.Info("Checking if target DataStore should be clean-up prior migration")
+
+				if exists, _ := targetConnection.DBExists(ctx, tcp.Status.Storage.Setup.Schema); exists {
+					log.Info("A colliding schema on target DataStore is present, cleaning up")
+
+					if dErr := targetConnection.DeleteDB(ctx, tcp.Status.Storage.Setup.Schema); dErr != nil {
+						return fmt.Errorf("error cleaning up prior migration: %s", dErr.Error())
+					}
+
+					log.Info("Cleaning up prior migration has been completed")
+				}
+			}
 			// Start migrating from the old Datastore to the new one
 			log.Info("migration from origin to target started")
 
@@ -110,6 +125,7 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command {
 
 	cmd.Flags().StringVar(&tenantControlPlane, "tenant-control-plane", "", "Namespaced-name of the TenantControlPlane that must be migrated (e.g.: default/test)")
 	cmd.Flags().StringVar(&targetDataStore, "target-datastore", "", "Name of the Datastore to which the TenantControlPlane will be migrated")
+	cmd.Flags().BoolVar(&cleanupPriorMigration, "cleanup-prior-migration", false, "When set to true, migration job will drop existing data in the target DataStore: useful to avoid stale data when migrating back and forth between DataStores.")
 	cmd.Flags().DurationVar(&timeout, "timeout", 5*time.Minute, "Amount of time for the context timeout")
 
 	_ = cmd.MarkFlagRequired("tenant-control-plane")

cmd/root.go

@@ -18,7 +18,7 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command {
 	return &cobra.Command{
 		Use:   "kamaji",
 		Short: "Build and operate Kubernetes at scale with a fraction of operational burden.",
-		PersistentPreRun: func(cmd *cobra.Command, args []string) {
+		PersistentPreRun: func(*cobra.Command, []string) {
 			utilruntime.Must(clientgoscheme.AddToScheme(scheme))
 			utilruntime.Must(kamajiv1alpha1.AddToScheme(scheme))
 			utilruntime.Must(appsv1.RegisterDefaults(scheme))

config/certmanager/certificate.yaml

@@ -1,39 +0,0 @@
-# The following manifests contain a self-signed issuer CR and a certificate CR.
-# More document can be found at https://docs.cert-manager.io
-# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes.
-apiVersion: cert-manager.io/v1
-kind: Issuer
-metadata:
-  labels:
-    app.kubernetes.io/name: issuer
-    app.kubernetes.io/instance: selfsigned-issuer
-    app.kubernetes.io/component: certificate
-    app.kubernetes.io/created-by: operator
-    app.kubernetes.io/part-of: operator
-    app.kubernetes.io/managed-by: kustomize
-  name: selfsigned-issuer
-  namespace: system
-spec:
-  selfSigned: {}
----
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
-  labels:
-    app.kubernetes.io/name: certificate
-    app.kubernetes.io/instance: serving-cert
-    app.kubernetes.io/component: certificate
-    app.kubernetes.io/created-by: operator
-    app.kubernetes.io/part-of: operator
-    app.kubernetes.io/managed-by: kustomize
-  name: serving-cert  # this name should match the one appeared in kustomizeconfig.yaml
-  namespace: system
-spec:
-  # $(SERVICE_NAME) and $(SERVICE_NAMESPACE) will be substituted by kustomize
-  dnsNames:
-  - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc
-  - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc.cluster.local
-  issuerRef:
-    kind: Issuer
-    name: selfsigned-issuer
-  secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize

config/certmanager/kustomization.yaml

@@ -1,5 +0,0 @@
-resources:
-- certificate.yaml
-
-configurations:
-- kustomizeconfig.yaml

config/certmanager/kustomizeconfig.yaml

@@ -1,16 +0,0 @@
-# This configuration is for teaching kustomize how to update name ref and var substitution 
-nameReference:
-- kind: Issuer
-  group: cert-manager.io
-  fieldSpecs:
-  - kind: Certificate
-    group: cert-manager.io
-    path: spec/issuerRef/name
-
-varReference:
-- kind: Certificate
-  group: cert-manager.io
-  path: spec/commonName
-- kind: Certificate
-  group: cert-manager.io
-  path: spec/dnsNames

config/crd/kustomization.yaml

@@ -1,19 +0,0 @@
-# This kustomization.yaml is not intended to be run by itself,
-# since it depends on service name and namespace that are out of this kustomize package.
-# It should be run by config/default
-resources:
-- bases/kamaji.clastix.io_tenantcontrolplanes.yaml
-- bases/kamaji.clastix.io_datastores.yaml
-#+kubebuilder:scaffold:crdkustomizeresource
-
-patchesStrategicMerge:
-- patches/webhook_in_clusters.yaml
-#+kubebuilder:scaffold:crdkustomizewebhookpatch
-
-- patches/cainjection_in_clusters.yaml
-- patches/cainjection_in_datastores.yaml
-#+kubebuilder:scaffold:crdkustomizecainjectionpatch
-
-# the following config is for teaching kustomize how to do kustomization for CRDs.
-configurations:
-- kustomizeconfig.yaml

config/crd/kustomizeconfig.yaml

@@ -1,19 +0,0 @@
-# This file is for teaching kustomize how to substitute name and namespace reference in CRD
-nameReference:
-- kind: Service
-  version: v1
-  fieldSpecs:
-  - kind: CustomResourceDefinition
-    version: v1
-    group: apiextensions.k8s.io
-    path: spec/conversion/webhook/clientConfig/service/name
-
-namespace:
-- kind: CustomResourceDefinition
-  version: v1
-  group: apiextensions.k8s.io
-  path: spec/conversion/webhook/clientConfig/service/namespace
-  create: false
-
-varReference:
-- path: metadata/annotations

config/crd/patches/cainjection_in_clusters.yaml

@@ -1,7 +0,0 @@
-# The following patch adds a directive for certmanager to inject CA into the CRD
-apiVersion: apiextensions.k8s.io/v1
-kind: CustomResourceDefinition
-metadata:
-  annotations:
-    cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME)
-  name: tenantcontrolplanes.kamaji.clastix.io

config/crd/patches/cainjection_in_datastores.yaml

@@ -1,7 +0,0 @@
-# The following patch adds a directive for certmanager to inject CA into the CRD
-apiVersion: apiextensions.k8s.io/v1
-kind: CustomResourceDefinition
-metadata:
-  annotations:
-    cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME)
-  name: datastores.kamaji.clastix.io

config/crd/patches/webhook_in_clusters.yaml

@@ -1,16 +0,0 @@
-# The following patch enables a conversion webhook for the CRD
-apiVersion: apiextensions.k8s.io/v1
-kind: CustomResourceDefinition
-metadata:
-  name: tenantcontrolplanes.kamaji.clastix.io
-spec:
-  conversion:
-    strategy: Webhook
-    webhook:
-      clientConfig:
-        service:
-          namespace: system
-          name: webhook-service
-          path: /convert
-      conversionReviewVersions:
-      - v1

config/crd/patches/webhook_in_datastores.yaml

@@ -1,16 +0,0 @@
-# The following patch enables a conversion webhook for the CRD
-apiVersion: apiextensions.k8s.io/v1
-kind: CustomResourceDefinition
-metadata:
-  name: datastores.kamaji.clastix.io
-spec:
-  conversion:
-    strategy: Webhook
-    webhook:
-      clientConfig:
-        service:
-          namespace: system
-          name: webhook-service
-          path: /convert
-      conversionReviewVersions:
-      - v1

config/default/kustomization.yaml

@@ -1,58 +0,0 @@
-# Adds namespace to all resources.
-namespace: kamaji-system
-
-# Value of this field is prepended to the
-# names of all resources, e.g. a deployment named
-# "wordpress" becomes "alices-wordpress".
-# Note that it should also match with the prefix (text before '-') of the namespace
-# field above.
-namePrefix: kamaji-
-
-# Labels to add to all resources and selectors.
-commonLabels:
-  cluster.x-k8s.io/provider: "kamaji-core"
-
-bases:
-- ../crd
-- ../rbac
-- ../manager
-- ../samples
-- ../webhook
-- ../certmanager
-- ../prometheus
-
-patchesStrategicMerge:
-# Mount the controller config file for loading manager configurations
-# through a ComponentConfig type
-#- manager_config_patch.yaml
-- manager_webhook_patch.yaml
-- webhookcainjection_patch.yaml
-
-# the following config is for teaching kustomize how to do var substitution
-vars:
-- name: CERTIFICATE_NAMESPACE # namespace of the certificate CR
-  objref:
-    kind: Certificate
-    group: cert-manager.io
-    version: v1
-    name: serving-cert # this name should match the one in certificate.yaml
-  fieldref:
-    fieldpath: metadata.namespace
-- name: CERTIFICATE_NAME
-  objref:
-    kind: Certificate
-    group: cert-manager.io
-    version: v1
-    name: serving-cert # this name should match the one in certificate.yaml
-- name: SERVICE_NAMESPACE # namespace of the service
-  objref:
-    kind: Service
-    version: v1
-    name: webhook-service
-  fieldref:
-    fieldpath: metadata.namespace
-- name: SERVICE_NAME
-  objref:
-    kind: Service
-    version: v1
-    name: webhook-service

config/default/manager_auth_proxy_patch.yaml

@@ -1,28 +0,0 @@
-# This patch inject a sidecar container which is a HTTP proxy for the
-# controller manager, it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews.
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: controller-manager
-  namespace: system
-spec:
-  template:
-    spec:
-      containers:
-      - name: kube-rbac-proxy
-        image: gcr.io/kubebuilder/kube-rbac-proxy:v0.8.0
-        args:
-        - "--secure-listen-address=0.0.0.0:8443"
-        - "--upstream=http://127.0.0.1:8080/"
-        - "--logtostderr=true"
-        - "--v=10"
-        ports:
-        - containerPort: 8443
-          protocol: TCP
-          name: https
-      - name: manager
-        args:
-        - "--health-probe-bind-address=:8081"
-        - "--metrics-bind-address=127.0.0.1:8080"
-        - "--leader-elect"
-        - "--datastore=kamaji-etcd"

config/default/manager_config_patch.yaml

@@ -1,20 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: controller-manager
-  namespace: system
-spec:
-  template:
-    spec:
-      containers:
-      - name: manager
-        args:
-        - "--config=controller_manager_config.yaml"
-        volumeMounts:
-        - name: manager-config
-          mountPath: /controller_manager_config.yaml
-          subPath: controller_manager_config.yaml
-      volumes:
-      - name: manager-config
-        configMap:
-          name: manager-config

config/default/manager_webhook_patch.yaml

@@ -1,23 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: controller-manager
-  namespace: system
-spec:
-  template:
-    spec:
-      containers:
-      - name: manager
-        ports:
-        - containerPort: 9443
-          name: webhook-server
-          protocol: TCP
-        volumeMounts:
-        - mountPath: /tmp/k8s-webhook-server/serving-certs
-          name: cert
-          readOnly: true
-      volumes:
-      - name: cert
-        secret:
-          defaultMode: 420
-          secretName: webhook-server-cert

config/default/webhookcainjection_patch.yaml

@@ -1,29 +0,0 @@
-# This patch add annotation to admission webhook config and
-# the variables $(CERTIFICATE_NAMESPACE) and $(CERTIFICATE_NAME) will be substituted by kustomize.
-apiVersion: admissionregistration.k8s.io/v1
-kind: MutatingWebhookConfiguration
-metadata:
-  labels:
-    app.kubernetes.io/name: mutatingwebhookconfiguration
-    app.kubernetes.io/instance: mutating-webhook-configuration
-    app.kubernetes.io/component: webhook
-    app.kubernetes.io/created-by: operator
-    app.kubernetes.io/part-of: operator
-    app.kubernetes.io/managed-by: kustomize
-  name: mutating-webhook-configuration
-  annotations:
-    cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME)
----
-apiVersion: admissionregistration.k8s.io/v1
-kind: ValidatingWebhookConfiguration
-metadata:
-  labels:
-    app.kubernetes.io/name: validatingwebhookconfiguration
-    app.kubernetes.io/instance: validating-webhook-configuration
-    app.kubernetes.io/component: webhook
-    app.kubernetes.io/created-by: operator
-    app.kubernetes.io/part-of: operator
-    app.kubernetes.io/managed-by: kustomize
-  name: validating-webhook-configuration
-  annotations:
-    cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME)

config/manager/controller_manager_config.yaml

@@ -1,11 +0,0 @@
-apiVersion: controller-runtime.sigs.k8s.io/v1alpha1
-kind: ControllerManagerConfig
-health:
-  healthProbeBindAddress: :8081
-metrics:
-  bindAddress: 127.0.0.1:8080
-webhook:
-  port: 9443
-leaderElection:
-  leaderElect: true
-  resourceName: 799b98bc.clastix.io

config/manager/kustomization.yaml

@@ -1,16 +0,0 @@
-resources:
-- manager.yaml
-
-generatorOptions:
-  disableNameSuffixHash: true
-
-configMapGenerator:
-- files:
-  - controller_manager_config.yaml
-  name: manager-config
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-images:
-- name: controller
-  newName: clastix/kamaji
-  newTag: v1.0.0

config/manager/manager.yaml

@@ -1,72 +0,0 @@
-apiVersion: v1
-kind: Namespace
-metadata:
-  labels:
-    control-plane: controller-manager
-  name: system
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: controller-manager
-  namespace: system
-  labels:
-    control-plane: controller-manager
-spec:
-  selector:
-    matchLabels:
-      control-plane: controller-manager
-  replicas: 1
-  template:
-    metadata:
-      labels:
-        control-plane: controller-manager
-    spec:
-      securityContext:
-        runAsNonRoot: true
-      containers:
-      - command:
-        - /kamaji
-        args:
-        - manager
-        - --leader-elect
-        - --datastore=kamaji-etcd
-        env:
-          - name: POD_NAMESPACE
-            valueFrom:
-              fieldRef:
-                fieldPath: metadata.namespace
-          - name: SERVICE_ACCOUNT
-            valueFrom:
-              fieldRef:
-                fieldPath: spec.serviceAccountName
-        image: controller:latest
-        imagePullPolicy: Always
-        name: manager
-        ports:
-          - containerPort: 8080
-            name: metrics
-            protocol: TCP
-        securityContext:
-          allowPrivilegeEscalation: false
-        livenessProbe:
-          httpGet:
-            path: /healthz
-            port: 8081
-          initialDelaySeconds: 15
-          periodSeconds: 20
-        readinessProbe:
-          httpGet:
-            path: /readyz
-            port: 8081
-          initialDelaySeconds: 5
-          periodSeconds: 10
-        resources:
-          limits:
-            cpu: 200m
-            memory: 100Mi
-          requests:
-            cpu: 100m
-            memory: 20Mi
-      serviceAccountName: controller-manager
-      terminationGracePeriodSeconds: 10

config/manifests/kustomization.yaml

@@ -1,27 +0,0 @@
-# These resources constitute the fully configured set of manifests
-# used to generate the 'manifests/' directory in a bundle.
-resources:
-- bases/operator.clusterserviceversion.yaml
-- ../default
-- ../samples
-- ../scorecard
-
-# [WEBHOOK] To enable webhooks, uncomment all the sections with [WEBHOOK] prefix.
-# Do NOT uncomment sections with prefix [CERTMANAGER], as OLM does not support cert-manager.
-# These patches remove the unnecessary "cert" volume and its manager container volumeMount.
-#patchesJson6902:
-#- target:
-#    group: apps
-#    version: v1
-#    kind: Deployment
-#    name: controller-manager
-#    namespace: system
-#  patch: |-
-#    # Remove the manager container's "cert" volumeMount, since OLM will create and mount a set of certs.
-#    # Update the indices in this path if adding or removing containers/volumeMounts in the manager's Deployment.
-#    - op: remove
-#      path: /spec/template/spec/containers/1/volumeMounts/0
-#    # Remove the "cert" volume, since OLM will create and mount a set of certs.
-#    # Update the indices in this path if adding or removing volumes in the manager's Deployment.
-#    - op: remove
-#      path: /spec/template/spec/volumes/0

config/metadata.yaml

@@ -1,10 +0,0 @@
-# maps release series of major.minor to cluster-api contract version
-# the contract version may change between minor or major versions, but *not*
-# between patch versions.
-#
-# update this file only when a new major or minor version is released
-apiVersion: clusterctl.cluster.x-k8s.io/v1alpha3
-releaseSeries:
-  - major: 0
-    minor: 3
-    contract: v1beta1

config/prometheus/kustomization.yaml

@@ -1,5 +0,0 @@
-resources:
-- monitor.yaml
-
-configurations:
-- kustomizeconfig.yaml

config/prometheus/kustomizeconfig.yaml

@@ -1,4 +0,0 @@
-varReference:
-  - kind: ServiceMonitor
-    group: monitoring.coreos.com
-    path: spec/namespaceSelector/matchNames

config/prometheus/monitor.yaml

@@ -1,19 +0,0 @@
-# Prometheus Monitor Service (Metrics)
-apiVersion: monitoring.coreos.com/v1
-kind: ServiceMonitor
-metadata:
-  labels:
-    control-plane: controller-manager
-  name: controller-manager-metrics-monitor
-  namespace: system
-spec:
-  endpoints:
-    - path: /metrics
-      port: metrics
-      scheme: http
-  namespaceSelector:
-    matchNames:
-      - $(SERVICE_NAMESPACE)
-  selector:
-    matchLabels:
-      control-plane: controller-manager

config/rbac/auth_proxy_client_clusterrole.yaml

@@ -1,9 +0,0 @@
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: metrics-reader
-rules:
-- nonResourceURLs:
-  - "/metrics"
-  verbs:
-  - get

config/rbac/auth_proxy_role.yaml

@@ -1,17 +0,0 @@
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: proxy-role
-rules:
-- apiGroups:
-  - authentication.k8s.io
-  resources:
-  - tokenreviews
-  verbs:
-  - create
-- apiGroups:
-  - authorization.k8s.io
-  resources:
-  - subjectaccessreviews
-  verbs:
-  - create

config/rbac/auth_proxy_role_binding.yaml

@@ -1,12 +0,0 @@
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: proxy-rolebinding
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: proxy-role
-subjects:
-- kind: ServiceAccount
-  name: controller-manager
-  namespace: system

config/rbac/auth_proxy_service.yaml

@@ -1,15 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  labels:
-    control-plane: controller-manager
-  name: controller-manager-metrics-service
-  namespace: system
-spec:
-  ports:
-  - name: https
-    port: 8443
-    protocol: TCP
-    targetPort: https
-  selector:
-    control-plane: controller-manager

config/rbac/datastore_editor_role.yaml

@@ -1,24 +0,0 @@
-# permissions for end users to edit datastores.
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: datastore-editor-role
-rules:
-- apiGroups:
-  - kamaji.clastix.io
-  resources:
-  - datastores
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch
-- apiGroups:
-  - kamaji.clastix.io
-  resources:
-  - datastores/status
-  verbs:
-  - get

config/rbac/datastore_viewer_role.yaml

@@ -1,20 +0,0 @@
-# permissions for end users to view datastores.
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: datastore-viewer-role
-rules:
-- apiGroups:
-  - kamaji.clastix.io
-  resources:
-  - datastores
-  verbs:
-  - get
-  - list
-  - watch
-- apiGroups:
-  - kamaji.clastix.io
-  resources:
-  - datastores/status
-  verbs:
-  - get