feat(deps): bump github.com/prometheus/client_golang (#899)

Bumps [github.com/prometheus/client_golang](https://github.com/prometheus/client_golang) from 1.22.0 to 1.23.0.
- [Release notes](https://github.com/prometheus/client_golang/releases)
- [Changelog](https://github.com/prometheus/client_golang/blob/v1.23.0/CHANGELOG.md)
- [Commits](https://github.com/prometheus/client_golang/compare/v1.22.0...v1.23.0)

---
updated-dependencies:
- dependency-name: github.com/prometheus/client_golang
  dependency-version: 1.23.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

.dockerignore

@@ -1,4 +0,0 @@
-# More info: https://docs.docker.com/engine/reference/builder/#dockerignore-file
-# Ignore build and test binaries.
-bin/
-testbin/

.github/dependabot.yml

@@ -0,0 +1,23 @@
+version: 2
+updates:
+  - package-ecosystem: gomod
+    directory: /
+    schedule:
+      interval: daily
+    rebase-strategy: disabled
+    commit-message:
+      prefix: "feat(deps)"
+    groups:
+      k8s:
+        patterns:
+          - "k8s.io*"
+      etcd:
+        patterns:
+          - "go.etcd.io/etcd/*"
+  - package-ecosystem: github-actions
+    directory: /
+    schedule:
+      interval: daily
+    rebase-strategy: disabled
+    commit-message:
+      prefix: "chore(ci)"

.github/release-template.md

@@ -0,0 +1,10 @@
+This edge release can be pulled from Docker Hub as follows:
+
+```
+docker pull clastix/kamaji:$TAG
+```
+
+> As from the v1.0.0 release, CLASTIX no longer provides stable release artefacts.
+>
+> Stable release artefacts are offered on a subscription basis by CLASTIX, the main Kamaji project contributor.
+> Learn more from CLASTIX's [Support](https://clastix.io/support/) section.

.github/workflows/ci.yaml

@@ -7,36 +7,45 @@ on:
     branches: [ "*" ]
 
 jobs:
+  test:
+    name: integration
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+      - run: make test
   golangci:
     name: lint
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-go@v3
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
         with:
-          go-version: '1.19'
-          check-latest: true
+          go-version-file: go.mod
       - name: Run golangci-lint
-        uses: golangci/golangci-lint-action@v3.2.0
-        with:
-          version: v1.49.0
-          only-new-issues: false
-          args: --timeout 5m --config .golangci.yml
+        run: make golint
+# TODO(prometherion): enable back once golangci-lint is built from v1.24 rather than v1.23
+#        uses: golangci/golangci-lint-action@v6.5.2
+#        with:
+#          version: v1.62.2
+#          only-new-issues: false
+#          args: --config .golangci.yml
   diff:
     name: diff
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      - uses: actions/setup-go@v3
+      - uses: actions/setup-go@v5
         with:
-          go-version: '1.19'
-          check-latest: true
-      - run: make yaml-installation-file
-      - name: Checking if YAML installer file is not aligned
+          go-version-file: go.mod
+      - run: make manifests
+      - name: Checking if generated manifests are not aligned
         run: if [[ $(git diff | wc -l) -gt 0 ]]; then echo ">>> Untracked generated files have not been committed" && git --no-pager diff && exit 1; fi
-      - name: Checking if YAML installer generated untracked files
+      - name: Checking if missing untracked files for generated manifests
         run: test -z "$(git ls-files --others --exclude-standard 2> /dev/null)"
       - name: Checking if source code is not formatted
         run: test -z "$(git diff 2> /dev/null)"

.github/workflows/docker-ci.yml

@@ -1,99 +0,0 @@
-name: docker-ci
-
-on:
-  push:
-    tags:        
-      - "v*"
-
-jobs:
-  docker-ci:
-    runs-on: ubuntu-22.04
-    steps:
-
-      - name: Checkout
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-
-      - name: Generate build-args
-        id: build-args
-        run: |
-          # Declare vars for internal use
-          VERSION=$(git describe --abbrev=0 --tags)
-          GIT_HEAD_COMMIT=$(git rev-parse --short HEAD)
-          GIT_TAG_COMMIT=$(git rev-parse --short $VERSION)
-          GIT_MODIFIED_1=$(git diff $GIT_HEAD_COMMIT $GIT_TAG_COMMIT --quiet && echo "" || echo ".dev")
-          GIT_MODIFIED_2=$(git diff --quiet && echo "" || echo ".dirty")
-          # Export to GH_ENV
-          echo "GIT_LAST_TAG=$VERSION" >> $GITHUB_ENV
-          echo "GIT_HEAD_COMMIT=$GIT_HEAD_COMMIT" >> $GITHUB_ENV
-          echo "GIT_TAG_COMMIT=$GIT_TAG_COMMIT" >> $GITHUB_ENV
-          echo "GIT_MODIFIED=$(echo "$GIT_MODIFIED_1""$GIT_MODIFIED_2")" >> $GITHUB_ENV
-          echo "GIT_REPO=$(git config --get remote.origin.url)" >> $GITHUB_ENV
-          echo "BUILD_DATE=$(git log -1 --format="%at" | xargs -I{} date -d @{} +%Y-%m-%dT%H:%M:%S)" >> $GITHUB_ENV
-
-      - name: Docker meta
-        id: meta
-        uses: docker/metadata-action@v3
-        with:
-          images: |
-             quay.io/${{ github.repository }}
-             docker.io/${{ github.repository }}
-          tags: |
-            type=semver,pattern={{raw}}
-          flavor: |
-            latest=false
-
-      - name: Set up QEMU
-        id: qemu
-        uses: docker/setup-qemu-action@v1
-        with:
-          platforms: arm64,arm
-
-      - name: Set up Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v1
-        with:
-          install: true
-
-      - name: Inspect builder
-        run: |
-          echo "Name:      ${{ steps.buildx.outputs.name }}"
-          echo "Endpoint:  ${{ steps.buildx.outputs.endpoint }}"
-          echo "Status:    ${{ steps.buildx.outputs.status }}"
-          echo "Flags:     ${{ steps.buildx.outputs.flags }}"
-          echo "Platforms: ${{ steps.buildx.outputs.platforms }}"
-
-      - name: Login to quay.io Container Registry
-        uses: docker/login-action@v1 
-        with:
-          registry: quay.io
-          username: ${{ secrets.QUAY_IO_USERNAME }}
-          password: ${{ secrets.QUAY_IO_TOKEN }}
-
-      - name: Login to docker.io Container Registry
-        uses: docker/login-action@v1 
-        with:
-          registry: docker.io
-          username: ${{ secrets.DOCKER_IO_USERNAME }}
-          password: ${{ secrets.DOCKER_IO_TOKEN }}
-
-      - name: Build and push
-        id: build-release
-        uses: docker/build-push-action@v2
-        with:
-          file: Dockerfile
-          context: .
-          platforms: linux/amd64,linux/arm64,linux/arm
-          push: true
-          tags: ${{ steps.meta.outputs.tags }}
-          build-args: |
-            GIT_LAST_TAG=${{ env.GIT_LAST_TAG }}
-            GIT_HEAD_COMMIT=${{ env.GIT_HEAD_COMMIT }}
-            GIT_TAG_COMMIT=${{ env.GIT_TAG_COMMIT }}
-            GIT_MODIFIED=${{ env.GIT_MODIFIED }}
-            GIT_REPO=${{ env.GIT_REPO }}
-            BUILD_DATE=${{ env.BUILD_DATE }}
-
-      - name: Image digest
-        run: echo ${{ steps.build-release.outputs.digest }}

.github/workflows/e2e.yaml

@@ -6,40 +6,46 @@ on:
     paths:
       - '.github/workflows/e2e.yml'
       - 'api/**'
+      - 'charts/kamaji/**'
       - 'controllers/**'
       - 'e2e/*'
-      - 'Dockerfile'
+      - '.ko.yaml'
       - 'go.*'
       - 'main.go'
       - 'Makefile'
       - 'internal/**'
+      - 'cmd/**'
   pull_request:
     branches: [ "*" ]
     paths:
       - '.github/workflows/e2e.yml'
       - 'api/**'
+      - 'charts/kamaji/**'
       - 'controllers/**'
       - 'e2e/*'
-      - 'Dockerfile'
+      - '.ko.yaml'
       - 'go.*'
       - 'main.go'
       - 'Makefile'
       - 'internal/**'
+      - 'cmd/**'
 
 jobs:
   kind:
     name: Kubernetes
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      - uses: actions/setup-go@v3
+      - uses: actions/setup-go@v5
         with:
-          go-version: '1.19'
+          go-version: '1.22'
           check-latest: true
       - run: |
           sudo apt-get update
           sudo apt-get install -y golang-cfssl
+          sudo swapoff -a
+          sudo modprobe br_netfilter
       - name: e2e testing
         run: make e2e

.github/workflows/helm.yaml

@@ -2,8 +2,7 @@ name: Helm Chart
 
 on:
   push:
-    branches: [ "*" ]
-    tags: [ "helm-v*" ]
+    branches: [ "master" ]
   pull_request:
     branches: [ "*" ]
 
@@ -12,7 +11,7 @@ jobs:
     name: diff
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - run: make -C charts/kamaji docs
@@ -21,17 +20,22 @@ jobs:
   lint:
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v2
-      - uses: azure/setup-helm@v1
+      - uses: actions/checkout@v4
+      - uses: azure/setup-helm@v4
         with:
           version: 3.3.4
+      - name: Building dependencies
+        run: |-
+          helm repo add clastix https://clastix.github.io/charts 
+          helm dependency build ./charts/kamaji
       - name: Linting Chart
         run: helm lint ./charts/kamaji
   release:
-    if: startsWith(github.ref, 'refs/tags/helm-v')
+    if: github.event_name == 'push' && github.ref == 'refs/heads/master'
+    needs: [ "lint", "diff" ]
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Publish Helm chart
         uses: stefanprodan/helm-gh-pages@master
         with:

.github/workflows/ko-build.yml

@@ -0,0 +1,31 @@
+name: Container image build
+
+on:
+  push:
+    tags:
+      - edge-*
+      - v*
+    branches:
+      - master
+
+jobs:
+  ko:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+      - name: "ko: install"
+        run: make ko
+      - name: "ko: login to quay.io container registry"
+        run: ./bin/ko login quay.io -u ${{ secrets.QUAY_IO_USERNAME }} -p ${{ secrets.QUAY_IO_TOKEN }}
+      - name: "ko: login to docker.io container registry"
+        run: ./bin/ko login docker.io -u ${{ secrets.DOCKER_IO_USERNAME }} -p ${{ secrets.DOCKER_IO_TOKEN }}
+      - name: "ko: build and push tag"
+        run: make VERSION=${{ github.ref_name }} KO_LOCAL=false KO_PUSH=true build
+        if: startsWith(github.ref, 'refs/tags/v') || startsWith(github.ref, 'refs/tags/edge-')
+      - name: "ko: build and push latest"
+        run: make VERSION=latest KO_LOCAL=false KO_PUSH=true build

.github/workflows/pr.yaml

@@ -0,0 +1,23 @@
+name: Check PR Title
+
+on:
+  pull_request:
+    types: [opened, edited, reopened, synchronize]
+
+jobs:
+  semantic-pr-title:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: amannn/action-semantic-pull-request@v5
+        with:
+          types: |
+            feat
+            fix
+            chore
+            docs
+            style
+            refactor
+            perf
+            test
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/release.yml

@@ -0,0 +1,52 @@
+name: Weekly Edge Release
+
+on:
+  schedule:
+    - cron: '0 7 * * 1'  # Every Monday at 9 AM CET
+  workflow_dispatch:
+
+permissions:
+  contents: write
+
+jobs:
+  release:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: generating date metadata
+        id: date
+        run: |
+          CURRENT_DATE=$(date -u +'%Y-%m-%d')
+          YY=$(date -u +'%y')
+          M=$(date -u +'%_m' | sed 's/ //g')
+          FIRST_OF_MONTH=$(date -u -d "$CURRENT_DATE" +%Y-%m-01)
+          WEEK_NUM=$(( (($(date -u +%s) - $(date -u -d "$FIRST_OF_MONTH" +%s)) / 86400 + $(date -u -d "$FIRST_OF_MONTH" +%u) - 1) / 7 + 1 ))
+
+          echo "yy=$YY" >> $GITHUB_OUTPUT
+          echo "month=$M" >> $GITHUB_OUTPUT
+          echo "week=$WEEK_NUM" >> $GITHUB_OUTPUT
+          echo "date=$CURRENT_DATE" >> $GITHUB_OUTPUT
+      - name: generating tag metadata
+        id: tag
+        run: |
+          TAG="edge-${{ steps.date.outputs.yy }}.${{ steps.date.outputs.month }}.${{ steps.date.outputs.week }}"
+          echo "tag=$TAG" >> $GITHUB_OUTPUT
+      - name: generate release notes from template
+        run: |
+          export TAG="${{ steps.tag.outputs.tag }}"
+          envsubst < .github/release-template.md > release-notes.md
+      - name: create and push git tag
+        run: |
+          git config user.name "github-actions"
+          git config user.email "github-actions@github.com"
+          git tag "${{ steps.tag.outputs.tag }}"
+          git push origin "${{ steps.tag.outputs.tag }}"
+      - name: create GitHub release
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          gh release create "${{ steps.tag.outputs.tag }}" \
+            --title "${{ steps.tag.outputs.tag }}" \
+            --notes-file release-notes.md

.gitignore

@@ -24,10 +24,17 @@ bin
 *~
 .vscode
 
+# Tilt files.
+.tiltbuild
+
 **/*.kubeconfig
 **/*.crt
 **/*.key
 **/*.pem
 **/*.csr
-**/server-csr.json
 .DS_Store
+
+**/server-csr.json
+!deploy/kine/mysql/server-csr.json
+!deploy/kine/nats/server-csr.json
+charts/kamaji/charts

.golangci.yml

@@ -1,48 +1,76 @@
-linters-settings:
-  gci:
-    sections:
-      - standard
-      - default
-      - prefix(github.com/clastix/kamaji)
-  goheader:
-    template: |-
-      Copyright 2022 Clastix Labs
-      SPDX-License-Identifier: Apache-2.0
-
+version: "2"
 linters:
+  default: all
   disable:
-    - wrapcheck
-    - gomnd
-    - scopelint
-    - varnamelen
-    - testpackage
-    - tagliatelle
-    - paralleltest
-    - ireturn
-    - goerr113
-    - gochecknoglobals
-    - exhaustivestruct
-    - wsl
-    - exhaustive
-    - nosprintfhostport
-    - nonamedreturns
-    - interfacebloat
-    - exhaustruct
-    - lll
-    - gosec
-    - gomoddirectives
-    - godox
-    - gochecknoinits
-    - funlen
-    - dupl
     - cyclop
-    # deprecated linters
-    - deadcode
-    - golint
-    - interfacer
-    - structcheck
-    - varcheck
-    - nosnakecase
-    - ifshort
-    - maligned
-  enable-all: true
+    - depguard
+    - dupl
+    - err113
+    - exhaustive
+    - exhaustruct
+    - funlen
+    - gochecknoglobals
+    - gochecknoinits
+    - gocognit
+    - godox
+    - gomoddirectives
+    - gosec
+    - interfacebloat
+    - ireturn
+    - lll
+    - mnd
+    - nestif
+    - nonamedreturns
+    - nosprintfhostport
+    - paralleltest
+    - perfsprint
+    - tagliatelle
+    - testpackage
+    - varnamelen
+    - wrapcheck
+    - wsl
+  settings:
+    staticcheck:
+      checks:
+        - all
+        - -QF1008
+    goheader:
+      template: |-
+        Copyright 2022 Clastix Labs
+        SPDX-License-Identifier: Apache-2.0
+    revive:
+      rules:
+        - name: dot-imports
+          arguments:
+            - allowedPackages:
+                - github.com/onsi/ginkgo/v2
+                - github.com/onsi/gomega
+  exclusions:
+    generated: lax
+    presets:
+      - comments
+      - common-false-positives
+      - legacy
+      - std-error-handling
+    paths:
+      - third_party$
+      - builtin$
+      - examples$
+formatters:
+  enable:
+    - gci
+    - gofmt
+    - gofumpt
+    - goimports
+  settings:
+    gci:
+      sections:
+        - standard
+        - default
+        - prefix(github.com/clastix/kamaji/)
+  exclusions:
+    generated: lax
+    paths:
+      - third_party$
+      - builtin$
+      - examples$

.ko.yaml

@@ -0,0 +1,9 @@
+defaultPlatforms:
+  - linux/arm64
+  - linux/amd64
+  - linux/arm
+builds:
+  - id: kamaji
+    main: .
+    ldflags:
+      - '{{ if index .Env "LD_FLAGS" }}{{ .Env.LD_FLAGS }}{{ end }}'

ADOPTERS.md

@@ -0,0 +1,40 @@
+# Adopters
+
+This is a list of companies that have adopted Kamaji.
+Feel free to open a Pull-Request to get yours listed.
+
+### Adopter list (alphabetically)
+
+| Type | Name | Since | Website | Use-Case |
+|:-|:-|:-|:-|:-|
+| Vendor | Aknostic | 2023 | [link](https://aknostic.com) | Aknostic is a cloud-native consultancy company using Kamaji to build a Kubernetes based PaaS. |
+| R&D | Aruba | 2024 | [link](https://www.aruba.it/home.aspx) | Aruba Cloud is an Italian Cloud Service Provider evaluating Kamaji to build and offer [Managed Kubernetes Service](https://my.arubacloud.com). |
+| Vendor | CBWS | 2025 | [link](https://cbws.nl) | CBWS is an European Cloud Provider using Kamaji to build and offer their [Managed Kubernetes Service](https://cbws.nl/cloud/kubernetes/). |
+| Vendor | DCloud | 2024 | [link](https://dcloud.co.id) | DCloud is an Indonesian Cloud Provider using Kamaji to build and offer [Managed Kubernetes Service](https://dcloud.co.id/dkubes.html). |
+| Vendor | Dinova | 2025 | [link](https://dinova.one/) | Dinova is an Italian cloud services provider that integrates Kamaji in its datacenters to offer fully managed Kubernetes clusters. |
+| End-user | KINX | 2024 | [link](https://kinx.net/?lang=en) | KINX is an Internet infrastructure service provider and will use kamaji for its new [Managed Kubernetes Service](https://kinx.net/service/cloud/kubernetes/intro/?lang=en). |
+| Vendor | Netsons | 2023 | [link](https://www.netsons.com) | Netsons is an Italian hosting and cloud provider and uses Kamaji in its [Managed Kubernetes](https://www.netsons.com/kubernetes) offering. |
+| Vendor | NVIDIA | 2024 | [link](https://github.com/NVIDIA/doca-platform) | DOCA Platform Framework manages provisioning and service orchestration for NVIDIA Bluefield DPUs. |
+| R&D | Orange | 2024 | [link](https://gitlab.com/Orange-OpenSource/kanod) | Orange is a French telecommunications company using Kamaji for experimental research purpose, with Kanod research solution. |
+| Vendor | Platform9 | 2024 | [link](https://elasticmachinepool.com) | Platform9 uses Kamaji in its offering - Elastic Machine Pool, which is a tool for optimizing the cost of running kubernetes clusters in EKS. |
+| Vendor | Qumulus | 2024 | [link](https://www.qumulus.io) | Qumulus is a cloud provider and plans to use Kamaji for it's hosted Kubernetes service |
+| End-user | sevensphere | 2023 | [link](https://www.sevensphere.io) | Sevensphere provides consulting services for end-user companies / cloud providers and uses Kamaji for designing cloud/on-premises Kubernetes-as-a-Service platform. |
+| End-user | Sicuro Tech Lab | 2024 | [link](https://sicurotechlab.it/) | Sicuro Tech Lab offers cloud infrastructure for Web Agencies and uses kamaji to provide managed k8s services. |
+| Vendor | Sovereign Cloud Stack | 2024 | [link](https://sovereigncloudstack.org) | Sovereign Cloud Stack develops a standardized cloud platform and uses Kamaji in there Kubernetes-as-a-Service reference implementation |
+| R&D | TIM | 2024 | [link](https://www.gruppotim.it) | TIM is an Italian telecommunications company using Kamaji for experimental research and development purposes. |
+| End-user | Tinext Cloud | 2025 | [link](https://cloud.tinext.com) | Tinex Cloud is a Swiss cloud service provider using Kamaji to build their Managed Kubernetes Services. |
+| Vendor | Ænix | 2023 | [link](https://aenix.io/) | Ænix provides consulting services for cloud providers and uses Kamaji for running Kubernetes-as-a-Service in free PaaS platform [Cozystack](https://cozystack.io). |
+| End-user | Rackspace | 2024 | [link](https://spot.rackspace.com/) | Rackspace Spot uses Kamaji to manage our instances, offering fully-managed kubernetes infrastructure, auctioned in an open market. |
+| R&D | IONOS Cloud | 2024 | [link](https://cloud.ionos.com/) | IONOS Cloud is a German Cloud Provider evaluating Kamaji for its [Managed Kubernetes platform](https://cloud.ionos.com/managed/kubernetes). |
+| Vendor | OVHCloud | 2025 | [link](https://www.ovhcloud.com/) | OVHCloud is a European Cloud Provider that will use Kamaji for its Managed Kubernetes Service offer. |
+| Vendor | WOBCOM GmbH | 2024 | [link](https://www.wobcom.de/) | WOBCOM provides an [**Open Digital Platform**](https://www.wobcom.de/geschaeftskunden/odp/) solution for Smart Cities, which is provided for customers in a Managed Kubernetes provided by Kamaji. |
+
+### Adopter Types
+
+**End-user**: The organization runs Kamaji in production in some way.
+
+**Integration**: The organization has a product that integrates with Kamaji, but does not contain Kamaji.
+
+**Vendor**: The organization packages Kamaji in their product and sells it as part of their product.
+
+**R&D**: Company that exploring innovative technologies  and solutions for research and development purposes.

Dockerfile

@@ -1,40 +0,0 @@
-# Build the manager binary
-FROM golang:1.19 as builder
-
-WORKDIR /workspace
-# Copy the Go Modules manifests
-COPY go.mod go.mod
-COPY go.sum go.sum
-# cache deps before building and copying source so that we don't need to re-download as much
-# and so that source changes don't invalidate our downloaded layer
-RUN go mod download
-
-# Copy the go source
-COPY main.go main.go
-COPY cmd/ cmd/
-COPY api/ api/
-COPY controllers/ controllers/
-COPY internal/ internal/
-COPY indexers/ indexers/
-
-# Build
-ARG TARGETARCH
-ARG GIT_HEAD_COMMIT
-ARG GIT_TAG_COMMIT
-ARG GIT_LAST_TAG
-ARG GIT_MODIFIED
-ARG GIT_REPO
-ARG BUILD_DATE
-
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=$TARGETARCH go build \
-    -ldflags "-X github.com/clastix/kamaji/internal.GitRepo=$GIT_REPO -X github.com/clastix/kamaji/internal.GitTag=$GIT_LAST_TAG -X github.com/clastix/kamaji/internal.GitCommit=$GIT_HEAD_COMMIT -X github.com/clastix/kamaji/internal.GitDirty=$GIT_MODIFIED -X github.com/clastix/kamaji/internal.BuildTime=$BUILD_DATE" \
-    -a -o kamaji main.go
-
-# Use distroless as minimal base image to package the manager binary
-# Refer to https://github.com/GoogleContainerTools/distroless for more details
-FROM gcr.io/distroless/static:nonroot
-WORKDIR /
-COPY --from=builder /workspace/kamaji .
-USER 65532:65532
-
-ENTRYPOINT ["/kamaji"]

Makefile

@@ -3,40 +3,24 @@
 # To re-generate a bundle for another specific version without changing the standard setup, you can:
 # - use the VERSION as arg of the bundle target (e.g make bundle VERSION=0.0.2)
 # - use environment variables to overwrite this value (e.g export VERSION=0.0.2)
-VERSION ?= 0.3.2
+VERSION ?= $(or $(shell git describe --abbrev=0 --tags 2>/dev/null),$(GIT_HEAD_COMMIT))
 
-# CHANNELS define the bundle channels used in the bundle.
-# Add a new line here if you would like to change its default config. (E.g CHANNELS = "candidate,fast,stable")
-# To re-generate a bundle for other specific channels without changing the standard setup, you can:
-# - use the CHANNELS as arg of the bundle target (e.g make bundle CHANNELS=candidate,fast,stable)
-# - use environment variables to overwrite this value (e.g export CHANNELS="candidate,fast,stable")
-ifneq ($(origin CHANNELS), undefined)
-BUNDLE_CHANNELS := --channels=$(CHANNELS)
-endif
+# ENVTEST_K8S_VERSION specifies the Kubernetes version to be used 
+# during testing with the envtest environment. This ensures that 
+# the tests run against the correct API and behavior for the 
+# specific Kubernetes release being targeted (v1.31.0 in this case).
+ENVTEST_K8S_VERSION = 1.31.0
 
-# DEFAULT_CHANNEL defines the default channel used in the bundle.
-# Add a new line here if you would like to change its default config. (E.g DEFAULT_CHANNEL = "stable")
-# To re-generate a bundle for any other default channel without changing the default setup, you can:
-# - use the DEFAULT_CHANNEL as arg of the bundle target (e.g make bundle DEFAULT_CHANNEL=stable)
-# - use environment variables to overwrite this value (e.g export DEFAULT_CHANNEL="stable")
-ifneq ($(origin DEFAULT_CHANNEL), undefined)
-BUNDLE_DEFAULT_CHANNEL := --default-channel=$(DEFAULT_CHANNEL)
-endif
-BUNDLE_METADATA_OPTS ?= $(BUNDLE_CHANNELS) $(BUNDLE_DEFAULT_CHANNEL)
-
-# IMAGE_TAG_BASE defines the docker.io namespace and part of the image name for remote images.
-# This variable is used to construct full image tags for bundle and catalog images.
-#
-# For example, running 'make bundle-build bundle-push catalog-build catalog-push' will build and push both
-# clastix.io/operator-bundle:$VERSION and clastix.io/operator-catalog:$VERSION.
-IMAGE_TAG_BASE ?= clastix.io/operator
-
-# BUNDLE_IMG defines the image:tag used for the bundle.
-# You can use it as an arg. (E.g make bundle-build BUNDLE_IMG=<some-registry>/<project-name-bundle>:<tag>)
-BUNDLE_IMG ?= $(IMAGE_TAG_BASE)-bundle:v$(VERSION)
+# ENVTEST_VERSION defines the version of the setup-envtest binary 
+# used to manage and download the Kubernetes binaries (like etcd, 
+# kube-apiserver, and kubectl) required for testing. This version 
+# ensures compatibility with the selected Kubernetes version and 
+# must align closely with recent releases (release-0.19 is chosen here).
+# Mismatches between these versions could result in compatibility issues.
+ENVTEST_VERSION ?= release-0.19
 
 # Image URL to use all building/pushing image targets
-IMG ?= clastix/kamaji:v$(VERSION)
+CONTAINER_REPOSITORY ?= docker.io/clastix/kamaji
 
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
 ifeq (,$(shell go env GOBIN))
@@ -50,6 +34,22 @@ endif
 SHELL = /usr/bin/env bash -o pipefail
 .SHELLFLAGS = -ec
 
+## Location to install dependencies to
+LOCALBIN ?= $(shell pwd)/bin
+$(LOCALBIN):
+	mkdir -p $(LOCALBIN)
+
+## Tool Binaries
+APIDOCS_GEN    ?= $(LOCALBIN)/crdoc
+CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen
+GINKGO         ?= $(LOCALBIN)/ginkgo
+GOLANGCI_LINT  ?= $(LOCALBIN)/golangci-lint
+HELM           ?= $(LOCALBIN)/helm
+KIND           ?= $(LOCALBIN)/kind
+KO             ?= $(LOCALBIN)/ko
+YQ             ?= $(LOCALBIN)/yq
+ENVTEST        ?= $(LOCALBIN)/setup-envtest
+
 all: build
 
 ##@ General
@@ -70,39 +70,70 @@ help: ## Display this help.
 
 ##@ Binary
 
+.PHONY: ko
+ko: $(KO) ## Download ko locally if necessary.
+$(KO): $(LOCALBIN)
+	test -s $(LOCALBIN)/ko || GOBIN=$(LOCALBIN) CGO_ENABLED=0 go install -ldflags="-s -w" github.com/google/ko@v0.14.1
+
+.PHONY: yq
+yq: $(YQ) ## Download yq locally if necessary.
+$(YQ): $(LOCALBIN)
+	test -s $(LOCALBIN)/yq || GOBIN=$(LOCALBIN) CGO_ENABLED=0 go install -ldflags="-s -w" github.com/mikefarah/yq/v4@v4.44.2
+
 .PHONY: helm
-HELM = $(shell pwd)/bin/helm
-helm: ## Download helm locally if necessary.
-	$(call go-install-tool,$(HELM),helm.sh/helm/v3/cmd/helm@v3.9.0)
+helm: $(HELM) ## Download helm locally if necessary.
+$(HELM): $(LOCALBIN)
+	test -s $(LOCALBIN)/helm || GOBIN=$(LOCALBIN) CGO_ENABLED=0 go install -ldflags="-s -w" helm.sh/helm/v3/cmd/helm@v3.9.0
 
-GINKGO = $(shell pwd)/bin/ginkgo
-ginkgo: ## Download ginkgo locally if necessary.
-	$(call go-install-tool,$(GINKGO),github.com/onsi/ginkgo/v2/ginkgo@v2.6.0)
+.PHONY: ginkgo
+ginkgo: $(GINKGO) ## Download ginkgo locally if necessary.
+$(GINKGO): $(LOCALBIN)
+	test -s $(LOCALBIN)/ginkgo || GOBIN=$(LOCALBIN) CGO_ENABLED=0 go install -ldflags="-s -w" github.com/onsi/ginkgo/v2/ginkgo
 
-KIND = $(shell pwd)/bin/kind
-kind: ## Download kind locally if necessary.
-	$(call go-install-tool,$(KIND),sigs.k8s.io/kind/cmd/kind@v0.14.0)
+.PHONY: kind
+kind: $(KIND) ## Download kind locally if necessary.
+$(KIND): $(LOCALBIN)
+	test -s $(LOCALBIN)/kind || GOBIN=$(LOCALBIN) CGO_ENABLED=0 go install -ldflags="-s -w" sigs.k8s.io/kind/cmd/kind@v0.14.0
 
-CONTROLLER_GEN = $(shell pwd)/bin/controller-gen
-controller-gen: ## Download controller-gen locally if necessary.
-	$(call go-install-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.11.4)
+.PHONY: controller-gen
+controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessary.
+$(CONTROLLER_GEN): $(LOCALBIN)
+	test -s $(LOCALBIN)/controller-gen || GOBIN=$(LOCALBIN) CGO_ENABLED=0 go install -ldflags="-s -w" sigs.k8s.io/controller-tools/cmd/controller-gen@v0.16.1
 
-GOLANGCI_LINT = $(shell pwd)/bin/golangci-lint
-golangci-lint: ## Download golangci-lint locally if necessary.
-	$(call go-install-tool,$(GOLANGCI_LINT),github.com/golangci/golangci-lint/cmd/golangci-lint@v1.49.0)
+.PHONY: golangci-lint
+golangci-lint: $(GOLANGCI_LINT) ## Download golangci-lint locally if necessary.
+$(GOLANGCI_LINT): $(LOCALBIN)
+	test -s $(LOCALBIN)/golangci-lint || GOBIN=$(LOCALBIN) CGO_ENABLED=0 go install -ldflags="-s -w" github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.0.2
 
-KUSTOMIZE = $(shell pwd)/bin/kustomize
-kustomize: ## Download kustomize locally if necessary.
-	$(call install-kustomize,$(KUSTOMIZE),3.8.7)
+.PHONY: apidocs-gen
+apidocs-gen: $(APIDOCS_GEN)  ## Download crdoc locally if necessary.
+$(APIDOCS_GEN): $(LOCALBIN)
+	test -s $(LOCALBIN)/crdoc || GOBIN=$(LOCALBIN) CGO_ENABLED=0 go install -ldflags="-s -w" fybrik.io/crdoc@latest
 
-APIDOCS_GEN = $(shell pwd)/bin/crdoc
-apidocs-gen: ## Download crdoc locally if necessary.
-	$(call go-install-tool,$(APIDOCS_GEN),fybrik.io/crdoc@latest)
+.PHONY: envtest
+envtest: $(ENVTEST) ## Download envtest-setup locally if necessary.
+$(ENVTEST): $(LOCALBIN)
+	test -s $(LOCALBIN)/setup-envtest || GOBIN=$(LOCALBIN) CGO_ENABLED=0 go install -ldflags="-s -w" sigs.k8s.io/controller-runtime/tools/setup-envtest@$(ENVTEST_VERSION)
 
 ##@ Development
 
-manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects.
-	$(CONTROLLER_GEN) rbac:roleName=manager-role crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases
+rbac: controller-gen yq
+	$(CONTROLLER_GEN) rbac:roleName=manager-role paths="./..." output:stdout | $(YQ) '.rules' > ./charts/kamaji/controller-gen/clusterrole.yaml
+
+webhook: controller-gen yq
+	$(CONTROLLER_GEN) webhook paths="./..." output:stdout | $(YQ) 'select(documentIndex == 0) | .webhooks' > ./charts/kamaji/controller-gen/mutating-webhook.yaml
+	$(CONTROLLER_GEN) webhook paths="./..." output:stdout | $(YQ) 'select(documentIndex == 1) | .webhooks' > ./charts/kamaji/controller-gen/validating-webhook.yaml
+	$(YQ) -i 'map(.clientConfig.service.name |= "{{ include \"kamaji.webhookServiceName\" . }}")' ./charts/kamaji/controller-gen/mutating-webhook.yaml
+	$(YQ) -i 'map(.clientConfig.service.namespace |= "{{ .Release.Namespace }}")' ./charts/kamaji/controller-gen/mutating-webhook.yaml
+	$(YQ) -i 'map(.clientConfig.service.name |= "{{ include \"kamaji.webhookServiceName\" . }}")' ./charts/kamaji/controller-gen/validating-webhook.yaml
+	$(YQ) -i 'map(.clientConfig.service.namespace |= "{{ .Release.Namespace }}")' ./charts/kamaji/controller-gen/validating-webhook.yaml
+
+crds: controller-gen yq
+	$(CONTROLLER_GEN) crd webhook paths="./..." output:stdout | $(YQ) 'select(documentIndex == 0)' > ./charts/kamaji/crds/kamaji.clastix.io_datastores.yaml
+	$(CONTROLLER_GEN) crd webhook paths="./..." output:stdout | $(YQ) 'select(documentIndex == 1)' > ./charts/kamaji/crds/kamaji.clastix.io_tenantcontrolplanes.yaml
+	$(YQ) -i '. *n load("./charts/kamaji/controller-gen/crd-conversion.yaml")' ./charts/kamaji/crds/kamaji.clastix.io_tenantcontrolplanes.yaml
+
+manifests: rbac webhook crds ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects.
 
 generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations.
 	$(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..."
@@ -110,8 +141,13 @@ generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and
 golint: golangci-lint ## Linting the code according to the styling guide.
 	$(GOLANGCI_LINT) run -c .golangci.yml
 
-test:
-	go test ./... -coverprofile cover.out
+## Run unit tests (all tests except E2E).
+.PHONY: test
+test: envtest ginkgo
+	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" $(GINKGO) -r -v -coverprofile cover.out --trace \
+		./api/... \
+		./cmd/... \
+		./internal/... \
 
 _datastore-mysql:
 	$(MAKE) NAME=$(NAME) -C deploy/kine/mysql mariadb
@@ -132,7 +168,11 @@ datastore-postgres:
 	$(MAKE) NAME=gold _datastore-postgres
 
 _datastore-etcd:
-	$(HELM) upgrade --install etcd-$(NAME) clastix/kamaji-etcd --create-namespace -n etcd-system --set datastore.enabled=true
+	$(HELM) upgrade --install etcd-$(NAME) clastix/kamaji-etcd --create-namespace -n etcd-system --set datastore.enabled=true --set fullnameOverride=etcd-$(NAME)
+
+_datastore-nats:
+	$(MAKE) NAME=$(NAME) NAMESPACE=nats-system -C deploy/kine/nats nats
+	kubectl apply -f $(shell pwd)/config/samples/kamaji_v1alpha1_datastore_nats_$(NAME).yaml
 
 datastore-etcd: helm
 	$(HELM) repo add clastix https://clastix.github.io/charts
@@ -141,161 +181,84 @@ datastore-etcd: helm
 	$(MAKE) NAME=silver _datastore-etcd
 	$(MAKE) NAME=gold _datastore-etcd
 
-datastores: datastore-mysql datastore-etcd datastore-postgres ## Install all Kamaji DataStores with multiple drivers, and different tiers.
+datastore-nats: helm
+	$(HELM) repo add nats https://nats-io.github.io/k8s/helm/charts/
+	$(HELM) repo update
+	$(MAKE) NAME=bronze _datastore-nats
+	$(MAKE) NAME=silver _datastore-nats
+	$(MAKE) NAME=gold _datastore-nats
+	$(MAKE) NAME=notls _datastore-nats
+
+datastores: datastore-mysql datastore-etcd datastore-postgres datastore-nats ## Install all Kamaji DataStores with multiple drivers, and different tiers.
 
 ##@ Build
 
 # Get information about git current status
 GIT_HEAD_COMMIT ?= $$(git rev-parse --short HEAD)
-GIT_TAG_COMMIT  ?= $$(git rev-parse --short v$(VERSION))
+GIT_TAG_COMMIT  ?= $$(git rev-parse --short $(VERSION))
 GIT_MODIFIED_1  ?= $$(git diff $(GIT_HEAD_COMMIT) $(GIT_TAG_COMMIT) --quiet && echo "" || echo ".dev")
 GIT_MODIFIED_2  ?= $$(git diff --quiet && echo "" || echo ".dirty")
 GIT_MODIFIED    ?= $$(echo "$(GIT_MODIFIED_1)$(GIT_MODIFIED_2)")
 GIT_REPO        ?= $$(git config --get remote.origin.url)
 BUILD_DATE      ?= $$(git log -1 --format="%at" | xargs -I{} date -d @{} +%Y-%m-%dT%H:%M:%S)
 
-build: generate fmt vet ## Build manager binary.
-	go build -o bin/manager main.go
+LD_FLAGS ?= "-X github.com/clastix/kamaji/internal.GitCommit=$(GIT_HEAD_COMMIT) \
+             -X github.com/clastix/kamaji/internal.GitTag=$(VERSION) \
+             -X github.com/clastix/kamaji/internal.GitDirty=$(GIT_MODIFIED) \
+             -X github.com/clastix/kamaji/internal.BuildTime=$(BUILD_DATE) \
+             -X github.com/clastix/kamaji/internal.GitRepo=$(GIT_REPO)"
 
-run: manifests generate fmt vet ## Run a controller from your host.
+KO_PUSH ?= false
+KO_LOCAL ?= true
+
+run: manifests generate ## Run a controller from your host.
 	go run ./main.go
 
-docker-build: ## Build docker image with the manager.
-	docker build -t ${IMG} . --build-arg GIT_HEAD_COMMIT=$(GIT_HEAD_COMMIT) \
-		--build-arg GIT_TAG_COMMIT=$(GIT_TAG_COMMIT) \
-		--build-arg GIT_MODIFIED=$(GIT_MODIFIED) \
-		--build-arg GIT_REPO=$(GIT_REPO) \
-		--build-arg GIT_LAST_TAG=$(VERSION) \
-		--build-arg BUILD_DATE=$(BUILD_DATE)
+build: $(KO)
+	LD_FLAGS=$(LD_FLAGS) \
+	KOCACHE=/tmp/ko-cache KO_DOCKER_REPO=${CONTAINER_REPOSITORY} \
+	$(KO) build ./ --bare --tags=$(VERSION) --local=$(KO_LOCAL) --push=$(KO_PUSH)
 
-docker-push: ## Push docker image with the manager.
-	docker push ${IMG}
-
-##@ Deployment
+##@ Development
 
 metallb:
-	kubectl apply -f https://raw.githubusercontent.com/metallb/metallb/v0.13.7/config/manifests/metallb-native.yaml
-	kubectl apply -f https://kind.sigs.k8s.io/examples/loadbalancer/metallb-config.yaml
-	echo ""
-	docker network inspect -f '{{.IPAM.Config}}' kind
+	kubectl apply -f "https://raw.githubusercontent.com/metallb/metallb/$$(curl "https://api.github.com/repos/metallb/metallb/releases/latest" | jq -r ".tag_name")/config/manifests/metallb-native.yaml"
+	kubectl wait pods -n metallb-system -l app=metallb,component=controller --for=condition=Ready --timeout=10m
+	kubectl wait pods -n metallb-system -l app=metallb,component=speaker --for=condition=Ready --timeout=2m
+	cat hack/metallb.yaml | sed -E "s|172.19|$$(docker network inspect -f '{{range .IPAM.Config}}{{.Gateway}}{{end}}' kind | sed -E 's|^([0-9]+\.[0-9]+)\..*$$|\1|g')|g" | kubectl apply -f -
 
 cert-manager:
-	kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.10.1/cert-manager.yaml
+	$(HELM) repo add jetstack https://charts.jetstack.io
+	$(HELM) upgrade --install cert-manager jetstack/cert-manager --namespace certmanager-system --create-namespace --set "installCRDs=true"
 
-dev: generate manifests uninstall install rbac ## Full installation for development purposes
-	go fmt ./...
-
-load: docker-build kind
-	$(KIND) load docker-image --name kamaji ${IMG}
-
-rbac: manifests kustomize ## Install RBAC into the K8s cluster specified in ~/.kube/config.
-	$(KUSTOMIZE) build config/rbac | kubectl apply -f -
-
-install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config.
-	$(KUSTOMIZE) build config/crd | kubectl apply -f -
-
-uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config.
-	$(KUSTOMIZE) build config/crd | kubectl delete -f -
-
-deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config.
-	cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
-	$(KUSTOMIZE) build config/default | kubectl apply -f -
-
-undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config.
-	$(KUSTOMIZE) build config/default | kubectl delete -f -
-
-yaml-installation-file: manifests kustomize ## Create yaml installation file
-	$(KUSTOMIZE) build config/default > config/install.yaml
-
-.PHONY: bundle
-bundle: manifests kustomize ## Generate bundle manifests and metadata, then validate generated files.
-	operator-sdk generate kustomize manifests -q
-	cd config/manager && $(KUSTOMIZE) edit set image controller=$(IMG)
-	$(KUSTOMIZE) build config/manifests | operator-sdk generate bundle -q --overwrite --version $(VERSION) $(BUNDLE_METADATA_OPTS)
-	operator-sdk bundle validate ./bundle
-
-.PHONY: bundle-build
-bundle-build: ## Build the bundle image.
-	docker build -f bundle.Dockerfile -t $(BUNDLE_IMG) .
-
-.PHONY: bundle-push
-bundle-push: ## Push the bundle image.
-	$(MAKE) docker-push IMG=$(BUNDLE_IMG)
-
-.PHONY: opm
-OPM = ./bin/opm
-opm: ## Download opm locally if necessary.
-ifeq (,$(wildcard $(OPM)))
-ifeq (,$(shell which opm 2>/dev/null))
-	@{ \
-	set -e ;\
-	mkdir -p $(dir $(OPM)) ;\
-	OS=$(shell go env GOOS) && ARCH=$(shell go env GOARCH) && \
-	curl -sSLo $(OPM) https://github.com/operator-framework/operator-registry/releases/download/v1.15.1/$${OS}-$${ARCH}-opm ;\
-	chmod +x $(OPM) ;\
-	}
-else
-OPM = $(shell which opm)
-endif
-endif
-
-# A comma-separated list of bundle images (e.g. make catalog-build BUNDLE_IMGS=example.com/operator-bundle:v0.1.0,example.com/operator-bundle:v0.2.0).
-# These images MUST exist in a registry and be pull-able.
-BUNDLE_IMGS ?= $(BUNDLE_IMG)
-
-# The image tag given to the resulting catalog image (e.g. make catalog-build CATALOG_IMG=example.com/operator-catalog:v0.2.0).
-CATALOG_IMG ?= $(IMAGE_TAG_BASE)-catalog:v$(VERSION)
-
-# Set CATALOG_BASE_IMG to an existing catalog image tag to add $BUNDLE_IMGS to that image.
-ifneq ($(origin CATALOG_BASE_IMG), undefined)
-FROM_INDEX_OPT := --from-index $(CATALOG_BASE_IMG)
-endif
-
-# Build a catalog image by adding bundle images to an empty catalog using the operator package manager tool, 'opm'.
-# This recipe invokes 'opm' in 'semver' bundle add mode. For more information on add modes, see:
-# https://github.com/operator-framework/community-operators/blob/7f1438c/docs/packaging-operator.md#updating-your-existing-operator
-.PHONY: catalog-build
-catalog-build: opm ## Build a catalog image.
-	$(OPM) index add --container-tool docker --mode semver --tag $(CATALOG_IMG) --bundles $(BUNDLE_IMGS) $(FROM_INDEX_OPT)
-
-# Push the catalog image.
-.PHONY: catalog-push
-catalog-push: ## Push a catalog image.
-	$(MAKE) docker-push IMG=$(CATALOG_IMG)
-
-define install-kustomize
-@[ -f $(1) ] || { \
-set -e ;\
-echo "Installing v$(2)" ;\
-cd bin ;\
-wget "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" ;\
-bash ./install_kustomize.sh $(2) ;\
-}
-endef
-
-# go-install-tool will 'go install' any package $2 and install it to $1.
-PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST))))
-define go-install-tool
-@[ -f $(1) ] || { \
-set -e ;\
-echo "Installing $(2)" ;\
-GOBIN=$(PROJECT_DIR)/bin go install $(2) ;\
-}
-endef
-
-.PHONY: env
-env:
-	@make -C deploy/kind kind ingress-nginx
+load: kind
+	$(KIND) load docker-image --name kamaji ${CONTAINER_REPOSITORY}:${VERSION}
 
 ##@ e2e
 
+.PHONY: env
+env: kind
+	$(KIND) create cluster --name kamaji
+
 .PHONY: e2e
-e2e: env load helm ginkgo cert-manager ## Create a KinD cluster, install Kamaji on it and run the test suite.
-	$(HELM) upgrade --debug --install kamaji ./charts/kamaji --create-namespace --namespace kamaji-system --set "image.pullPolicy=Never"
+e2e: env build load helm ginkgo cert-manager ## Create a KinD cluster, install Kamaji on it and run the test suite.
+	$(HELM) repo add clastix https://clastix.github.io/charts
+	$(HELM) dependency build ./charts/kamaji
+	$(HELM) upgrade --debug --install kamaji ./charts/kamaji --create-namespace --namespace kamaji-system --set "image.tag=$(VERSION)" --set "image.pullPolicy=Never" --set "telemetry.disabled=true"
 	$(MAKE) datastores
 	$(GINKGO) -v ./e2e
 
 ##@ Document
 
+CAPI_URL = https://github.com/clastix/cluster-api-control-plane-provider-kamaji.git
+CAPI_DIR := $(shell mktemp -d)
+CRDS_DIR := $(shell mktemp -d)
+
+.PHONY: apidoc
 apidoc: apidocs-gen
-	$(APIDOCS_GEN) crdoc --resources config/crd/bases --output docs/content/reference/api.md --template docs/templates/reference-cr.tmpl
+	@cp charts/kamaji/crds/*.yaml $(CRDS_DIR)
+	@git clone $(CAPI_URL) $(CAPI_DIR)
+	@cp $(CAPI_DIR)/config/crd/bases/*.yaml $(CRDS_DIR)
+	@rm -rf $(CAPI_DIR)
+	$(APIDOCS_GEN) crdoc --resources $(CRDS_DIR) --output docs/content/reference/api.md --template docs/templates/reference-cr.tmpl
+	@rm -rf $(CRDS_DIR)

README.md

@@ -3,66 +3,159 @@
 <p align="left">
   <img src="https://img.shields.io/github/license/clastix/kamaji"/>
   <img src="https://img.shields.io/github/go-mod/go-version/clastix/kamaji"/>
-  <a href="https://github.com/clastix/kamaji/releases">
-    <img src="https://img.shields.io/github/v/release/clastix/kamaji"/>
-    <img src="https://goreportcard.com/badge/github.com/clastix/kamaji">
-  </a>
+  <a href="https://github.com/clastix/kamaji/releases"><img src="https://img.shields.io/github/v/release/clastix/kamaji"/></a>
+  <img src="https://goreportcard.com/badge/github.com/clastix/kamaji">
+  <a href="https://kubernetes.slack.com/archives/C03GLTTMWNN"><img alt="#kamaji on Kubernetes Slack" src="https://img.shields.io/badge/slack-@kubernetes/kamaji-blue.svg?logo=slack"/></a>
 </p>
 
 ![Logo](assets/logo-black.png#gh-light-mode-only)
 ![Logo](assets/logo-white.png#gh-dark-mode-only)
 
-**Kamaji** deploys and operates **Kubernetes Control Plane** at scale with a fraction of the operational burden. Kamaji is special because the Control Plane components are running in a single pod instead of dedicated machines. This solution makes running multiple Control Planes cheaper and easier to deploy and operate.
+### 🤔 What is Kamaji?
 
-<img src="docs/content/images/architecture.png"  width="600">
+**Kamaji** is the **Kubernetes Control Plane Manager** leveraging on the concept of [**Hosted Control Plane**](https://clastix.io/post/the-raise-of-hosted-control-plane-in-kubernetes/).
 
-## Features
+Kamaji's approach is based on running the Kubernetes Control Plane components in Pods instead of dedicated machines.
+This allows operating Kubernetes clusters at scale, with a fraction of the operational burden.
+Thanks to this approach, running multiple Control Planes can be cheaper and easier to deploy and operate.
 
-- **Self Service Kubernetes:** leave users the freedom to self-provision their Kubernetes clusters according to the assigned boundaries.
-- **Multi-cluster Management:** centrally manage multiple clusters from a single admin cluster. Happy SREs. 
-- **Cheaper Control Planes:** place multiple control planes on a single node, instead of having three nodes for a single control plane.
-- **Stronger Multi-Tenancy:** leave users to access the control plane with admin permissions while keeping them isolated at the infrastructure level.
-- **Kubernetes Inception:** use Kubernetes to manage Kubernetes by re-using all the Kubernetes goodies you already know and love.
-- **Full APIs compliant:** all clusters are CNCF compliant built with upstream Kubernetes binaries
+_Kamaji is like a fleet of Site Reliability Engineers with expertise codified into its logic, working 24/7 to keep up and running your Control Planes._
 
-## Roadmap
+<img src="docs/content/images/architecture.png"  width="600" style="display: block; margin: 0 auto">
+
+### 📖 How it works
+
+Kamaji is extending the Kubernetes API capabilities thanks to [Custom Resource Definitions](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/#customresourcedefinitions).
+
+By installing Kamaji, two pairs of new APIs will be available:
+
+- `TenantControlPlane`, the instance definition of your desired Kubernetes Control Plane
+- `Datastore`, the backing store used by one (or more) `TenantControlPlane`
+
+The `TenantControlPlane` (short-named as `tcp`) objects are Namespace-scoped and allows configuring every aspect of your desired Control Plane.
+Besides the Kubernetes configuration values, you can specify the Pod options such as limit, request, tolerations, node selector, etc.,
+as well as how these should be exposed (e.g.: using a `ClusterIP`, a `LoadBalancer`, or a `NodePort`).
+
+The `TenantControlPlane` is the stateless definition of the Control Plane allowing to set up the required components for a full-fledged Kubernetest cluster.
+The state is managed by the `Datastore` API, a cluster-scoped resource which can hold the data of one or more Kubernetes clusters.
+
+> For further information about the API specifications and all the available options,
+> refer to the official [API reference](https://kamaji.clastix.io/reference/api/#tenantcontrolplane).
+
+### ⭐️ Main features
+
+- **Fast provisioning time**: depending on the infrastructure, Tenant Control Planes are up and ready to serve traffic in **16 seconds**.
+- **Streamlined update**: the rollout to a new Kubernetes version for a given Tenant Control Plane takes just **10 seconds**, with a Blue/Green deployment to avoid serving mixed Kubernetes versions.
+- **Resource optimization**: thanks to the Datastore decoupling, there's no need of odd number instances (e.g.: RAFT consensus) by allowing to save up to 60% of HW resources.
+- **Scale from zero to the moon**: scale down the instance when there's no usage, or automatically scale to support the traffic spikes reusing the Kubernetes patterns.
+- **Declarative approach, constant reconciliation**: thanks to the Operator pattern, drift detection happens in real-time, maintaining the desired state.
+- **Automated certificates management**: Kamaji leverages on `kubeadm` and the certificates are automatically created and rotated for you.
+- **Managing core addons**: Kamaji allows configuring automatically `kube-proxy`, `CoreDNS`, and `konnectivity`, with automatic remediation in case of user errors (e.g.: deleting the `CoreDNS` deployment).
+- **Auto Healing**: the `TenantControlPlane` objects in the management cluster are tracked by Kamaji, in case of deletion of those, everything is created in an idempotent way.
+- **Datastore multi-tenancy**: optionally, Kamaji allows running multiple Control Planes on the same _Datastore_ instance leveraging on the multi-tenancy of each driver, decreasing operations and optimizing costs.
+- **Overcoming `etcd` limitations**: optionally, Kamaji allows using a different _Datastore_ thanks to [`kine`](https://github.com/k3s-io/kine) by supporting `MySQL`, `PostgreSQL`, or `NATS` as an alternative.
+- **Simplifying mixed-networks setup**: thanks to [`Konnectivity`](https://kubernetes.io/docs/tasks/extend-kubernetes/setup-konnectivity/),
+  the Tenant Control Plane is connected to the worker nodes hosted in a different network, overcoming the no-NAT availability when dealing with nodes with a non routable IP address
+  (e.g.: worker nodes in a different infrastructure).
+
+### 🚀 Use cases
+
+- [**Creating a private Managed Kubernetes Service**](https://clastix.io/post/netsons-builds-a-managed-kubernetes-service-with-kamaji-and-open-stack/)
+- [**Building a Platform as a Service**](https://aenix.io/cozystack/)
+- [**Overcoming public Managed Kubernetes Services**](https://clastix.io/post/overcoming-eks-limitations-with-kamaji-on-aws/) such as EKS
+- [**Hybrid infrastructures**](https://clastix.io/post/bridging-the-gap-hybrid-kubernetes-clusters-with-remote-control-planes/):
+  host the Control Plane on the Cloud and worker nodes on prem or vice-versa, according to your needs.
+- [**Kubernetes at the edge**](https://clastix.io/post/edgevolution-unleashing-the-power-of-kubernetes-clusters-for-a-revolutionary-edge-computing-experience/):
+  take full advantage of the _Kubernetes API Server as a service_ paradigm.
+- **Kubernetes Control Plane as a Service:** centrally manage multiple Kubernetes clusters from a single management point (_Multi-Cluster management_). 
+- **High-density Control Plane:** place multiple control planes on the same infrastructure, instead of having dedicated machines for each control plane.
+- **Strong Multi-tenancy:** leave users to access the control plane with admin permissions while keeping them isolated at the infrastructure level.
+- **Kubernetes Inception:** use Kubernetes to manage Kubernetes with automation, high-availability, fault tolerance, and autoscaling out of the box. 
+- **Bring Your Own Device:** keep the control plane isolated from data plane. Worker nodes can join and run consistently from everywhere: cloud, edge, and data-center.
+- **Full CNCF compliant:** all clusters are built with upstream Kubernetes binaries, resulting in full CNCF compliant Kubernetes clusters.
+
+> 🤔 You'd like to do the same but don't know how?
+> 💡 [CLASTIX](https://clastix.io/) can help you with your needs!
+
+### 🧑‍💻‍ Production grade
+
+Kamaji is empowering several businesses, and it counts public adopters.
+Check out the [adopters](./ADOPTERS.md) file to learn more.
+
+> 🤗 If you're using Kamaji, share your love by opening a PR!
+
+### 🍦 Vanilla Kubernetes clusters
+
+Kamaji is **not** yet-another-Kubernetes distribution: you have full freedom on the technology stack to provide to end users.
+Kamaji is a perfect fit for Platform Engineering, hiding the complexity of the Control Plane management to developers and DevOps engineers.
+
+The provided Kubernetes Control Planes are [CNCF compliant clusters](https://kamaji.clastix.io/reference/conformance/).
+
+<img src="https://raw.githubusercontent.com/cncf/artwork/master/projects/kubernetes/certified-kubernetes/versionless/color/certified-kubernetes-color.png" style="display: block; width: 75px; margin: 0 auto">
+
+### 🐢 Cluster API support
+
+Kamaji is **not** a [Cluster API](https://cluster-api.sigs.k8s.io/) replacement, rather, it plays very well with it.
+
+Since Kamaji is just focusing on the Control Plane a [Kamaji's Cluster API Control Plane provider](https://github.com/clastix/cluster-api-control-plane-provider-kamaji) has been developed.
+
+### 🛣️ Roadmap
 
 - [x] Dynamic address on Load Balancer
 - [x] Zero Downtime Tenant Control Plane upgrade
-- [x] Join worker nodes from anywhere
-- [x] Alternative datastore MySQL and PostgreSQL
-- [x] Pool of multiple datastores
-- [x] Seamless migration between datastores
+- [x] [Join worker nodes from anywhere thanks to Konnectivity](https://kamaji.clastix.io/concepts/#konnectivity)
+- [x] [Alternative datastore MySQL, PostgreSQL, NATS](https://kamaji.clastix.io/guides/alternative-datastore/)
+- [x] [Pool of multiple datastores](https://kamaji.clastix.io/concepts/#datastores)
+- [x] [Seamless migration between datastores](https://kamaji.clastix.io/guides/datastore-migration/)
 - [ ] Automatic assignment to a datastore
 - [ ] Autoscaling of Tenant Control Plane
-- [x] Provisioning through Cluster APIs
+- [x] [Provisioning through Cluster APIs](https://github.com/clastix/cluster-api-control-plane-provider-kamaji)
 - [ ] Terraform provider
-- [ ] Custom Prometheus metrics for monitoring and alerting
+- [ ] Custom Prometheus metrics
 
+### 🎥 Multimedia
 
-## Documentation
-Please, check the project's [documentation](https://kamaji.clastix.io/) for getting started with Kamaji.
+- Playlist ▶️ [Tutorials and How-Tos by Dario Tranchitella, CLASTIX](https://www.youtube.com/playlist?list=PLjiUjoV4Ws_3pNsUpTXI-KKk731nD2MQY)
+- YouTube ▶️ [Metal³ provisioning with Kamaji Hosted Control Planes by Huy Mai, Ericsson](https://youtu.be/u9sbURj6jXY?t=10536)
+- YouTube ▶️ [Hands-on introduction to Kamaji](https://www.youtube.com/watch?v=HhevxwQWQ88)
+- YouTube ▶️ [Scaling Kubernetes up to 1,000 Control Planes](https://www.youtube.com/watch?v=W_HXRXJh96U)
+- YouTube ▶️ [Equinix, Kamaji, and Cluster API](https://www.youtube.com/watch?v=TLBTqROj_wA)
+- YouTube ▶️ [Rancher & Kamaji: solving multitenancy challenges in the Kubernetes world](https://www.youtube.com/watch?v=VXHNrMmlF8U)
+- YouTube ▶️ [Enabling Self-Service Kubernetes clusters with Kamaji and Paralus](https://www.youtube.com/watch?v=JWA2LwZazM0)
+- YouTube ▶️ [Hosted Control Plane on Kubernetes (HPC) with Kamaji and K0mostron by Hervé Leclerc, ALTER WAY](https://www.youtube.com/watch?v=vmRdE2ngn78)
 
-## Contributions
-Kamaji is Open Source with Apache 2 license and any contribution is welcome. Open an issue or suggest an enhancement on the GitHub [project's page](https://github.com/clastix/kamaji). Join the [Kubernetes Slack Workspace](https://slack.k8s.io/) and the [`#kamaji`](https://kubernetes.slack.com/archives/C03GLTTMWNN) channel to meet end-users and contributors.
+### 🏷️ Versioning
 
-## FAQs
-Q. What does Kamaji mean?
+Versioning adheres to the [Semantic Versioning](http://semver.org/) principles.
+A full list of the available releases is available in the GitHub repository's [**Release** section](https://github.com/clastix/kamaji/releases).
 
-A. Kamaji is named as the character _Kamaji_ from the Japanese movie [_Spirited Away_](https://en.wikipedia.org/wiki/Spirited_Away).
+### 📄 Documentation
 
-Q. Is Kamaji another Kubernetes distribution?
+Further documentation can be found on the official [Kamaji documentation website](https://kamaji.clastix.io/).
 
-A. No, Kamaji is a Kubernetes Operator you can install on top of any Kubernetes cluster to provide hundreds or thousands of managed Kubernetes clusters as a service. We tested Kamaji on vanilla Kubernetes 1.22+, KinD, and Azure AKS. We expect it to work smoothly on other Kubernetes distributions. The tenant clusters made with Kamaji are conformant CNCF Kubernetes clusters as we leverage [`kubeadm`](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/).
+### 🤝 Contributions
 
-Q. Is it safe to run Kubernetes control plane components in a pod instead of dedicated virtual machines?
+Contributions are highly appreciated and very welcomed!
 
-A. Yes, the tenant control plane components are packaged in the same way they are running in bare metal or virtual nodes. We leverage the `kubeadm` code to set up the control plane components as they were running on their own server. The unchanged images of upstream `kube-apiserver`, `kube-scheduler`, and `kube-controller-manager` are used.
+In case of bugs, please, check if the issue has been already opened by checking the [GitHub Issues](https://github.com/clastix/kamaji/issues) section.
+In case it isn't, you can open a new one: a detailed report will help us to replicate it, assess it, and work on a fix.
 
-Q. You already provide a Kubernetes multi-tenancy solution with [Capsule](https://capsule.clastix.io). Why does Kamaji matter?
+You can express your intention in working on the fix on your own.
+The commit messages are checked according to the described [semantics](https://github.com/projectcapsule/capsule/blob/main/CONTRIBUTING.md#semantics).
+Commits are used to generate the changelog, and their author will be referenced in it.
 
-A. A multi-tenancy solution, like Capsule shares the Kubernetes control plane among all tenants keeping tenant namespaces isolated by policies. While the solution is the right choice by balancing between features and ease of usage, there are cases where a tenant user requires access to the control plane, for example, when a tenant requires to manage CRDs on his own. With Kamaji, you can provide cluster admin permissions to the tenant.
+In case of **✨ Feature Requests** please use the [Discussion's Feature Request section](https://github.com/clastix/kamaji/discussions/categories/feature-requests).
 
-Q. Well you convinced me, how to get a try?
+### 📝 License
 
-A. It is possible to get started with Kamaji on a laptop with [KinD](getting-started.md) installed.
+Kamaji is licensed under Apache 2.0.
+The code is provided as-is with no warranties.
+
+### 🛟 Commercial Support
+
+![CLASTIX](https://avatars.githubusercontent.com/u/39170129?s=50&v=4) [CLASTIX](https://clastix.io/) is the commercial company behind Kamaji and the Cluster API Control Plane provider.
+
+If you're looking to run Kamaji in production and would like to learn more, **CLASTIX** can help by offering [Open Source support plans](https://clastix.io/support),
+as well as providing a comprehensive Enterprise Platform named [CLASTIX Enterprise Platform](https://clastix.cloud/), built on top of the Kamaji and [Capsule](https://capsule.clastix.io/) project (now donated to CNCF as a Sandbox project).
+
+Feel free to get in touch with the provided [Contact form](https://clastix.io/contact).

api/v1alpha1/datastore_types.go

@@ -8,7 +8,8 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
-// +kubebuilder:validation:Enum=etcd;MySQL;PostgreSQL
+//+kubebuilder:validation:Enum=etcd;MySQL;PostgreSQL;NATS
+//+kubebuilder:validation:XValidation:rule="self == oldSelf",message="Datastore driver is immutable"
 
 type Driver string
 
@@ -16,13 +17,21 @@ var (
 	EtcdDriver           Driver = "etcd"
 	KineMySQLDriver      Driver = "MySQL"
 	KinePostgreSQLDriver Driver = "PostgreSQL"
+	KineNatsDriver       Driver = "NATS"
 )
 
-// +kubebuilder:validation:MinItems=1
+//+kubebuilder:validation:MinItems=1
 
 type Endpoints []string
 
 // DataStoreSpec defines the desired state of DataStore.
+// +kubebuilder:validation:XValidation:rule="(self.driver == \"etcd\") ? (self.tlsConfig != null && (has(self.tlsConfig.certificateAuthority.privateKey.secretReference) || has(self.tlsConfig.certificateAuthority.privateKey.content))) : true", message="certificateAuthority privateKey must have secretReference or content when driver is etcd"
+// +kubebuilder:validation:XValidation:rule="(self.driver == \"etcd\") ? (self.tlsConfig != null && (has(self.tlsConfig.clientCertificate.certificate.secretReference) || has(self.tlsConfig.clientCertificate.certificate.content))) : true", message="clientCertificate must have secretReference or content when driver is etcd"
+// +kubebuilder:validation:XValidation:rule="(self.driver == \"etcd\") ? (self.tlsConfig != null && (has(self.tlsConfig.clientCertificate.privateKey.secretReference) || has(self.tlsConfig.clientCertificate.privateKey.content))) : true", message="clientCertificate privateKey must have secretReference or content when driver is etcd"
+// +kubebuilder:validation:XValidation:rule="(self.driver != \"etcd\" && has(self.tlsConfig) && has(self.tlsConfig.clientCertificate)) ? (((has(self.tlsConfig.clientCertificate.certificate.secretReference) || has(self.tlsConfig.clientCertificate.certificate.content)))) : true", message="When driver is not etcd and tlsConfig exists, clientCertificate must be null or contain valid content"
+// +kubebuilder:validation:XValidation:rule="(self.driver != \"etcd\" && has(self.basicAuth)) ? ((has(self.basicAuth.username.secretReference) || has(self.basicAuth.username.content))) : true", message="When driver is not etcd and basicAuth exists, username must have secretReference or content"
+// +kubebuilder:validation:XValidation:rule="(self.driver != \"etcd\" && has(self.basicAuth)) ? ((has(self.basicAuth.password.secretReference) || has(self.basicAuth.password.content))) : true", message="When driver is not etcd and basicAuth exists, password must have secretReference or content"
+// +kubebuilder:validation:XValidation:rule="(self.driver != \"etcd\") ? (has(self.tlsConfig) || has(self.basicAuth)) : true", message="When driver is not etcd, either tlsConfig or basicAuth must be provided"
 type DataStoreSpec struct {
 	// The driver to use to connect to the shared datastore.
 	Driver Driver `json:"driver"`
@@ -33,7 +42,8 @@ type DataStoreSpec struct {
 	// This value is optional.
 	BasicAuth *BasicAuth `json:"basicAuth,omitempty"`
 	// Defines the TLS/SSL configuration required to connect to the data store in a secure way.
-	TLSConfig TLSConfig `json:"tlsConfig"`
+	// This value is optional.
+	TLSConfig *TLSConfig `json:"tlsConfig,omitempty"`
 }
 
 // TLSConfig contains the information used to connect to the data store using a secured connection.
@@ -42,7 +52,7 @@ type TLSConfig struct {
 	// The key reference is required since etcd authentication is based on certificates, and Kamaji is responsible in creating this.
 	CertificateAuthority CertKeyPair `json:"certificateAuthority"`
 	// Specifies the SSL/TLS key and private key pair used to connect to the data store.
-	ClientCertificate ClientCertificate `json:"clientCertificate"`
+	ClientCertificate *ClientCertificate `json:"clientCertificate,omitempty"`
 }
 
 type ClientCertificate struct {
@@ -89,6 +99,7 @@ type DataStoreStatus struct {
 //+kubebuilder:resource:scope=Cluster
 //+kubebuilder:printcolumn:name="Driver",type="string",JSONPath=".spec.driver",description="Kamaji data store driver"
 //+kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="Age"
+//+kubebuilder:metadata:annotations={"cert-manager.io/inject-ca-from=kamaji-system/kamaji-serving-cert"}
 
 // DataStore is the Schema for the datastores API.
 type DataStore struct {

api/v1alpha1/indexer_datastore_usedsecret.go

@@ -43,20 +43,24 @@ func (d *DatastoreUsedSecret) ExtractValue() client.IndexerFunc {
 			}
 		}
 
-		if ds.Spec.TLSConfig.CertificateAuthority.Certificate.SecretRef != nil {
-			res = append(res, d.namespacedName(*ds.Spec.TLSConfig.CertificateAuthority.Certificate.SecretRef))
-		}
+		if ds.Spec.TLSConfig != nil {
+			if ds.Spec.TLSConfig.CertificateAuthority.Certificate.SecretRef != nil {
+				res = append(res, d.namespacedName(*ds.Spec.TLSConfig.CertificateAuthority.Certificate.SecretRef))
+			}
 
-		if ds.Spec.TLSConfig.CertificateAuthority.PrivateKey != nil && ds.Spec.TLSConfig.CertificateAuthority.PrivateKey.SecretRef != nil {
-			res = append(res, d.namespacedName(*ds.Spec.TLSConfig.CertificateAuthority.PrivateKey.SecretRef))
-		}
+			if ds.Spec.TLSConfig.CertificateAuthority.PrivateKey != nil && ds.Spec.TLSConfig.CertificateAuthority.PrivateKey.SecretRef != nil {
+				res = append(res, d.namespacedName(*ds.Spec.TLSConfig.CertificateAuthority.PrivateKey.SecretRef))
+			}
 
-		if ds.Spec.TLSConfig.ClientCertificate.Certificate.SecretRef != nil {
-			res = append(res, d.namespacedName(*ds.Spec.TLSConfig.ClientCertificate.Certificate.SecretRef))
-		}
+			if ds.Spec.TLSConfig.ClientCertificate != nil {
+				if ds.Spec.TLSConfig.ClientCertificate.Certificate.SecretRef != nil {
+					res = append(res, d.namespacedName(*ds.Spec.TLSConfig.ClientCertificate.Certificate.SecretRef))
+				}
 
-		if ds.Spec.TLSConfig.ClientCertificate.PrivateKey.SecretRef != nil {
-			res = append(res, d.namespacedName(*ds.Spec.TLSConfig.ClientCertificate.PrivateKey.SecretRef))
+				if ds.Spec.TLSConfig.ClientCertificate.PrivateKey.SecretRef != nil {
+					res = append(res, d.namespacedName(*ds.Spec.TLSConfig.ClientCertificate.PrivateKey.SecretRef))
+				}
+			}
 		}
 
 		return res

api/v1alpha1/suite_test.go

@@ -0,0 +1,55 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package v1alpha1
+
+import (
+	"path/filepath"
+	"testing"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/rest"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/envtest"
+)
+
+var (
+	cfg       *rest.Config
+	k8sClient client.Client
+	testEnv   *envtest.Environment
+)
+
+func TestAPIs(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "TenantControlPlane Suite")
+}
+
+var _ = BeforeSuite(func() {
+	By("bootstrapping test environment")
+	testEnv = &envtest.Environment{
+		CRDDirectoryPaths: []string{
+			filepath.Join("..", "..", "charts", "kamaji", "crds"),
+			// filepath.Join("../..", "chart", "kamaji", "crds"),
+		},
+	}
+
+	var err error
+	cfg, err = testEnv.Start()
+	Expect(err).ToNot(HaveOccurred())
+	Expect(cfg).ToNot(BeNil())
+
+	err = AddToScheme(scheme.Scheme)
+	Expect(err).NotTo(HaveOccurred())
+
+	k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme})
+	Expect(err).ToNot(HaveOccurred())
+	Expect(k8sClient).ToNot(BeNil())
+})
+
+var _ = AfterSuite(func() {
+	By("tearing down the test environment")
+	err := testEnv.Stop()
+	Expect(err).ToNot(HaveOccurred())
+})

api/v1alpha1/tenantcontrolplane_const.go

@@ -0,0 +1,10 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package v1alpha1
+
+const (
+	// PausedReconciliationAnnotation is an annotation that can be applied to
+	// Tenant Control Plane objects to prevent the controller from processing such a resource.
+	PausedReconciliationAnnotation = "kamaji.clastix.io/paused"
+)

api/v1alpha1/tenantcontrolplane_funcs.go

@@ -61,10 +61,35 @@ func (in *TenantControlPlane) DeclaredControlPlaneAddress(ctx context.Context, c
 			return "", kamajierrors.NonExposedLoadBalancerError{}
 		}
 
-		for _, lb := range loadBalancerStatus.Ingress {
-			if ip := lb.IP; len(ip) > 0 {
-				return ip, nil
-			}
+		return getLoadBalancerAddress(loadBalancerStatus.Ingress)
+	}
+
+	return "", kamajierrors.MissingValidIPError{}
+}
+
+// getLoadBalancerAddress extracts the IP address from LoadBalancer ingress.
+// It also checks and rejects hostname usage for LoadBalancer ingress.
+//
+// Reasons for not supporting hostnames:
+// - DNS resolution can differ across environments, leading to inconsistent behavior.
+// - It may cause connectivity problems between Kubernetes components.
+// - The DNS resolution could change over time, potentially breaking cluster-to-API-server connections.
+//
+// Recommended solutions:
+// - Use a static IP address to ensure stable and predictable communication within the cluster.
+// - If a hostname is necessary, consider setting up a Virtual IP (VIP) for the given hostname.
+// - Alternatively, use an external load balancer that can provide a stable IP address.
+//
+// Note: Implementing L7 routing with the API Server requires a deep understanding of the implications.
+// Users should be aware of the complexities involved, including potential issues with TLS passthrough
+// for client-based certificate authentication in Ingress expositions.
+func getLoadBalancerAddress(ingress []corev1.LoadBalancerIngress) (string, error) {
+	for _, lb := range ingress {
+		if ip := lb.IP; len(ip) > 0 {
+			return ip, nil
+		}
+		if hostname := lb.Hostname; len(hostname) > 0 {
+			return "", fmt.Errorf("hostname not supported for LoadBalancer ingress: use static IP instead")
 		}
 	}
 

api/v1alpha1/tenantcontrolplane_interfaces.go

@@ -8,5 +8,5 @@ package v1alpha1
 // +kubebuilder:object:generate=false
 type KubeadmConfigChecksumDependant interface {
 	GetChecksum() string
-	SetChecksum(string)
+	SetChecksum(checksum string)
 }

api/v1alpha1/tenantcontrolplane_kubeadmphase_funcs.go

@@ -7,7 +7,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
-func (in KubeadmPhaseStatus) GetChecksum() string {
+func (in *KubeadmPhaseStatus) GetChecksum() string {
 	return in.Checksum
 }
 

api/v1alpha1/tenantcontrolplane_registrysettings.go

@@ -4,15 +4,15 @@
 package v1alpha1
 
 type RegistrySettings struct {
-	// +kubebuilder:default="registry.k8s.io"
+	//+kubebuilder:default="registry.k8s.io"
 	Registry string `json:"registry,omitempty"`
 	// The tag to append to all the Control Plane container images.
 	// Optional.
 	TagSuffix string `json:"tagSuffix,omitempty"`
-	// +kubebuilder:default="kube-apiserver"
+	//+kubebuilder:default="kube-apiserver"
 	APIServerImage string `json:"apiServerImage,omitempty"`
-	// +kubebuilder:default="kube-controller-manager"
+	//+kubebuilder:default="kube-controller-manager"
 	ControllerManagerImage string `json:"controllerManagerImage,omitempty"`
-	// +kubebuilder:default="kube-scheduler"
+	//+kubebuilder:default="kube-scheduler"
 	SchedulerImage string `json:"schedulerImage,omitempty"`
 }

api/v1alpha1/tenantcontrolplane_status.go

@@ -122,6 +122,12 @@ type ExternalKubernetesObjectStatus struct {
 	LastUpdate metav1.Time `json:"lastUpdate,omitempty"`
 }
 
+type KonnectivityAgentStatus struct {
+	ExternalKubernetesObjectStatus `json:",inline"`
+
+	Mode KonnectivityAgentMode `json:"mode,omitempty"`
+}
+
 // KonnectivityStatus defines the status of Konnectivity as Addon.
 type KonnectivityStatus struct {
 	Enabled            bool                            `json:"enabled"`
@@ -130,7 +136,7 @@ type KonnectivityStatus struct {
 	Kubeconfig         KubeconfigStatus                `json:"kubeconfig,omitempty"`
 	ServiceAccount     ExternalKubernetesObjectStatus  `json:"sa,omitempty"`
 	ClusterRoleBinding ExternalKubernetesObjectStatus  `json:"clusterrolebinding,omitempty"`
-	Agent              ExternalKubernetesObjectStatus  `json:"agent,omitempty"`
+	Agent              KonnectivityAgentStatus         `json:"agent,omitempty"`
 	Service            KubernetesServiceStatus         `json:"service,omitempty"`
 }
 
@@ -183,11 +189,12 @@ type KubernetesStatus struct {
 	Ingress    *KubernetesIngressStatus   `json:"ingress,omitempty"`
 }
 
-// +kubebuilder:validation:Enum=Provisioning;CertificateAuthorityRotating;Upgrading;Migrating;Ready;NotReady
+// +kubebuilder:validation:Enum=Provisioning;CertificateAuthorityRotating;Upgrading;Migrating;Ready;NotReady;Sleeping
 type KubernetesVersionStatus string
 
 var (
 	VersionProvisioning KubernetesVersionStatus = "Provisioning"
+	VersionSleeping     KubernetesVersionStatus = "Sleeping"
 	VersionCARotating   KubernetesVersionStatus = "CertificateAuthorityRotating"
 	VersionUpgrading    KubernetesVersionStatus = "Upgrading"
 	VersionMigrating    KubernetesVersionStatus = "Migrating"
@@ -198,7 +205,7 @@ var (
 type KubernetesVersion struct {
 	// Version is the running Kubernetes version of the Tenant Control Plane.
 	Version string `json:"version,omitempty"`
-	// +kubebuilder:default=Provisioning
+	//+kubebuilder:default=Provisioning
 	// Status returns the current status of the Kubernetes version, such as its provisioning state, or completed upgrade.
 	Status *KubernetesVersionStatus `json:"status,omitempty"`
 }

api/v1alpha1/tenantcontrolplane_types.go

@@ -11,25 +11,46 @@ import (
 
 // NetworkProfileSpec defines the desired state of NetworkProfile.
 type NetworkProfileSpec struct {
+	// LoadBalancerSourceRanges restricts the IP ranges that can access
+	// the LoadBalancer type Service. This field defines a list of IP
+	// address ranges (in CIDR format) that are allowed to access the service.
+	// If left empty, the service will allow traffic from all IP ranges (0.0.0.0/0).
+	// This feature is useful for restricting access to API servers or services
+	// to specific networks for security purposes.
+	// Example: {"192.168.1.0/24", "10.0.0.0/8"}
+	LoadBalancerSourceRanges []string `json:"loadBalancerSourceRanges,omitempty"`
+	// Specify the LoadBalancer class in case of multiple load balancer implementations.
+	// Field supported only for Tenant Control Plane instances exposed using a LoadBalancer Service.
+	// +kubebuilder:validation:MinLength=1
+	// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="LoadBalancerClass is immutable"
+	LoadBalancerClass *string `json:"loadBalancerClass,omitempty"`
 	// Address where API server of will be exposed.
 	// In case of LoadBalancer Service, this can be empty in order to use the exposed IP provided by the cloud controller manager.
 	Address string `json:"address,omitempty"`
+	// The default domain name used for DNS resolution within the cluster.
+	//+kubebuilder:default="cluster.local"
+	//+kubebuilder:validation:XValidation:rule="self == oldSelf",message="changing the cluster domain is not supported"
+	//+kubebuilder:validation:Pattern=.*\..*
+	ClusterDomain string `json:"clusterDomain,omitempty"`
 	// AllowAddressAsExternalIP will include tenantControlPlane.Spec.NetworkProfile.Address in the section of
 	// ExternalIPs of the Kubernetes Service (only ClusterIP or NodePort)
 	AllowAddressAsExternalIP bool `json:"allowAddressAsExternalIP,omitempty"`
 	// Port where API server of will be exposed
-	// +kubebuilder:default=6443
+	//+kubebuilder:default=6443
 	Port int32 `json:"port,omitempty"`
 	// CertSANs sets extra Subject Alternative Names (SANs) for the API Server signing certificate.
 	// Use this field to add additional hostnames when exposing the Tenant Control Plane with third solutions.
 	CertSANs []string `json:"certSANs,omitempty"`
-	// Kubernetes Service
-	// +kubebuilder:default="10.96.0.0/16"
+	// CIDR for Kubernetes Services: if empty, defaulted to 10.96.0.0/16.
+	//+kubebuilder:default="10.96.0.0/16"
 	ServiceCIDR string `json:"serviceCidr,omitempty"`
-	// CIDR for Kubernetes Pods
-	// +kubebuilder:default="10.244.0.0/16"
+	// CIDR for Kubernetes Pods: if empty, defaulted to 10.244.0.0/16.
+	//+kubebuilder:default="10.244.0.0/16"
 	PodCIDR string `json:"podCidr,omitempty"`
-	// +kubebuilder:default={"10.96.0.10"}
+	// The DNS Service for internal resolution, it must match the Service CIDR.
+	// In case of an empty value, it is automatically computed according to the Service CIDR, e.g.:
+	// Service CIDR 10.96.0.0/16, the resulting DNS Service IP will be 10.96.0.10 for IPv4,
+	// for IPv6 from the CIDR 2001:db8:abcd::/64 the resulting DNS Service IP will be 2001:db8:abcd::10.
 	DNSServiceIPs []string `json:"dnsServiceIPs,omitempty"`
 }
 
@@ -46,11 +67,12 @@ const (
 
 type KubeletSpec struct {
 	// Ordered list of the preferred NodeAddressTypes to use for kubelet connections.
-	// Default to Hostname, InternalIP, ExternalIP.
-	// +kubebuilder:default={"Hostname","InternalIP","ExternalIP"}
-	// +kubebuilder:validation:MinItems=1
+	// Default to InternalIP, ExternalIP, Hostname.
+	//+kubebuilder:default={"InternalIP","ExternalIP","Hostname"}
+	//+kubebuilder:validation:MinItems=1
+	//+listType=set
 	PreferredAddressTypes []KubeletPreferredAddressType `json:"preferredAddressTypes,omitempty"`
-	// CGroupFS defines the  cgroup driver for Kubelet
+	// CGroupFS defines the cgroup driver for Kubelet
 	// https://kubernetes.io/docs/tasks/administer-cluster/kubeadm/configure-cgroup-driver/
 	CGroupFS CGroupDriver `json:"cgroupfs,omitempty"`
 }
@@ -63,7 +85,7 @@ type KubernetesSpec struct {
 
 	// List of enabled Admission Controllers for the Tenant cluster.
 	// Full reference available here: https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers
-	// +kubebuilder:default=CertificateApproval;CertificateSigning;CertificateSubjectRestriction;DefaultIngressClass;DefaultStorageClass;DefaultTolerationSeconds;LimitRanger;MutatingAdmissionWebhook;NamespaceLifecycle;PersistentVolumeClaimResize;Priority;ResourceQuota;RuntimeClass;ServiceAccount;StorageObjectInUseProtection;TaintNodesByCondition;ValidatingAdmissionWebhook
+	//+kubebuilder:default=CertificateApproval;CertificateSigning;CertificateSubjectRestriction;DefaultIngressClass;DefaultStorageClass;DefaultTolerationSeconds;LimitRanger;MutatingAdmissionWebhook;NamespaceLifecycle;PersistentVolumeClaimResize;Priority;ResourceQuota;RuntimeClass;ServiceAccount;StorageObjectInUseProtection;TaintNodesByCondition;ValidatingAdmissionWebhook
 	AdmissionControllers AdmissionControllers `json:"admissionControllers,omitempty"`
 }
 
@@ -105,10 +127,10 @@ type ControlPlaneComponentsResources struct {
 type DeploymentSpec struct {
 	// RegistrySettings allows to override the default images for the given Tenant Control Plane instance.
 	// It could be used to point to a different container registry rather than the public one.
-	// +kubebuilder:default={registry:"registry.k8s.io",apiServerImage:"kube-apiserver",controllerManagerImage:"kube-controller-manager",schedulerImage:"kube-scheduler"}
+	//+kubebuilder:default={registry:"registry.k8s.io",apiServerImage:"kube-apiserver",controllerManagerImage:"kube-controller-manager",schedulerImage:"kube-scheduler"}
 	RegistrySettings RegistrySettings `json:"registrySettings,omitempty"`
-	// +kubebuilder:default=2
-	Replicas int32 `json:"replicas,omitempty"`
+	//+kubebuilder:default=2
+	Replicas *int32 `json:"replicas,omitempty"`
 	// NodeSelector is a selector which must be true for the pod to fit on a node.
 	// Selector which must match a node's labels for the pod to be scheduled on that node.
 	// More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/
@@ -121,7 +143,7 @@ type DeploymentSpec struct {
 	RuntimeClassName string `json:"runtimeClassName,omitempty"`
 	// Strategy describes how to replace existing pods with new ones for the given Tenant Control Plane.
 	// Default value is set to Rolling Update, with a blue/green strategy.
-	// +kubebuilder:default={type:"RollingUpdate",rollingUpdate:{maxUnavailable:0,maxSurge:"100%"}}
+	//+kubebuilder:default={type:"RollingUpdate",rollingUpdate:{maxUnavailable:0,maxSurge:"100%"}}
 	Strategy appsv1.DeploymentStrategy `json:"strategy,omitempty"`
 	// If specified, the Tenant Control Plane pod's tolerations.
 	// More info: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/
@@ -138,9 +160,12 @@ type DeploymentSpec struct {
 	// (kube-apiserver, controller-manager, and scheduler).
 	Resources *ControlPlaneComponentsResources `json:"resources,omitempty"`
 	// ExtraArgs allows adding additional arguments to the Control Plane components,
-	// such as kube-apiserver, controller-manager, and scheduler.
-	ExtraArgs          *ControlPlaneExtraArgs `json:"extraArgs,omitempty"`
-	AdditionalMetadata AdditionalMetadata     `json:"additionalMetadata,omitempty"`
+	// such as kube-apiserver, controller-manager, and scheduler. WARNING - This option
+	// can override existing parameters and cause components to misbehave in unxpected ways.
+	// Only modify if you know what you are doing.
+	ExtraArgs             *ControlPlaneExtraArgs `json:"extraArgs,omitempty"`
+	AdditionalMetadata    AdditionalMetadata     `json:"additionalMetadata,omitempty"`
+	PodAdditionalMetadata AdditionalMetadata     `json:"podAdditionalMetadata,omitempty"`
 	// AdditionalInitContainers allows adding additional init containers to the Control Plane deployment.
 	AdditionalInitContainers []corev1.Container `json:"additionalInitContainers,omitempty"`
 	// AdditionalContainers allows adding additional containers to the Control Plane deployment.
@@ -150,6 +175,9 @@ type DeploymentSpec struct {
 	// AdditionalVolumeMounts allows to mount an additional volume into each component of the Control Plane
 	// (kube-apiserver, controller-manager, and scheduler).
 	AdditionalVolumeMounts *AdditionalVolumeMounts `json:"additionalVolumeMounts,omitempty"`
+	//+kubebuilder:default="default"
+	// ServiceAccountName allows to specify the service account to be mounted to the pods of the Control plane deployment
+	ServiceAccountName string `json:"serviceAccountName,omitempty"`
 }
 
 // AdditionalVolumeMounts allows mounting additional volumes to the Control Plane components.
@@ -189,37 +217,67 @@ type ImageOverrideTrait struct {
 }
 
 // ExtraArgs allows adding additional arguments to said component.
+// WARNING - This option can override existing konnectivity
+// parameters and cause konnectivity components to misbehave in
+// unxpected ways. Only modify if you know what you are doing.
 type ExtraArgs []string
 
 type KonnectivityServerSpec struct {
 	// The port which Konnectivity server is listening to.
 	Port int32 `json:"port"`
 	// Container image version of the Konnectivity server.
-	// +kubebuilder:default=v0.0.32
+	//+kubebuilder:default=v0.28.6
 	Version string `json:"version,omitempty"`
 	// Container image used by the Konnectivity server.
-	// +kubebuilder:default=registry.k8s.io/kas-network-proxy/proxy-server
+	//+kubebuilder:default=registry.k8s.io/kas-network-proxy/proxy-server
 	Image string `json:"image,omitempty"`
 	// Resources define the amount of CPU and memory to allocate to the Konnectivity server.
 	Resources *corev1.ResourceRequirements `json:"resources,omitempty"`
 	ExtraArgs ExtraArgs                    `json:"extraArgs,omitempty"`
 }
 
+type KonnectivityAgentMode string
+
+var (
+	KonnectivityAgentModeDaemonSet  KonnectivityAgentMode = "DaemonSet"
+	KonnectivityAgentModeDeployment KonnectivityAgentMode = "Deployment"
+)
+
+//+kubebuilder:validation:XValidation:rule="!(self.mode == 'DaemonSet' && has(self.replicas) && self.replicas != 0) && !(self.mode == 'Deployment' && self.replicas == 0)",message="replicas must be 0 when mode is DaemonSet, and greater than 0 when mode is Deployment"
+
 type KonnectivityAgentSpec struct {
 	// AgentImage defines the container image for Konnectivity's agent.
-	// +kubebuilder:default=registry.k8s.io/kas-network-proxy/proxy-agent
+	//+kubebuilder:default=registry.k8s.io/kas-network-proxy/proxy-agent
 	Image string `json:"image,omitempty"`
 	// Version for Konnectivity agent.
-	// +kubebuilder:default=v0.0.32
-	Version   string    `json:"version,omitempty"`
-	ExtraArgs ExtraArgs `json:"extraArgs,omitempty"`
+	//+kubebuilder:default=v0.28.6
+	Version string `json:"version,omitempty"`
+	// Tolerations for the deployed agent.
+	// Can be customized to start the konnectivity-agent even if the nodes are not ready or tainted.
+	//+kubebuilder:default={{key: "CriticalAddonsOnly", operator: "Exists"}}
+	Tolerations []corev1.Toleration `json:"tolerations,omitempty"`
+	ExtraArgs   ExtraArgs           `json:"extraArgs,omitempty"`
+	// HostNetwork enables the konnectivity agent to use the Host network namespace.
+	// By enabling this mode, the Agent doesn't need to wait for the CNI initialisation,
+	// enabling a sort of out-of-band access to nodes for troubleshooting scenarios,
+	// or when the agent needs direct access to the host network.
+	//+kubebuilder:default=false
+	HostNetwork bool `json:"hostNetwork,omitempty"`
+	// Mode allows specifying the Agent deployment mode: Deployment, or DaemonSet (default).
+	//+kubebuilder:default="DaemonSet"
+	//+kubebuilder:validation:Enum=DaemonSet;Deployment
+	Mode KonnectivityAgentMode `json:"mode,omitempty"`
+	// Replicas defines the number of replicas when Mode is Deployment.
+	// Must be 0 if Mode is DaemonSet.
+	//+kubebuilder:validation:Optional
+	Replicas int32 `json:"replicas,omitempty"`
 }
 
 // KonnectivitySpec defines the spec for Konnectivity.
 type KonnectivitySpec struct {
-	// +kubebuilder:default={version:"v0.0.32",image:"registry.k8s.io/kas-network-proxy/proxy-server",port:8132}
+	//+kubebuilder:default={version:"v0.28.6",image:"registry.k8s.io/kas-network-proxy/proxy-server",port:8132}
 	KonnectivityServerSpec KonnectivityServerSpec `json:"server,omitempty"`
-	// +kubebuilder:default={version:"v0.0.32",image:"registry.k8s.io/kas-network-proxy/proxy-agent"}
+	//+kubebuilder:default={version:"v0.28.6",image:"registry.k8s.io/kas-network-proxy/proxy-agent",mode:"DaemonSet"}
 	KonnectivityAgentSpec KonnectivityAgentSpec `json:"agent,omitempty"`
 }
 
@@ -236,12 +294,34 @@ type AddonsSpec struct {
 }
 
 // TenantControlPlaneSpec defines the desired state of TenantControlPlane.
+// +kubebuilder:validation:XValidation:rule="!has(oldSelf.dataStore) || has(self.dataStore)", message="unsetting the dataStore is not supported"
+// +kubebuilder:validation:XValidation:rule="!has(oldSelf.dataStoreSchema) || has(self.dataStoreSchema)", message="unsetting the dataStoreSchema is not supported"
+// +kubebuilder:validation:XValidation:rule="!has(oldSelf.dataStoreUsername) || has(self.dataStoreUsername)", message="unsetting the dataStoreUsername is not supported"
+// +kubebuilder:validation:XValidation:rule="!has(self.networkProfile.loadBalancerSourceRanges) || (size(self.networkProfile.loadBalancerSourceRanges) == 0 || self.controlPlane.service.serviceType == 'LoadBalancer')", message="LoadBalancer source ranges are supported only with LoadBalancer service type"
+// +kubebuilder:validation:XValidation:rule="!has(self.networkProfile.loadBalancerClass) || self.controlPlane.service.serviceType == 'LoadBalancer'", message="LoadBalancerClass is supported only with LoadBalancer service type"
+// +kubebuilder:validation:XValidation:rule="self.controlPlane.service.serviceType != 'LoadBalancer' || (oldSelf.controlPlane.service.serviceType != 'LoadBalancer' && self.controlPlane.service.serviceType == 'LoadBalancer') || has(self.networkProfile.loadBalancerClass) == has(oldSelf.networkProfile.loadBalancerClass)",message="LoadBalancerClass cannot be set or unset at runtime"
+
 type TenantControlPlaneSpec struct {
-	// DataStore allows to specify a DataStore that should be used to store the Kubernetes data for the given Tenant Control Plane.
-	// This parameter is optional and acts as an override over the default one which is used by the Kamaji Operator.
-	// Migration from a different DataStore to another one is not yet supported and the reconciliation will be blocked.
-	DataStore    string       `json:"dataStore,omitempty"`
-	ControlPlane ControlPlane `json:"controlPlane"`
+	// DataStore specifies the DataStore that should be used to store the Kubernetes data for the given Tenant Control Plane.
+	// When Kamaji runs with the default DataStore flag, all empty values will inherit the default value.
+	// By leaving it empty and running Kamaji with no default DataStore flag, it is possible to achieve automatic assignment to a specific DataStore object.
+	//
+	// Migration from one DataStore to another backed by the same Driver is possible. See: https://kamaji.clastix.io/guides/datastore-migration/
+	// Migration from one DataStore to another backed by a different Driver is not supported.
+	DataStore string `json:"dataStore,omitempty"`
+	// DataStoreSchema allows to specify the name of the database (for relational DataStores) or the key prefix (for etcd). This
+	// value is optional and immutable. Note that Kamaji currently doesn't ensure that DataStoreSchema values are unique. It's up
+	// to the user to avoid clashes between different TenantControlPlanes. If not set upon creation, Kamaji will default the
+	// DataStoreSchema by concatenating the namespace and name of the TenantControlPlane.
+	// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="changing the dataStoreSchema is not supported"
+	DataStoreSchema string `json:"dataStoreSchema,omitempty"`
+	// DataStoreUsername allows to specify the username of the database (for relational DataStores). This
+	// value is optional and immutable. Note that Kamaji currently doesn't ensure that DataStoreUsername values are unique. It's up
+	// to the user to avoid clashes between different TenantControlPlanes. If not set upon creation, Kamaji will default the
+	// DataStoreUsername by concatenating the namespace and name of the TenantControlPlane.
+	// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="changing the dataStoreUsername is not supported"
+	DataStoreUsername string       `json:"dataStoreUsername,omitempty"`
+	ControlPlane      ControlPlane `json:"controlPlane"`
 	// Kubernetes specification for tenant control plane
 	Kubernetes KubernetesSpec `json:"kubernetes"`
 	// NetworkProfile specifies how the network is
@@ -250,16 +330,18 @@ type TenantControlPlaneSpec struct {
 	Addons AddonsSpec `json:"addons,omitempty"`
 }
 
-// +kubebuilder:object:root=true
-// +kubebuilder:subresource:status
-// +kubebuilder:subresource:scale:specpath=.spec.controlPlane.deployment.replicas,statuspath=.status.kubernetesResources.deployment.replicas,selectorpath=.status.kubernetesResources.deployment.selector
-// +kubebuilder:resource:shortName=tcp
-// +kubebuilder:printcolumn:name="Version",type="string",JSONPath=".spec.kubernetes.version",description="Kubernetes version"
-// +kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.kubernetesResources.version.status",description="Status"
-// +kubebuilder:printcolumn:name="Control-Plane endpoint",type="string",JSONPath=".status.controlPlaneEndpoint",description="Tenant Control Plane Endpoint (API server)"
-// +kubebuilder:printcolumn:name="Kubeconfig",type="string",JSONPath=".status.kubeconfig.admin.secretName",description="Secret which contains admin kubeconfig"
+//+kubebuilder:object:root=true
+//+kubebuilder:subresource:status
+//+kubebuilder:subresource:scale:specpath=.spec.controlPlane.deployment.replicas,statuspath=.status.kubernetesResources.deployment.replicas,selectorpath=.status.kubernetesResources.deployment.selector
+//+kubebuilder:resource:categories=kamaji,shortName=tcp
+//+kubebuilder:printcolumn:name="Version",type="string",JSONPath=".spec.kubernetes.version",description="Kubernetes version"
+//+kubebuilder:printcolumn:name="Installed Version",type="string",JSONPath=".status.kubernetesResources.version.version",description="The actual installed Kubernetes version from status"
+//+kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.kubernetesResources.version.status",description="Status"
+//+kubebuilder:printcolumn:name="Control-Plane endpoint",type="string",JSONPath=".status.controlPlaneEndpoint",description="Tenant Control Plane Endpoint (API server)"
+//+kubebuilder:printcolumn:name="Kubeconfig",type="string",JSONPath=".status.kubeconfig.admin.secretName",description="Secret which contains admin kubeconfig"
 //+kubebuilder:printcolumn:name="Datastore",type="string",JSONPath=".status.storage.dataStoreName",description="DataStore actually used"
-// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="Age"
+//+kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="Age"
+//+kubebuilder:metadata:annotations={"cert-manager.io/inject-ca-from=kamaji-system/kamaji-serving-cert"}
 
 // TenantControlPlane is the Schema for the tenantcontrolplanes API.
 type TenantControlPlane struct {

api/v1alpha1/tenantcontrolplane_types_test.go

@@ -0,0 +1,78 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package v1alpha1
+
+import (
+	"context"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+var _ = Describe("Cluster controller", func() {
+	var (
+		ctx context.Context
+		tcp *TenantControlPlane
+	)
+
+	BeforeEach(func() {
+		ctx = context.Background()
+		tcp = &TenantControlPlane{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "tcp",
+				Namespace: "default",
+			},
+			Spec: TenantControlPlaneSpec{},
+		}
+	})
+
+	AfterEach(func() {
+		if err := k8sClient.Delete(ctx, tcp); err != nil && !apierrors.IsNotFound(err) {
+			Expect(err).NotTo(HaveOccurred())
+		}
+	})
+
+	Context("LoadBalancer Source Ranges", func() {
+		It("allows creation when no CIDR ranges are provided", func() {
+			tcp.Spec.ControlPlane.Service.ServiceType = ServiceTypeLoadBalancer
+
+			err := k8sClient.Create(ctx, tcp)
+			Expect(err).NotTo(HaveOccurred())
+		})
+
+		It("allows creation with an explicitly empty CIDR list", func() {
+			tcp.Spec.ControlPlane.Service.ServiceType = ServiceTypeLoadBalancer
+			tcp.Spec.NetworkProfile.LoadBalancerSourceRanges = []string{}
+
+			err := k8sClient.Create(ctx, tcp)
+			Expect(err).NotTo(HaveOccurred())
+		})
+
+		It("allows creation when service type is not LoadBalancer and it has an empty CIDR list", func() {
+			tcp.Spec.ControlPlane.Service.ServiceType = ServiceTypeNodePort
+
+			err := k8sClient.Create(ctx, tcp)
+			Expect(err).NotTo(HaveOccurred())
+		})
+
+		It("allows CIDR ranges when service type is LoadBalancer", func() {
+			tcp.Spec.ControlPlane.Service.ServiceType = ServiceTypeLoadBalancer
+			tcp.Spec.NetworkProfile.LoadBalancerSourceRanges = []string{"192.168.0.0/24"}
+
+			err := k8sClient.Create(ctx, tcp)
+			Expect(err).NotTo(HaveOccurred())
+		})
+
+		It("denies CIDR ranges when service type is not LoadBalancer", func() {
+			tcp.Spec.ControlPlane.Service.ServiceType = ServiceTypeNodePort
+			tcp.Spec.NetworkProfile.LoadBalancerSourceRanges = []string{"192.168.0.0/24"}
+
+			err := k8sClient.Create(ctx, tcp)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("LoadBalancer source ranges are supported only with LoadBalancer service type"))
+		})
+	})
+})

api/v1alpha1/types.go

@@ -28,9 +28,10 @@ func (c CGroupDriver) String() string {
 }
 
 const (
-	ServiceTypeLoadBalancer = (ServiceType)(corev1.ServiceTypeLoadBalancer)
-	ServiceTypeClusterIP    = (ServiceType)(corev1.ServiceTypeClusterIP)
-	ServiceTypeNodePort     = (ServiceType)(corev1.ServiceTypeNodePort)
+	ServiceTypeLoadBalancer       = (ServiceType)(corev1.ServiceTypeLoadBalancer)
+	ServiceTypeClusterIP          = (ServiceType)(corev1.ServiceTypeClusterIP)
+	ServiceTypeNodePort           = (ServiceType)(corev1.ServiceTypeNodePort)
+	KubeconfigSecretKeyAnnotation = "kamaji.clastix.io/kubeconfig-secret-key"
 )
 
 // +kubebuilder:validation:Enum=ClusterIP;NodePort;LoadBalancer

api/v1alpha1/validations_test.go

@@ -0,0 +1,177 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package v1alpha1
+
+import (
+	"context"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+var _ = Describe("Datastores validation test", func() {
+	var (
+		ctx context.Context
+		ds  *DataStore
+	)
+
+	BeforeEach(func() {
+		ctx = context.Background()
+		ds = &DataStore{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "ds",
+				Namespace: "default",
+			},
+			Spec: DataStoreSpec{},
+		}
+	})
+
+	AfterEach(func() {
+		if err := k8sClient.Delete(ctx, ds); err != nil && !apierrors.IsNotFound(err) {
+			Expect(err).NotTo(HaveOccurred())
+		}
+	})
+
+	Context("DataStores fields", func() {
+		It("datastores of type ETCD must have their TLS configurations set correctly", func() {
+			ds = &DataStore{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "bad-etcd",
+				},
+				Spec: DataStoreSpec{
+					Driver:    "etcd",
+					Endpoints: []string{"etcd-server:2379"},
+					TLSConfig: &TLSConfig{
+						CertificateAuthority: CertKeyPair{},
+						ClientCertificate:    &ClientCertificate{},
+					},
+				},
+			}
+
+			err := k8sClient.Create(ctx, ds)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("certificateAuthority privateKey must have secretReference or content when driver is etcd"))
+		})
+
+		It("valid ETCD DataStore should be created", func() {
+			var (
+				cert = []byte("cert")
+				key  = []byte("privkey")
+			)
+
+			ds = &DataStore{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "good-etcd",
+				},
+				Spec: DataStoreSpec{
+					Driver:    "etcd",
+					Endpoints: []string{"etcd-server:2379"},
+					TLSConfig: &TLSConfig{
+						CertificateAuthority: CertKeyPair{
+							Certificate: ContentRef{
+								Content: cert,
+							},
+							PrivateKey: &ContentRef{
+								Content: key,
+							},
+						},
+						ClientCertificate: &ClientCertificate{
+							Certificate: ContentRef{
+								Content: cert,
+							},
+							PrivateKey: ContentRef{
+								Content: key,
+							},
+						},
+					},
+				},
+			}
+
+			err := k8sClient.Create(ctx, ds)
+			Expect(err).To(Not(HaveOccurred()))
+		})
+
+		It("datastores of type PostgreSQL must have either basicAuth or tlsConfig", func() {
+			ds = &DataStore{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "bad-pg",
+				},
+				Spec: DataStoreSpec{
+					Driver:    "PostgreSQL",
+					Endpoints: []string{"pg-server:5432"},
+				},
+			}
+
+			err := k8sClient.Create(ctx, ds)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("When driver is not etcd, either tlsConfig or basicAuth must be provided"))
+		})
+
+		It("datastores of type PostgreSQL can have basicAuth", func() {
+			ds = &DataStore{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "good-pg",
+				},
+				Spec: DataStoreSpec{
+					Driver:    "PostgreSQL",
+					Endpoints: []string{"pg-server:5432"},
+					BasicAuth: &BasicAuth{
+						Username: ContentRef{
+							Content: []byte("postgres"),
+						},
+						Password: ContentRef{
+							Content: []byte("postgres"),
+						},
+					},
+				},
+			}
+
+			err := k8sClient.Create(ctx, ds)
+			Expect(err).To(Not(HaveOccurred()))
+		})
+
+		It("datastores of type PostgreSQL must have tlsConfig with proper content", func() {
+			ds = &DataStore{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "bad-pg",
+				},
+				Spec: DataStoreSpec{
+					Driver:    "PostgreSQL",
+					Endpoints: []string{"pg-server:5432"},
+					TLSConfig: &TLSConfig{
+						ClientCertificate: &ClientCertificate{},
+					},
+				},
+			}
+
+			err := k8sClient.Create(context.Background(), ds)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("When driver is not etcd and tlsConfig exists, clientCertificate must be null or contain valid content"))
+		})
+
+		It("datastores of type PostgreSQL need a proper clientCertificate", func() {
+			ds = &DataStore{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "good-pg",
+				},
+				Spec: DataStoreSpec{
+					Driver:    "PostgreSQL",
+					Endpoints: []string{"pg-server:5432"},
+					TLSConfig: &TLSConfig{
+						ClientCertificate: &ClientCertificate{
+							Certificate: ContentRef{
+								Content: []byte("cert"),
+							},
+						},
+					},
+				},
+			}
+
+			err := k8sClient.Create(context.Background(), ds)
+			Expect(err).ToNot(HaveOccurred())
+		})
+	})
+})

api/v1alpha1/zz_generated.deepcopy.go

@@ -1,5 +1,4 @@
 //go:build !ignore_autogenerated
-// +build !ignore_autogenerated
 
 // Copyright 2022 Clastix Labs
 // SPDX-License-Identifier: Apache-2.0
@@ -526,7 +525,11 @@ func (in *DataStoreSpec) DeepCopyInto(out *DataStoreSpec) {
 		*out = new(BasicAuth)
 		(*in).DeepCopyInto(*out)
 	}
-	in.TLSConfig.DeepCopyInto(&out.TLSConfig)
+	if in.TLSConfig != nil {
+		in, out := &in.TLSConfig, &out.TLSConfig
+		*out = new(TLSConfig)
+		(*in).DeepCopyInto(*out)
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataStoreSpec.
@@ -578,6 +581,11 @@ func (in *DatastoreUsedSecret) DeepCopy() *DatastoreUsedSecret {
 func (in *DeploymentSpec) DeepCopyInto(out *DeploymentSpec) {
 	*out = *in
 	out.RegistrySettings = in.RegistrySettings
+	if in.Replicas != nil {
+		in, out := &in.Replicas, &out.Replicas
+		*out = new(int32)
+		**out = **in
+	}
 	if in.NodeSelector != nil {
 		in, out := &in.NodeSelector, &out.NodeSelector
 		*out = make(map[string]string, len(*in))
@@ -616,6 +624,7 @@ func (in *DeploymentSpec) DeepCopyInto(out *DeploymentSpec) {
 		(*in).DeepCopyInto(*out)
 	}
 	in.AdditionalMetadata.DeepCopyInto(&out.AdditionalMetadata)
+	in.PodAdditionalMetadata.DeepCopyInto(&out.PodAdditionalMetadata)
 	if in.AdditionalInitContainers != nil {
 		in, out := &in.AdditionalInitContainers, &out.AdditionalInitContainers
 		*out = make([]v1.Container, len(*in))
@@ -775,6 +784,13 @@ func (in *IngressSpec) DeepCopy() *IngressSpec {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *KonnectivityAgentSpec) DeepCopyInto(out *KonnectivityAgentSpec) {
 	*out = *in
+	if in.Tolerations != nil {
+		in, out := &in.Tolerations, &out.Tolerations
+		*out = make([]v1.Toleration, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
 	if in.ExtraArgs != nil {
 		in, out := &in.ExtraArgs, &out.ExtraArgs
 		*out = make(ExtraArgs, len(*in))
@@ -792,6 +808,22 @@ func (in *KonnectivityAgentSpec) DeepCopy() *KonnectivityAgentSpec {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *KonnectivityAgentStatus) DeepCopyInto(out *KonnectivityAgentStatus) {
+	*out = *in
+	in.ExternalKubernetesObjectStatus.DeepCopyInto(&out.ExternalKubernetesObjectStatus)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KonnectivityAgentStatus.
+func (in *KonnectivityAgentStatus) DeepCopy() *KonnectivityAgentStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(KonnectivityAgentStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *KonnectivityConfigMap) DeepCopyInto(out *KonnectivityConfigMap) {
 	*out = *in
@@ -1089,6 +1121,16 @@ func (in *KubernetesVersion) DeepCopy() *KubernetesVersion {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *NetworkProfileSpec) DeepCopyInto(out *NetworkProfileSpec) {
 	*out = *in
+	if in.LoadBalancerSourceRanges != nil {
+		in, out := &in.LoadBalancerSourceRanges, &out.LoadBalancerSourceRanges
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
+	if in.LoadBalancerClass != nil {
+		in, out := &in.LoadBalancerClass, &out.LoadBalancerClass
+		*out = new(string)
+		**out = **in
+	}
 	if in.CertSANs != nil {
 		in, out := &in.CertSANs, &out.CertSANs
 		*out = make([]string, len(*in))
@@ -1196,7 +1238,11 @@ func (in *StorageStatus) DeepCopy() *StorageStatus {
 func (in *TLSConfig) DeepCopyInto(out *TLSConfig) {
 	*out = *in
 	in.CertificateAuthority.DeepCopyInto(&out.CertificateAuthority)
-	in.ClientCertificate.DeepCopyInto(&out.ClientCertificate)
+	if in.ClientCertificate != nil {
+		in, out := &in.ClientCertificate, &out.ClientCertificate
+		*out = new(ClientCertificate)
+		(*in).DeepCopyInto(*out)
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TLSConfig.

charts/kamaji/.helmignore

@@ -21,3 +21,8 @@
 .idea/
 *.tmproj
 .vscode/
+# Helm source files
+README.md.gotmpl
+.helmignore
+# Build tools
+Makefile

charts/kamaji/Chart.lock

@@ -0,0 +1,6 @@
+dependencies:
+- name: kamaji-etcd
+  repository: https://clastix.github.io/charts
+  version: 0.11.0
+digest: sha256:96b4115b8c02f771f809ec1bed3be3a3903e7e8315d6966aa54b0f73230ea421
+generated: "2025-07-03T09:19:19.835421461+02:00"

charts/kamaji/Chart.yaml

@@ -1,22 +1,50 @@
 apiVersion: v2
-appVersion: v0.3.2
-description: Kamaji deploys and operates Kubernetes at scale with a fraction of the operational burden. Kamaji turns any Kubernetes cluster into an “admin cluster” to orchestrate other Kubernetes clusters called “tenant clusters”. Kamaji is special because the Control Plane components are running in a single pod instead of dedicated machines. This solution makes running multiple Control Planes cheaper and easier to deploy and operate.
+appVersion: latest
+description: Kamaji is the Hosted Control Plane Manager for Kubernetes.
 home: https://github.com/clastix/kamaji
 icon: https://github.com/clastix/kamaji/raw/master/assets/logo-colored.png
 kubeVersion: ">=1.21.0-0"
 maintainers:
 - email: dario@tranchitella.eu
   name: Dario Tranchitella
+  url: https://clastix.io
 - email: me@maxgio.it
   name: Massimiliano Giovagnoli
 - email: me@bsctl.io
   name: Adriano Pezzuto
+  url: https://clastix.io
 name: kamaji
 sources:
 - https://github.com/clastix/kamaji
 type: application
-version: 0.12.3
+version: 0.0.0+latest
+dependencies:
+- name: kamaji-etcd
+  repository: https://clastix.github.io/charts
+  version: ">=0.11.0"
+  condition: kamaji-etcd.deploy
 annotations:
   catalog.cattle.io/certified: partner
   catalog.cattle.io/release-name: kamaji
-  catalog.cattle.io/display-name: Kamaji
+  catalog.cattle.io/display-name: Kamaji
+  artifacthub.io/crds: |
+    - kind: TenantControlPlane
+      version: v1alpha1
+      name: tenantcontrolplanes.kamaji.clastix.io
+      displayName: TenantControlPlane
+      description: TenantControlPlane defines the desired state for a Control Plane backed by Kamaji.
+    - kind: DataStore
+      version: v1alpha1
+      name: datastores.kamaji.clastix.io
+      displayName: DataStore
+      description: DataStores is holding all the required details to communicate with a Datastore, such as etcd, MySQL, PostgreSQL, and NATS.
+  artifacthub.io/links: |
+    - name: CLASTIX
+      url: https://clastix.io
+    - name: support
+      url: https://clastix.io/support
+  artifacthub.io/operator: "true"
+  artifacthub.io/operatorCapabilities: "full lifecycle"
+  artifacthub.io/changes: |
+    - kind: added
+      description: Releasing latest chart at every push

charts/kamaji/README.md

@@ -1,16 +1,16 @@
 # kamaji
 
-![Version: 0.12.3](https://img.shields.io/badge/Version-0.12.3-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v0.3.2](https://img.shields.io/badge/AppVersion-v0.3.2-informational?style=flat-square)
+![Version: 0.0.0+latest](https://img.shields.io/badge/Version-0.0.0+latest-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: latest](https://img.shields.io/badge/AppVersion-latest-informational?style=flat-square)
 
-Kamaji deploys and operates Kubernetes at scale with a fraction of the operational burden. Kamaji turns any Kubernetes cluster into an “admin cluster” to orchestrate other Kubernetes clusters called “tenant clusters”. Kamaji is special because the Control Plane components are running in a single pod instead of dedicated machines. This solution makes running multiple Control Planes cheaper and easier to deploy and operate.
+Kamaji is the Hosted Control Plane Manager for Kubernetes.
 
 ## Maintainers
 
 | Name | Email | Url |
 | ---- | ------ | --- |
-| Dario Tranchitella | <dario@tranchitella.eu> |  |
+| Dario Tranchitella | <dario@tranchitella.eu> | <https://clastix.io> |
 | Massimiliano Giovagnoli | <me@maxgio.it> |  |
-| Adriano Pezzuto | <me@bsctl.io> |  |
+| Adriano Pezzuto | <me@bsctl.io> | <https://clastix.io> |
 
 ## Source Code
 
@@ -20,6 +20,10 @@ Kamaji deploys and operates Kubernetes at scale with a fraction of the operation
 
 Kubernetes: `>=1.21.0-0`
 
+| Repository | Name | Version |
+|------------|------|---------|
+| https://clastix.github.io/charts | kamaji-etcd | >=0.11.0 |
+
 [Kamaji](https://github.com/clastix/kamaji) requires a [multi-tenant `etcd`](https://github.com/clastix/kamaji-internal/blob/master/deploy/getting-started-with-kamaji.md#setup-internal-multi-tenant-etcd) cluster.
 This Helm Chart starting from v0.1.1 provides the installation of an internal `etcd` in order to streamline the local test. If you'd like to use an externally managed etcd instance, you can specify the overrides and by setting the value `etcd.deploy=false`.
 
@@ -27,9 +31,13 @@ This Helm Chart starting from v0.1.1 provides the installation of an internal `e
 
 ## Install Kamaji
 
+To add clastix helm repository:
+
+        helm repo add clastix https://clastix.github.io/charts
+
 To install the Chart with the release name `kamaji`:
 
-        helm upgrade --install --namespace kamaji-system --create-namespace clastix/kamaji
+        helm upgrade --install --namespace kamaji-system --create-namespace kamaji clastix/kamaji
 
 Show the status:
 
@@ -66,44 +74,7 @@ Here the values you can override:
 | Key | Type | Default | Description |
 |-----|------|---------|-------------|
 | affinity | object | `{}` | Kubernetes affinity rules to apply to Kamaji controller pods |
-| datastore.basicAuth.passwordSecret.keyPath | string | `nil` | The Secret key where the data is stored. |
-| datastore.basicAuth.passwordSecret.name | string | `nil` | The name of the Secret containing the password used to connect to the relational database. |
-| datastore.basicAuth.passwordSecret.namespace | string | `nil` | The namespace of the Secret containing the password used to connect to the relational database. |
-| datastore.basicAuth.usernameSecret.keyPath | string | `nil` | The Secret key where the data is stored. |
-| datastore.basicAuth.usernameSecret.name | string | `nil` | The name of the Secret containing the username used to connect to the relational database. |
-| datastore.basicAuth.usernameSecret.namespace | string | `nil` | The namespace of the Secret containing the username used to connect to the relational database. |
-| datastore.driver | string | `"etcd"` | (string) The Kamaji Datastore driver, supported: etcd, MySQL, PostgreSQL (defaults=etcd). |
-| datastore.endpoints | list | `[]` | (array) List of endpoints of the selected Datastore. When letting the Chart install the etcd datastore, this field is populated automatically. |
-| datastore.nameOverride | string | `nil` | The Datastore name override, if empty defaults to `default` |
-| datastore.tlsConfig.certificateAuthority.certificate.keyPath | string | `nil` | Key of the Secret which contains the content of the certificate. |
-| datastore.tlsConfig.certificateAuthority.certificate.name | string | `nil` | Name of the Secret containing the CA required to establish the mandatory SSL/TLS connection to the datastore. |
-| datastore.tlsConfig.certificateAuthority.certificate.namespace | string | `nil` | Namespace of the Secret containing the CA required to establish the mandatory SSL/TLS connection to the datastore. |
-| datastore.tlsConfig.certificateAuthority.privateKey.keyPath | string | `nil` | Key of the Secret which contains the content of the private key. |
-| datastore.tlsConfig.certificateAuthority.privateKey.name | string | `nil` | Name of the Secret containing the CA private key required to establish the mandatory SSL/TLS connection to the datastore. |
-| datastore.tlsConfig.certificateAuthority.privateKey.namespace | string | `nil` | Namespace of the Secret containing the CA private key required to establish the mandatory SSL/TLS connection to the datastore. |
-| datastore.tlsConfig.clientCertificate.certificate.keyPath | string | `nil` | Key of the Secret which contains the content of the certificate. |
-| datastore.tlsConfig.clientCertificate.certificate.name | string | `nil` | Name of the Secret containing the client certificate required to establish the mandatory SSL/TLS connection to the datastore. |
-| datastore.tlsConfig.clientCertificate.certificate.namespace | string | `nil` | Namespace of the Secret containing the client certificate required to establish the mandatory SSL/TLS connection to the datastore. |
-| datastore.tlsConfig.clientCertificate.privateKey.keyPath | string | `nil` | Key of the Secret which contains the content of the private key. |
-| datastore.tlsConfig.clientCertificate.privateKey.name | string | `nil` | Name of the Secret containing the client certificate private key required to establish the mandatory SSL/TLS connection to the datastore. |
-| datastore.tlsConfig.clientCertificate.privateKey.namespace | string | `nil` | Namespace of the Secret containing the client certificate private key required to establish the mandatory SSL/TLS connection to the datastore. |
-| etcd.compactionInterval | int | `0` | ETCD Compaction interval (e.g. "5m0s"). (default: "0" (disabled)) |
-| etcd.deploy | bool | `true` | Install an etcd with enabled multi-tenancy along with Kamaji |
-| etcd.image | object | `{"pullPolicy":"IfNotPresent","repository":"quay.io/coreos/etcd","tag":"v3.5.6"}` | Install specific etcd image |
-| etcd.livenessProbe | object | `{"failureThreshold":8,"httpGet":{"path":"/health?serializable=true","port":2381,"scheme":"HTTP"},"initialDelaySeconds":10,"periodSeconds":10,"timeoutSeconds":15}` | The livenessProbe for the etcd container |
-| etcd.overrides.caSecret.name | string | `"etcd-certs"` | Name of the secret which contains CA's certificate and private key. (default: "etcd-certs") |
-| etcd.overrides.caSecret.namespace | string | `"kamaji-system"` | Namespace of the secret which contains CA's certificate and private key. (default: "kamaji-system") |
-| etcd.overrides.clientSecret.name | string | `"root-client-certs"` | Name of the secret which contains ETCD client certificates. (default: "root-client-certs") |
-| etcd.overrides.clientSecret.namespace | string | `"kamaji-system"` | Name of the namespace where the secret which contains ETCD client certificates is. (default: "kamaji-system") |
-| etcd.overrides.endpoints | object | `{"etcd-0":"etcd-0.etcd.kamaji-system.svc.cluster.local","etcd-1":"etcd-1.etcd.kamaji-system.svc.cluster.local","etcd-2":"etcd-2.etcd.kamaji-system.svc.cluster.local"}` | (map) Dictionary of the endpoints for the etcd cluster's members, key is the name of the etcd server. Don't define the protocol (TLS is automatically inflected), or any port, inflected from .etcd.peerApiPort value. |
-| etcd.peerApiPort | int | `2380` | The peer API port which servers are listening to. |
-| etcd.persistence.accessModes[0] | string | `"ReadWriteOnce"` |  |
-| etcd.persistence.customAnnotations | object | `{}` | The custom annotations to add to the PVC |
-| etcd.persistence.size | string | `"10Gi"` |  |
-| etcd.persistence.storageClass | string | `""` |  |
-| etcd.port | int | `2379` | The client request port. |
-| etcd.serviceAccount.create | bool | `true` | Create a ServiceAccount, required to install and provision the etcd backing storage (default: true) |
-| etcd.serviceAccount.name | string | `""` | Define the ServiceAccount name to use during the setup and provision of the etcd backing storage (default: "") |
+| defaultDatastoreName | string | `"default"` | If specified, all the Kamaji instances with an unassigned DataStore will inherit this default value. |
 | extraArgs | list | `[]` | A list of extra arguments to add to the kamaji controller default ones |
 | fullnameOverride | string | `""` |  |
 | healthProbeBindAddress | string | `":8081"` | The address the probe endpoint binds to. (default ":8081") |
@@ -111,9 +82,10 @@ Here the values you can override:
 | image.repository | string | `"clastix/kamaji"` | The container image of the Kamaji controller. |
 | image.tag | string | `nil` | Overrides the image tag whose default is the chart appVersion. |
 | imagePullSecrets | list | `[]` |  |
+| kamaji-etcd | object | `{"clusterDomain":"cluster.local","datastore":{"enabled":true,"name":"default"},"deploy":true,"fullnameOverride":"kamaji-etcd"}` | Subchart: See https://github.com/clastix/kamaji-etcd/blob/master/charts/kamaji-etcd/values.yaml |
 | livenessProbe | object | `{"httpGet":{"path":"/healthz","port":"healthcheck"},"initialDelaySeconds":15,"periodSeconds":20}` | The livenessProbe for the controller container |
-| loggingDevel.enable | bool | `false` | (string) Development Mode defaults(encoder=consoleEncoder,logLevel=Debug,stackTraceLevel=Warn). Production Mode defaults(encoder=jsonEncoder,logLevel=Info,stackTraceLevel=Error) (default false) |
-| metricsBindAddress | string | `":8080"` | (string) The address the metric endpoint binds to. (default ":8080") |
+| loggingDevel.enable | bool | `false` | Development Mode defaults(encoder=consoleEncoder,logLevel=Debug,stackTraceLevel=Warn). Production Mode defaults(encoder=jsonEncoder,logLevel=Info,stackTraceLevel=Error) (default false) |
+| metricsBindAddress | string | `":8080"` | The address the metric endpoint binds to. (default ":8080") |
 | nameOverride | string | `""` |  |
 | nodeSelector | object | `{}` | Kubernetes node selector rules to schedule Kamaji controller |
 | podAnnotations | object | `{}` | The annotations to apply to the Kamaji controller pods. |
@@ -129,6 +101,7 @@ Here the values you can override:
 | serviceAccount.create | bool | `true` |  |
 | serviceAccount.name | string | `"kamaji-controller-manager"` |  |
 | serviceMonitor.enabled | bool | `false` | Toggle the ServiceMonitor true if you have Prometheus Operator installed and configured |
+| telemetry | object | `{"disabled":false}` | Disable the analytics traces collection |
 | temporaryDirectoryPath | string | `"/tmp/kamaji"` | Directory which will be used to work with temporary files. (default "/tmp/kamaji") |
 | tolerations | list | `[]` | Kubernetes node taints that the Kamaji controller pods would tolerate |
 

charts/kamaji/README.md.gotmpl

@@ -18,10 +18,15 @@ This Helm Chart starting from v0.1.1 provides the installation of an internal `e
 
 ## Install Kamaji
 
+To add clastix helm repository:
+
+
+        helm repo add clastix https://clastix.github.io/charts
+
 To install the Chart with the release name `kamaji`:
 
 
-        helm upgrade --install --namespace kamaji-system --create-namespace clastix/kamaji
+        helm upgrade --install --namespace kamaji-system --create-namespace kamaji clastix/kamaji
 
 Show the status:
 

charts/kamaji/app-readme.md

@@ -1,12 +0,0 @@
-# Kamaji
-
-Kamaji deploys and operates Kubernetes at scale with a fraction of the operational burden.
-
-Useful links:
-- [Kamaji Github repository](https://github.com/clastix/kamaji)
-- [Kamaji Documentation](https://kamaji.clastix.io)
-
-## Requirements
-
-* Kubernetes v1.22+
-* Helm v3

charts/kamaji/controller-gen/clusterrole.yaml

@@ -0,0 +1,76 @@
+- apiGroups:
+    - apps
+  resources:
+    - deployments
+  verbs:
+    - create
+    - delete
+    - get
+    - list
+    - patch
+    - update
+    - watch
+- apiGroups:
+    - batch
+  resources:
+    - jobs
+  verbs:
+    - create
+    - delete
+    - get
+    - list
+    - watch
+- apiGroups:
+    - ""
+  resources:
+    - configmaps
+    - secrets
+    - services
+  verbs:
+    - create
+    - delete
+    - get
+    - list
+    - patch
+    - update
+    - watch
+- apiGroups:
+    - kamaji.clastix.io
+  resources:
+    - datastores
+    - tenantcontrolplanes
+  verbs:
+    - create
+    - delete
+    - get
+    - list
+    - patch
+    - update
+    - watch
+- apiGroups:
+    - kamaji.clastix.io
+  resources:
+    - datastores/status
+    - tenantcontrolplanes/status
+  verbs:
+    - get
+    - patch
+    - update
+- apiGroups:
+    - kamaji.clastix.io
+  resources:
+    - tenantcontrolplanes/finalizers
+  verbs:
+    - update
+- apiGroups:
+    - networking.k8s.io
+  resources:
+    - ingresses
+  verbs:
+    - create
+    - delete
+    - get
+    - list
+    - patch
+    - update
+    - watch

charts/kamaji/controller-gen/crd-conversion.yaml

@@ -0,0 +1,11 @@
+spec:
+  conversion:
+    strategy: Webhook
+    webhook:
+      clientConfig:
+        service:
+          name: kamaji-webhook-service
+          namespace: kamaji-system
+          path: /convert
+      conversionReviewVersions:
+        - v1

charts/kamaji/controller-gen/mutating-webhook.yaml

@@ -0,0 +1,20 @@
+- admissionReviewVersions:
+    - v1
+  clientConfig:
+    service:
+      name: '{{ include "kamaji.webhookServiceName" . }}'
+      namespace: '{{ .Release.Namespace }}'
+      path: /mutate-kamaji-clastix-io-v1alpha1-tenantcontrolplane
+  failurePolicy: Fail
+  name: mtenantcontrolplane.kb.io
+  rules:
+    - apiGroups:
+        - kamaji.clastix.io
+      apiVersions:
+        - v1alpha1
+      operations:
+        - CREATE
+        - UPDATE
+      resources:
+        - tenantcontrolplanes
+  sideEffects: None

charts/kamaji/controller-gen/validating-webhook.yaml

@@ -0,0 +1,81 @@
+- admissionReviewVersions:
+    - v1
+  clientConfig:
+    service:
+      name: '{{ include "kamaji.webhookServiceName" . }}'
+      namespace: '{{ .Release.Namespace }}'
+      path: /telemetry
+  failurePolicy: Ignore
+  name: telemetry.kamaji.clastix.io
+  rules:
+    - apiGroups:
+        - kamaji.clastix.io
+      apiVersions:
+        - v1alpha1
+      operations:
+        - CREATE
+        - UPDATE
+        - DELETE
+      resources:
+        - tenantcontrolplanes
+  sideEffects: None
+- admissionReviewVersions:
+    - v1
+  clientConfig:
+    service:
+      name: '{{ include "kamaji.webhookServiceName" . }}'
+      namespace: '{{ .Release.Namespace }}'
+      path: /validate-kamaji-clastix-io-v1alpha1-datastore
+  failurePolicy: Fail
+  name: vdatastore.kb.io
+  rules:
+    - apiGroups:
+        - kamaji.clastix.io
+      apiVersions:
+        - v1alpha1
+      operations:
+        - CREATE
+        - UPDATE
+        - DELETE
+      resources:
+        - datastores
+  sideEffects: None
+- admissionReviewVersions:
+    - v1
+  clientConfig:
+    service:
+      name: '{{ include "kamaji.webhookServiceName" . }}'
+      namespace: '{{ .Release.Namespace }}'
+      path: /validate--v1-secret
+  failurePolicy: Ignore
+  name: vdatastoresecrets.kb.io
+  rules:
+    - apiGroups:
+        - ""
+      apiVersions:
+        - v1
+      operations:
+        - DELETE
+      resources:
+        - secrets
+  sideEffects: None
+- admissionReviewVersions:
+    - v1
+  clientConfig:
+    service:
+      name: '{{ include "kamaji.webhookServiceName" . }}'
+      namespace: '{{ .Release.Namespace }}'
+      path: /validate-kamaji-clastix-io-v1alpha1-tenantcontrolplane
+  failurePolicy: Fail
+  name: vtenantcontrolplane.kb.io
+  rules:
+    - apiGroups:
+        - kamaji.clastix.io
+      apiVersions:
+        - v1alpha1
+      operations:
+        - CREATE
+        - UPDATE
+      resources:
+        - tenantcontrolplanes
+  sideEffects: None

charts/kamaji/crds/kamaji.clastix.io_datastores.yaml

@@ -4,7 +4,7 @@ kind: CustomResourceDefinition
 metadata:
   annotations:
     cert-manager.io/inject-ca-from: kamaji-system/kamaji-serving-cert
-    controller-gen.kubebuilder.io/version: v0.11.4
+    controller-gen.kubebuilder.io/version: v0.16.1
   name: datastores.kamaji.clastix.io
 spec:
   group: kamaji.clastix.io
@@ -30,10 +30,19 @@ spec:
           description: DataStore is the Schema for the datastores API.
           properties:
             apiVersion:
-              description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              description: |-
+                APIVersion defines the versioned schema of this representation of an object.
+                Servers should convert recognized schemas to the latest internal value, and
+                may reject unrecognized values.
+                More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
               type: string
             kind:
-              description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              description: |-
+                Kind is a string value representing the REST resource this object represents.
+                Servers may infer this from the endpoint the client submits requests to.
+                Cannot be updated.
+                In CamelCase.
+                More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
               type: string
             metadata:
               type: object
@@ -41,18 +50,24 @@ spec:
               description: DataStoreSpec defines the desired state of DataStore.
               properties:
                 basicAuth:
-                  description: In case of authentication enabled for the given data store, specifies the username and password pair. This value is optional.
+                  description: |-
+                    In case of authentication enabled for the given data store, specifies the username and password pair.
+                    This value is optional.
                   properties:
                     password:
                       properties:
                         content:
-                          description: Bare content of the file, base64 encoded. It has precedence over the SecretReference value.
+                          description: |-
+                            Bare content of the file, base64 encoded.
+                            It has precedence over the SecretReference value.
                           format: byte
                           type: string
                         secretReference:
                           properties:
                             keyPath:
-                              description: Name of the key for the given Secret reference where the content is stored. This value is mandatory.
+                              description: |-
+                                Name of the key for the given Secret reference where the content is stored.
+                                This value is mandatory.
                               minLength: 1
                               type: string
                             name:
@@ -69,13 +84,17 @@ spec:
                     username:
                       properties:
                         content:
-                          description: Bare content of the file, base64 encoded. It has precedence over the SecretReference value.
+                          description: |-
+                            Bare content of the file, base64 encoded.
+                            It has precedence over the SecretReference value.
                           format: byte
                           type: string
                         secretReference:
                           properties:
                             keyPath:
-                              description: Name of the key for the given Secret reference where the content is stored. This value is mandatory.
+                              description: |-
+                                Name of the key for the given Secret reference where the content is stored.
+                                This value is mandatory.
                               minLength: 1
                               type: string
                             name:
@@ -99,29 +118,43 @@ spec:
                     - etcd
                     - MySQL
                     - PostgreSQL
+                    - NATS
                   type: string
+                  x-kubernetes-validations:
+                    - message: Datastore driver is immutable
+                      rule: self == oldSelf
                 endpoints:
-                  description: List of the endpoints to connect to the shared datastore. No need for protocol, just bare IP/FQDN and port.
+                  description: |-
+                    List of the endpoints to connect to the shared datastore.
+                    No need for protocol, just bare IP/FQDN and port.
                   items:
                     type: string
                   minItems: 1
                   type: array
                 tlsConfig:
-                  description: Defines the TLS/SSL configuration required to connect to the data store in a secure way.
+                  description: |-
+                    Defines the TLS/SSL configuration required to connect to the data store in a secure way.
+                    This value is optional.
                   properties:
                     certificateAuthority:
-                      description: Retrieve the Certificate Authority certificate and private key, such as bare content of the file, or a SecretReference. The key reference is required since etcd authentication is based on certificates, and Kamaji is responsible in creating this.
+                      description: |-
+                        Retrieve the Certificate Authority certificate and private key, such as bare content of the file, or a SecretReference.
+                        The key reference is required since etcd authentication is based on certificates, and Kamaji is responsible in creating this.
                       properties:
                         certificate:
                           properties:
                             content:
-                              description: Bare content of the file, base64 encoded. It has precedence over the SecretReference value.
+                              description: |-
+                                Bare content of the file, base64 encoded.
+                                It has precedence over the SecretReference value.
                               format: byte
                               type: string
                             secretReference:
                               properties:
                                 keyPath:
-                                  description: Name of the key for the given Secret reference where the content is stored. This value is mandatory.
+                                  description: |-
+                                    Name of the key for the given Secret reference where the content is stored.
+                                    This value is mandatory.
                                   minLength: 1
                                   type: string
                                 name:
@@ -138,13 +171,17 @@ spec:
                         privateKey:
                           properties:
                             content:
-                              description: Bare content of the file, base64 encoded. It has precedence over the SecretReference value.
+                              description: |-
+                                Bare content of the file, base64 encoded.
+                                It has precedence over the SecretReference value.
                               format: byte
                               type: string
                             secretReference:
                               properties:
                                 keyPath:
-                                  description: Name of the key for the given Secret reference where the content is stored. This value is mandatory.
+                                  description: |-
+                                    Name of the key for the given Secret reference where the content is stored.
+                                    This value is mandatory.
                                   minLength: 1
                                   type: string
                                 name:
@@ -167,13 +204,17 @@ spec:
                         certificate:
                           properties:
                             content:
-                              description: Bare content of the file, base64 encoded. It has precedence over the SecretReference value.
+                              description: |-
+                                Bare content of the file, base64 encoded.
+                                It has precedence over the SecretReference value.
                               format: byte
                               type: string
                             secretReference:
                               properties:
                                 keyPath:
-                                  description: Name of the key for the given Secret reference where the content is stored. This value is mandatory.
+                                  description: |-
+                                    Name of the key for the given Secret reference where the content is stored.
+                                    This value is mandatory.
                                   minLength: 1
                                   type: string
                                 name:
@@ -190,13 +231,17 @@ spec:
                         privateKey:
                           properties:
                             content:
-                              description: Bare content of the file, base64 encoded. It has precedence over the SecretReference value.
+                              description: |-
+                                Bare content of the file, base64 encoded.
+                                It has precedence over the SecretReference value.
                               format: byte
                               type: string
                             secretReference:
                               properties:
                                 keyPath:
-                                  description: Name of the key for the given Secret reference where the content is stored. This value is mandatory.
+                                  description: |-
+                                    Name of the key for the given Secret reference where the content is stored.
+                                    This value is mandatory.
                                   minLength: 1
                                   type: string
                                 name:
@@ -216,13 +261,26 @@ spec:
                       type: object
                   required:
                     - certificateAuthority
-                    - clientCertificate
                   type: object
               required:
                 - driver
                 - endpoints
-                - tlsConfig
               type: object
+              x-kubernetes-validations:
+                - message: certificateAuthority privateKey must have secretReference or content when driver is etcd
+                  rule: '(self.driver == "etcd") ? (self.tlsConfig != null && (has(self.tlsConfig.certificateAuthority.privateKey.secretReference) || has(self.tlsConfig.certificateAuthority.privateKey.content))) : true'
+                - message: clientCertificate must have secretReference or content when driver is etcd
+                  rule: '(self.driver == "etcd") ? (self.tlsConfig != null && (has(self.tlsConfig.clientCertificate.certificate.secretReference) || has(self.tlsConfig.clientCertificate.certificate.content))) : true'
+                - message: clientCertificate privateKey must have secretReference or content when driver is etcd
+                  rule: '(self.driver == "etcd") ? (self.tlsConfig != null && (has(self.tlsConfig.clientCertificate.privateKey.secretReference) || has(self.tlsConfig.clientCertificate.privateKey.content))) : true'
+                - message: When driver is not etcd and tlsConfig exists, clientCertificate must be null or contain valid content
+                  rule: '(self.driver != "etcd" && has(self.tlsConfig) && has(self.tlsConfig.clientCertificate)) ? (((has(self.tlsConfig.clientCertificate.certificate.secretReference) || has(self.tlsConfig.clientCertificate.certificate.content)))) : true'
+                - message: When driver is not etcd and basicAuth exists, username must have secretReference or content
+                  rule: '(self.driver != "etcd" && has(self.basicAuth)) ? ((has(self.basicAuth.username.secretReference) || has(self.basicAuth.username.content))) : true'
+                - message: When driver is not etcd and basicAuth exists, password must have secretReference or content
+                  rule: '(self.driver != "etcd" && has(self.basicAuth)) ? ((has(self.basicAuth.password.secretReference) || has(self.basicAuth.password.content))) : true'
+                - message: When driver is not etcd, either tlsConfig or basicAuth must be provided
+                  rule: '(self.driver != "etcd") ? (has(self.tlsConfig) || has(self.basicAuth)) : true'
             status:
               description: DataStoreStatus defines the observed state of DataStore.
               properties:

charts/kamaji/templates/_helpers_datastore.tpl

@@ -1,90 +0,0 @@
-{{/*
-Create a default fully qualified datastore name.
-*/}}
-{{- define "datastore.fullname" -}}
-{{- default "default" .Values.datastore.nameOverride | trunc 63 | trimSuffix "-" }}
-{{- end }}
-
-{{/*
-Common labels
-*/}}
-{{- define "datastore.labels" -}}
-kamaji.clastix.io/datastore: {{ .Values.datastore.driver }}
-helm.sh/chart: {{ include "kamaji.chart" . }}
-{{ include "kamaji.selectorLabels" . }}
-{{- if .Chart.AppVersion }}
-app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
-{{- end }}
-app.kubernetes.io/managed-by: {{ .Release.Service }}
-{{- end }}
-
-{{/*
-Datastore endpoints, in case of ETCD, retrieving the one provided by the chart.
-*/}}
-{{- define "datastore.endpoints" -}}
-{{- if eq .Values.datastore.driver "etcd" }}
-{{ include "etcd.endpoints" . }}
-{{- else }}
-{{ .Values.datastore.endpoints }}
-{{- end }}
-{{- end }}
-
-{{/*
-The Certificate Authority section for the DataSource object.
-*/}}
-{{- define "datastore.certificateAuthority" -}}
-{{- if eq .Values.datastore.driver "etcd" }}
-certificate:
-  secretReference:
-    name: {{ include "etcd.caSecretName" . }}
-    namespace: {{ include "etcd.caSecretNamespace" . }}
-    keyPath: ca.crt
-privateKey:
-  secretReference:
-    name: {{ include "etcd.caSecretName" . }}
-    namespace: {{ include "etcd.caSecretNamespace" . }}
-    keyPath: ca.key
-{{- else }}
-certificate:
-  secretReference:
-    name: {{ .Values.datastore.tlsConfig.certificateAuthority.certificate.name }}
-    namespace: {{ .Values.datastore.tlsConfig.certificateAuthority.certificate.namespace }}
-    keyPath: {{ .Values.datastore.tlsConfig.certificateAuthority.certificate.keyPath }}
-{{- if .Values.datastore.tlsConfig.certificateAuthority.privateKey.name }}
-privateKey:
-  secretReference:
-    name: {{ .Values.datastore.tlsConfig.certificateAuthority.privateKey.name }}
-    namespace: {{ .Values.datastore.tlsConfig.certificateAuthority.privateKey.namespace }}
-    keyPath: {{ .Values.datastore.tlsConfig.certificateAuthority.privateKey.keyPath }}
-{{- end }}
-{{- end }}
-{{- end }}
-
-{{/*
-The Client Certificate section for the DataSource object.
-*/}}
-{{- define "datastore.clientCertificate" -}}
-{{- if eq .Values.datastore.driver "etcd" }}
-certificate:
-    secretReference:
-      name: {{ include "etcd.clientSecretName" . }}
-      namespace: {{ include "etcd.clientSecretNamespace" . }}
-      keyPath: tls.crt
-privateKey:
-    secretReference:
-      name: {{ include "etcd.clientSecretName" . }}
-      namespace: {{ include "etcd.clientSecretNamespace" . }}
-      keyPath: tls.key
-{{- else }}
-certificate:
-  secretReference:
-    name: {{ .Values.datastore.tlsConfig.clientCertificate.certificate.name }}
-    namespace: {{ .Values.datastore.tlsConfig.clientCertificate.certificate.namespace }}
-    keyPath: {{ .Values.datastore.tlsConfig.clientCertificate.certificate.keyPath }}
-privateKey:
-  secretReference:
-    name: {{ .Values.datastore.tlsConfig.clientCertificate.privateKey.name }}
-    namespace: {{ .Values.datastore.tlsConfig.clientCertificate.privateKey.namespace }}
-    keyPath: {{ .Values.datastore.tlsConfig.clientCertificate.privateKey.keyPath }}
-{{- end }}
-{{- end }}

charts/kamaji/templates/_helpers_etcd.tpl

@@ -1,142 +0,0 @@
-{{/*
-Create a default fully qualified etcd name.
-*/}}
-{{- define "etcd.fullname" -}}
-{{- printf "etcd" }}
-{{- end }}
-
-{{/*
-Create the name of the service account to use
-*/}}
-{{- define "etcd.serviceAccountName" -}}
-{{- if .Values.etcd.serviceAccount.create }}
-{{- default (include "etcd.fullname" .) .Values.etcd.serviceAccount.name }}
-{{- else }}
-{{- default "default" .Values.etcd.serviceAccount.name }}
-{{- end }}
-{{- end }}
-
-{{/*
-Create the name of the Service to use
-*/}}
-{{- define "etcd.serviceName" -}}
-{{- printf "%s" (include "etcd.fullname" .) | trunc 63 | trimSuffix "-" }}
-{{- end }}
-
-{{/*
-Common labels
-*/}}
-{{- define "etcd.labels" -}}
-app.kubernetes.io/name: {{ include "kamaji.name" . }}
-app.kubernetes.io/instance: {{ .Release.Name }}
-app.kubernetes.io/components: etcd
-{{- end }}
-
-{{/*
-Selector labels.
-*/}}
-{{- define "etcd.selectorLabels" -}}
-app.kubernetes.io/name: {{ include "kamaji.name" . }}
-app.kubernetes.io/instance: {{ .Release.Name }}
-app.kubernetes.io/component: etcd
-{{- end }}
-
-{{/*
-Name of the etcd CA secret.
-*/}}
-{{- define "etcd.caSecretName" }}
-{{- if .Values.etcd.deploy }}
-{{- printf "%s-%s" (include "etcd.fullname" .) "certs" | trunc 63 | trimSuffix "-" }}
-{{- else }}
-{{- required "A valid .Values.etcd.overrides.caSecret.name required!" .Values.etcd.overrides.caSecret.name }}
-{{- end }}
-{{- end }}
-
-{{/*
-Namespace of the etcd CA secret.
-*/}}
-{{- define "etcd.caSecretNamespace" }}
-{{- if .Values.etcd.deploy }}
-{{- .Release.Namespace }}
-{{- else }}
-{{- required "A valid .Values.etcd.overrides.caSecret.namespace required!" .Values.etcd.overrides.caSecret.namespace }}
-{{- end }}
-{{- end }}
-
-{{/*
-Name of the certificate signing requests for the certificates required by etcd.
-*/}}
-{{- define "etcd.csrConfigMapName" }}
-{{- printf "%s-csr" (include "etcd.fullname" .) }}
-{{- end }}
-
-{{/*
-Name of the etcd root-client secret.
-*/}}
-{{- define "etcd.clientSecretName" }}
-{{- if .Values.etcd.deploy }}
-{{- printf "root-client-certs" }}
-{{- else }}
-{{- required "A valid .Values.etcd.overrides.clientSecret.name required!" .Values.etcd.overrides.clientSecret.name }}
-{{- end }}
-{{- end }}
-
-{{/*
-Namespace of the etcd root-client secret.
-*/}}
-{{- define "etcd.clientSecretNamespace" }}
-{{- if .Values.etcd.deploy }}
-{{- .Release.Namespace }}
-{{- else }}
-{{- required "A valid .Values.etcd.overrides.clientSecret.namespace required!" .Values.etcd.overrides.clientSecret.namespace }}
-{{- end }}
-{{- end }}
-
-{{/*
-Comma separated list of etcd endpoints, using the overrides in case of unmanaged etcd.
-*/}}
-{{- define "etcd.endpoints" }}
-{{- $list := list -}}
-{{- if .Values.etcd.deploy }}
-    {{- range $count := until 3 -}}
-        {{- $list = append $list (printf "%s-%d.%s.%s.svc.cluster.local:%d" "etcd" $count ( include "etcd.serviceName" . ) $.Release.Namespace (int $.Values.etcd.port) ) -}}
-    {{- end }}
-{{- else if .Values.etcd.overrides.endpoints }}
-    {{- range $v := .Values.etcd.overrides.endpoints -}}
-        {{- $list = append $list (printf "%s:%d" $v (int $.Values.etcd.port) ) -}}
-    {{- end -}}
-{{- else if not .Values.etcd.overrides.endpoints }}
-    {{- fail "A valid .Values.etcd.overrides.endpoints required!" }}
-{{- end }}
-{{- $list | toYaml }}
-{{- end }}
-
-{{/*
-Key-value of the etcd peers, using the overrides in case of unmanaged etcd.
-*/}}
-{{- define "etcd.initialCluster" }}
-{{- $list := list -}}
-{{- if .Values.etcd.deploy }}
-    {{- range $i, $count := until 3 -}}
-        {{- $list = append $list ( printf "etcd-%d=https://%s-%d.%s.%s.svc.cluster.local:%d" $i "etcd" $count ( include "etcd.serviceName" . ) $.Release.Namespace (int $.Values.etcd.peerApiPort) ) -}}
-    {{- end }}
-{{- else if .Values.etcd.overrides.endpoints }}
-    {{- range $k, $v := .Values.etcd.overrides.endpoints -}}
-        {{- $list = append $list ( printf "%s=%s:%d" $k $v (int $.Values.etcd.peerApiPort) ) -}}
-    {{- end -}}
-{{- else if not .Values.etcd.overrides.endpoints }}
-    {{- fail "A valid .Values.etcd.overrides.endpoints required!" }}
-{{- end }}
-{{- join "," $list -}}
-{{- end }}
-
-{{/*
-Retrieve the current Kubernetes version to launch a kubectl container with the minimum version skew possible.
-*/}}
-{{- define "etcd.jobsTagKubeVersion" -}}
-{{- if contains "-eks-" .Capabilities.KubeVersion.GitVersion }}
-{{- print "v" .Capabilities.KubeVersion.Major "." (.Capabilities.KubeVersion.Minor | replace "+" "") -}}
-{{- else }}
-{{- print "v" .Capabilities.KubeVersion.Major "." .Capabilities.KubeVersion.Minor -}}
-{{- end }}
-{{- end }}

charts/kamaji/templates/controller.yaml

@@ -19,10 +19,6 @@ spec:
       labels:
         {{- include "kamaji.selectorLabels" . | nindent 8 }}
     spec:
-      {{- with .Values.imagePullSecrets }}
-      imagePullSecrets:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
       securityContext:
         {{- toYaml .Values.podSecurityContext | nindent 8 }}
       serviceAccountName: {{ include "kamaji.serviceAccountName" . }}
@@ -33,15 +29,18 @@ spec:
         - --leader-elect
         - --metrics-bind-address={{ .Values.metricsBindAddress }}
         - --tmp-directory={{ .Values.temporaryDirectoryPath }}
-        - --datastore={{ include "datastore.fullname" . }}
+        {{- if not (eq .Values.defaultDatastoreName "") }}
+        - --datastore={{ .Values.defaultDatastoreName }}
+        {{- end }}
+        {{- if .Values.telemetry.disabled }}
+        - --disable-telemetry
+        {{- end }}
         {{- if .Values.loggingDevel.enable }}
         - --zap-devel
         {{- end }}
         {{- with .Values.extraArgs }}
         {{- toYaml . | nindent 8 }}
         {{- end }}
-        command:
-        - /kamaji
         env:
           - name: POD_NAMESPACE
             valueFrom:

charts/kamaji/templates/datastore.yaml

@@ -1,26 +0,0 @@
-apiVersion: kamaji.clastix.io/v1alpha1
-kind: DataStore
-metadata:
-  name: {{ include "datastore.fullname" . }}
-  annotations:
-    "helm.sh/hook": pre-install
-  labels:
-    {{- include "datastore.labels" . | nindent 4 }}
-spec:
-  driver: {{ .Values.datastore.driver }}
-  endpoints:
-    {{- include "datastore.endpoints" . | indent 4 }}
-{{- if (and .Values.datastore.basicAuth.usernameSecret.name .Values.datastore.basicAuth.passwordSecret.name) }}
-  basicAuth:
-    username:
-      secretReference:
-        {{- .Values.datastore.basicAuth.usernameSecret | toYaml | nindent 8 }}
-    password:
-      secretReference:
-        {{- .Values.datastore.basicAuth.passwordSecret | toYaml | nindent 8 }}
-{{- end }}
-  tlsConfig:
-    certificateAuthority:
-      {{- include "datastore.certificateAuthority" . | indent 6 }}
-    clientCertificate:
-      {{- include "datastore.clientCertificate" . | indent 6 }}

charts/kamaji/templates/etcd_cm.yaml

@@ -1,98 +0,0 @@
-{{- if .Values.etcd.deploy }}
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  labels:
-    {{- include "etcd.labels" . | nindent 4 }}
-  name: {{ include "etcd.csrConfigMapName" . }}
-  namespace: {{ .Release.Namespace }}
-  annotations:
-    "helm.sh/hook": pre-install
-    "helm.sh/hook-weight": "-5"
-    "helm.sh/hook-delete-policy": "hook-succeeded,hook-failed"
-data:
-  ca-csr.json: |-
-    {
-      "CN": "Clastix CA",
-      "key": {
-        "algo": "rsa",
-        "size": 2048
-      },
-      "names": [
-        {
-          "C": "IT",
-          "ST": "Italy",
-          "L": "Milan"
-        }
-      ]
-    }
-  config.json: |-
-    {
-      "signing": {
-        "default": {
-          "expiry": "8760h"
-        },
-        "profiles": {
-          "server-authentication": {
-            "usages": ["signing", "key encipherment", "server auth"],
-            "expiry": "8760h"
-          },
-          "client-authentication": {
-            "usages": ["signing", "key encipherment", "client auth"],
-            "expiry": "8760h"
-          },
-          "peer-authentication": {
-            "usages": ["signing", "key encipherment", "server auth", "client auth"],
-            "expiry": "8760h"
-          }
-        }
-      }
-    }
-  server-csr.json: |-
-    {
-      "CN": "etcd",
-      "key": {
-        "algo": "rsa",
-        "size": 2048
-      },
-      "hosts": [
-{{- range $count := until 3 -}}
-        {{ printf "\"etcd-%d.%s.%s.svc.cluster.local\"," $count (include "etcd.serviceName" .) $.Release.Namespace }}
-{{- end }}
-        "etcd-server.{{ .Release.Namespace }}.svc.cluster.local",
-        "etcd-server.{{ .Release.Namespace }}.svc",
-        "etcd-server",
-        "127.0.0.1"
-      ]
-    }
-  peer-csr.json: |-
-    {
-      "CN": "etcd",
-      "key": {
-        "algo": "rsa",
-        "size": 2048
-      },
-      "hosts": [
-{{- range $count := until 3 -}}
-        {{ printf "\"etcd-%d\"," $count }}
-        {{ printf "\"etcd-%d.%s\"," $count (include "etcd.serviceName" .) }}
-        {{ printf "\"etcd-%d.%s.%s.svc\"," $count (include "etcd.serviceName" .) $.Release.Namespace }}
-        {{ printf "\"etcd-%d.%s.%s.svc.cluster.local\"," $count (include "etcd.serviceName" .) $.Release.Namespace }}
-{{- end }}
-        "127.0.0.1"
-      ]
-    }
-  root-client-csr.json: |-
-    {
-      "CN": "root",
-      "key": {
-        "algo": "rsa",
-        "size": 2048
-      },
-      "names": [
-        {
-          "O": "system:masters"
-        }
-      ]
-    }
-{{- end }}

charts/kamaji/templates/etcd_job_postdelete.yaml

@@ -1,35 +0,0 @@
-{{- if .Values.etcd.deploy }}
-apiVersion: batch/v1
-kind: Job
-metadata:
-  labels:
-    {{- include "etcd.labels" . | nindent 4 }}
-  annotations:
-    "helm.sh/hook": pre-delete
-    "helm.sh/hook-weight": "-5"
-    "helm.sh/hook-delete-policy": "hook-succeeded,hook-failed"
-  name: "{{ .Release.Name }}-etcd-teardown"
-  namespace: {{ .Release.Namespace }}
-spec:
-  template:
-    metadata:
-      name: "{{ .Release.Name }}"
-    spec:
-      serviceAccountName: {{ include "etcd.serviceAccountName" . }}
-      restartPolicy: Never
-      containers:
-        - name: kubectl
-          image: {{ printf "clastix/kubectl:%s" (include "etcd.jobsTagKubeVersion" .) }}
-          command:
-            - kubectl
-            - --namespace={{ .Release.Namespace }}
-            - delete
-            - secret
-            - --ignore-not-found=true
-            - {{ include "etcd.caSecretName" . }}
-            - {{ include "etcd.clientSecretName" . }}
-      {{- with .Values.tolerations }}
-      tolerations:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-{{- end }}

charts/kamaji/templates/etcd_job_postinstall.yaml

@@ -1,70 +0,0 @@
-{{- if .Values.etcd.deploy }}
-apiVersion: batch/v1
-kind: Job
-metadata:
-  labels:
-    {{- include "etcd.labels" . | nindent 4 }}
-  annotations:
-    "helm.sh/hook": post-install
-    "helm.sh/hook-weight": "-5"
-    "helm.sh/hook-delete-policy": "hook-succeeded,hook-failed"
-  name: "{{ .Release.Name }}-etcd-setup"
-  namespace: {{ .Release.Namespace }}
-spec:
-  template:
-    metadata:
-      name: "{{ .Release.Name }}"
-    spec:
-      serviceAccountName: {{ include "etcd.serviceAccountName" . }}
-      restartPolicy: Never
-      initContainers:
-        - name: kubectl
-          image: {{ printf "clastix/kubectl:%s" (include "etcd.jobsTagKubeVersion" .) }}
-          command:
-            - sh
-            - -c
-            - |-
-              kubectl --namespace={{ .Release.Namespace }} rollout status sts/etcd --timeout=300s
-      containers:
-        - command:
-            - bash
-            - -c
-            - |-
-              etcdctl member list -w table &&
-              etcdctl user add --no-password=true root &&
-              etcdctl role add root &&
-              etcdctl user grant-role root root &&
-              etcdctl auth enable
-          env:
-            - name: ETCDCTL_ENDPOINTS
-              value: https://etcd-0.{{ include "etcd.serviceName" . }}.{{ .Release.Namespace }}.svc.cluster.local:2379
-            - name: ETCDCTL_CACERT
-              value: /opt/certs/ca/ca.crt
-            - name: ETCDCTL_CERT
-              value: /opt/certs/root-certs/tls.crt
-            - name: ETCDCTL_KEY
-              value: /opt/certs/root-certs/tls.key
-          image: quay.io/coreos/etcd:v3.5.1
-          imagePullPolicy: Always
-          name: etcd-client
-          volumeMounts:
-            - name: root-certs
-              mountPath: /opt/certs/root-certs
-            - name: certs
-              mountPath: /opt/certs/ca
-      securityContext:
-        runAsUser: 1000
-        runAsGroup: 1000
-        fsGroup: 1000
-      volumes:
-        - name: root-certs
-          secret:
-            secretName: {{ include "etcd.clientSecretName" . }}
-        - name: certs
-          secret:
-            secretName: {{ include "etcd.caSecretName" . }}
-      {{- with .Values.tolerations }}
-      tolerations:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-{{- end }}

charts/kamaji/templates/etcd_job_preinstall.yaml

@@ -1,64 +0,0 @@
-{{- if .Values.etcd.deploy }}
-apiVersion: batch/v1
-kind: Job
-metadata:
-  labels:
-    {{- include "etcd.labels" . | nindent 4 }}
-  annotations:
-    "helm.sh/hook": pre-install
-    "helm.sh/hook-weight": "-5"
-    "helm.sh/hook-delete-policy": "hook-succeeded"
-  name: "{{ .Release.Name }}-etcd-certs"
-  namespace: {{ .Release.Namespace }}
-spec:
-  template:
-    metadata:
-      name: "{{ .Release.Name }}"
-    spec:
-      serviceAccountName: {{ include "etcd.serviceAccountName" . }}
-      restartPolicy: Never
-      initContainers:
-        - name: cfssl
-          image: cfssl/cfssl:latest
-          command:
-            - bash
-            - -c
-            - |-
-              cfssl gencert -initca /csr/ca-csr.json | cfssljson -bare /certs/ca &&
-              mv /certs/ca.pem /certs/ca.crt && mv /certs/ca-key.pem /certs/ca.key &&
-              cfssl gencert -ca=/certs/ca.crt -ca-key=/certs/ca.key -config=/csr/config.json -profile=peer-authentication /csr/peer-csr.json | cfssljson -bare /certs/peer &&
-              cfssl gencert -ca=/certs/ca.crt -ca-key=/certs/ca.key -config=/csr/config.json -profile=peer-authentication /csr/server-csr.json | cfssljson -bare /certs/server &&
-              cfssl gencert -ca=/certs/ca.crt -ca-key=/certs/ca.key -config=/csr/config.json -profile=client-authentication /csr/root-client-csr.json | cfssljson -bare /certs/root-client
-          volumeMounts:
-            - mountPath: /certs
-              name: certs
-            - mountPath: /csr
-              name: csr
-      containers:
-        - name: kubectl
-          image: {{ printf "clastix/kubectl:%s" (include "etcd.jobsTagKubeVersion" .) }}
-          command:
-            - sh
-            - -c
-            - |-
-              kubectl --namespace={{ .Release.Namespace }} delete secret --ignore-not-found=true {{ include "etcd.caSecretName" . }} {{ include "etcd.clientSecretName" . }} &&
-              kubectl --namespace={{ .Release.Namespace }} create secret generic {{ include "etcd.caSecretName" . }} --from-file=/certs/ca.crt --from-file=/certs/ca.key --from-file=/certs/peer-key.pem --from-file=/certs/peer.pem --from-file=/certs/server-key.pem --from-file=/certs/server.pem &&
-              kubectl --namespace={{ .Release.Namespace }} create secret tls {{ include "etcd.clientSecretName" . }} --key=/certs/root-client-key.pem --cert=/certs/root-client.pem
-          volumeMounts:
-            - mountPath: /certs
-              name: certs
-      securityContext:
-        runAsUser: 1000
-        runAsGroup: 1000
-        fsGroup: 1000
-      volumes:
-        - name: csr
-          configMap:
-            name: {{ include "etcd.csrConfigMapName" . }}
-        - name: certs
-          emptyDir: {}
-      {{- with .Values.tolerations }}
-      tolerations:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-{{- end }}

charts/kamaji/templates/etcd_rbac.yaml

@@ -1,55 +0,0 @@
-{{- if .Values.etcd.deploy }}
-apiVersion: rbac.authorization.k8s.io/v1
-kind: Role
-metadata:
-  labels:
-    {{- include "etcd.labels" . | nindent 4 }}
-  name: etcd-gen-certs-role
-  annotations:
-    "helm.sh/hook": pre-install
-    "helm.sh/hook-weight": "-5"
-  namespace: {{ .Release.Namespace }}
-rules:
-  - apiGroups:
-      - ""
-    resources:
-      - secrets
-    verbs:
-      - delete
-    resourceNames:
-      - {{ include "etcd.caSecretName" . }}
-      - {{ include "etcd.clientSecretName" . }}
-  - apiGroups:
-      - ""
-    resources:
-      - secrets
-    verbs:
-      - create
-  - apiGroups:
-      - apps
-    resources:
-      - statefulsets
-    verbs:
-      - get
-      - list
-      - watch
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  labels:
-    {{- include "etcd.labels" . | nindent 4 }}
-  name: etcd-gen-certs-rolebiding
-  namespace: {{ .Release.Namespace }}
-  annotations:
-    "helm.sh/hook": pre-install
-    "helm.sh/hook-weight": "-5"
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: Role
-  name: etcd-gen-certs-role
-subjects:
-  - kind: ServiceAccount
-    name: {{ include "etcd.serviceAccountName" . }}
-    namespace: {{ .Release.Namespace }}
-{{- end }}

charts/kamaji/templates/etcd_sa.yaml

@@ -1,12 +0,0 @@
-{{- if .Values.etcd.deploy }}
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  labels:
-    {{- include "etcd.labels" . | nindent 4 }}
-  name: {{ include "etcd.serviceAccountName" . }}
-  annotations:
-    "helm.sh/hook": pre-install
-    "helm.sh/hook-weight": "-5"
-  namespace: {{ .Release.Namespace }}
-{{- end }}

charts/kamaji/templates/etcd_service.yaml

@@ -1,18 +0,0 @@
-{{- if .Values.etcd.deploy }}
-apiVersion: v1
-kind: Service
-metadata:
-  labels:
-    {{- include "etcd.labels" . | nindent 4 }}
-  name: {{ include "etcd.serviceName" . }}
-  namespace: {{ .Release.Namespace }}
-spec:
-  clusterIP: None
-  ports:
-    - port: {{ .Values.etcd.port }}
-      name: client
-    - port: {{ .Values.etcd.peerApiPort }}
-      name: peer
-  selector:
-    {{- include "etcd.selectorLabels" . | nindent 4 }}
-{{- end }}

charts/kamaji/templates/etcd_sts.yaml

@@ -1,97 +0,0 @@
-{{- if .Values.etcd.deploy }}
-apiVersion: apps/v1
-kind: StatefulSet
-metadata:
-  labels:
-    {{- include "etcd.labels" . | nindent 4 }}
-  name: {{ include "etcd.fullname" . }}
-  namespace: {{ .Release.Namespace }}
-spec:
-  serviceName: {{ include "etcd.serviceName" . }}
-  selector:
-    matchLabels:
-      {{- include "etcd.selectorLabels" . | nindent 6 }}
-  replicas: 3
-  template:
-    metadata:
-      name: etcd
-      labels:
-        {{- include "etcd.selectorLabels" . | nindent 8 }}
-    spec:
-      volumes:
-        - name: certs
-          secret:
-            secretName: {{ include "etcd.caSecretName" . }}
-      containers:
-        - name: etcd
-          image: {{ .Values.etcd.image.repository }}:{{ .Values.etcd.image.tag | default "v3.5.4" }}
-          imagePullPolicy: {{ .Values.etcd.image.pullPolicy }}
-          ports:
-            - containerPort: 2379
-              name: client
-            - containerPort: 2380
-              name: peer
-          volumeMounts:
-            - name: data
-              mountPath: /var/run/etcd
-            - name: certs
-              mountPath: /etc/etcd/pki
-          command:
-            - etcd
-            - --data-dir=/var/run/etcd
-            - --name=$(POD_NAME)
-            - --initial-cluster-state=new
-            - --initial-cluster={{ include "etcd.initialCluster" . }}
-            - --initial-advertise-peer-urls=https://$(POD_NAME).etcd.$(POD_NAMESPACE).svc.cluster.local:2380
-            - --advertise-client-urls=https://$(POD_NAME).etcd.$(POD_NAMESPACE).svc.cluster.local:2379
-            - --initial-cluster-token=kamaji
-            - --listen-client-urls=https://0.0.0.0:2379
-            - --listen-metrics-urls=http://0.0.0.0:2381
-            - --listen-peer-urls=https://0.0.0.0:2380
-            - --client-cert-auth=true
-            - --peer-client-cert-auth=true
-            - --trusted-ca-file=/etc/etcd/pki/ca.crt
-            - --cert-file=/etc/etcd/pki/server.pem
-            - --key-file=/etc/etcd/pki/server-key.pem
-            - --peer-trusted-ca-file=/etc/etcd/pki/ca.crt
-            - --peer-cert-file=/etc/etcd/pki/peer.pem
-            - --peer-key-file=/etc/etcd/pki/peer-key.pem
-            - --auto-compaction-mode=periodic
-            - --auto-compaction-retention=5m
-            - --snapshot-count=10000
-            - --quota-backend-bytes=8589934592
-            - --v=8
-          env:
-            - name: POD_NAME
-              valueFrom:
-                fieldRef:
-                  fieldPath: metadata.name
-            - name: POD_NAMESPACE
-              valueFrom:
-                fieldRef:
-                  fieldPath: metadata.namespace
-          {{- with .Values.etcd.livenessProbe }}
-          livenessProbe:
-            {{- toYaml . | nindent 12 }}
-          {{- end }}
-          {{- with .Values.etcd.startupProbe }}
-          startupProbe:
-            {{- toYaml . | nindent 12 }}
-          {{- end }}
-  volumeClaimTemplates:
-  - metadata:
-      name: data
-      {{- with .Values.etcd.persistence.customAnnotations }}
-      annotations:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-    spec:
-      storageClassName: {{ .Values.etcd.persistence.storageClassName }}
-      accessModes:
-      {{- range .Values.etcd.persistence.accessModes }}
-      - {{ . | quote }}
-      {{- end }}
-      resources:
-        requests:
-          storage: {{ .Values.etcd.persistence.size }}
-{{- end }}

charts/kamaji/templates/mutatingwebhookconfiguration.yaml

@@ -8,23 +8,4 @@ metadata:
     {{- include "kamaji.labels" $data | nindent 4 }}
   name: kamaji-mutating-webhook-configuration
 webhooks:
-  - admissionReviewVersions:
-      - v1
-    clientConfig:
-      service:
-        name: {{ include "kamaji.webhookServiceName" . }}
-        namespace: {{ .Release.Namespace }}
-        path: /mutate-kamaji-clastix-io-v1alpha1-tenantcontrolplane
-    failurePolicy: Fail
-    name: mtenantcontrolplane.kb.io
-    rules:
-      - apiGroups:
-          - kamaji.clastix.io
-        apiVersions:
-          - v1alpha1
-        operations:
-          - CREATE
-          - UPDATE
-        resources:
-          - tenantcontrolplanes
-    sideEffects: None
+{{ tpl (.Files.Get "controller-gen/mutating-webhook.yaml") . }}

charts/kamaji/templates/rbac.yaml

@@ -9,6 +9,10 @@ metadata:
     {{- toYaml . | nindent 4 }}
   {{- end }}
   namespace: {{ .Release.Namespace }}
+{{- with .Values.imagePullSecrets }}
+imagePullSecrets:
+  {{- toYaml . | nindent 2 }}
+{{- end }}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
@@ -54,122 +58,7 @@ metadata:
   creationTimestamp: null
   name: kamaji-manager-role
 rules:
-- apiGroups:
-  - apps
-  resources:
-  - deployments
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch
-- apiGroups:
-    - batch
-  resources:
-    - jobs
-  verbs:
-    - create
-    - delete
-    - get
-    - list
-    - watch
-- apiGroups:
-  - ""
-  resources:
-  - configmaps
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch
-- apiGroups:
-  - ""
-  resources:
-  - secrets
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch
-- apiGroups:
-  - ""
-  resources:
-  - services
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch
-- apiGroups:
-    - kamaji.clastix.io
-  resources:
-    - datastores
-  verbs:
-    - create
-    - delete
-    - get
-    - list
-    - patch
-    - update
-    - watch
-- apiGroups:
-    - kamaji.clastix.io
-  resources:
-    - datastores/status
-  verbs:
-    - get
-    - patch
-    - update
-- apiGroups:
-  - kamaji.clastix.io
-  resources:
-  - tenantcontrolplanes
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch
-- apiGroups:
-  - kamaji.clastix.io
-  resources:
-  - tenantcontrolplanes/finalizers
-  verbs:
-  - update
-- apiGroups:
-  - kamaji.clastix.io
-  resources:
-  - tenantcontrolplanes/status
-  verbs:
-  - get
-  - patch
-  - update
-- apiGroups:
-  - networking.k8s.io
-  resources:
-  - ingresses
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch
+{{ tpl (.Files.Get "controller-gen/clusterrole.yaml") . }}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole

charts/kamaji/templates/validatingwebhookconfiguration.yaml

@@ -8,63 +8,4 @@ metadata:
     {{- include "kamaji.labels" $data | nindent 4 }}
   name: kamaji-validating-webhook-configuration
 webhooks:
-  - admissionReviewVersions:
-      - v1
-    clientConfig:
-      service:
-        name: {{ include "kamaji.webhookServiceName" . }}
-        namespace: {{ .Release.Namespace }}
-        path: /validate--v1-secret
-    failurePolicy: Ignore
-    name: vdatastoresecrets.kb.io
-    rules:
-      - apiGroups:
-          - ""
-        apiVersions:
-          - v1
-        operations:
-          - DELETE
-        resources:
-          - secrets
-    sideEffects: None
-  - admissionReviewVersions:
-      - v1
-    clientConfig:
-      service:
-        name: {{ include "kamaji.webhookServiceName" . }}
-        namespace: {{ .Release.Namespace }}
-        path: /validate-kamaji-clastix-io-v1alpha1-datastore
-    failurePolicy: Fail
-    name: vdatastore.kb.io
-    rules:
-      - apiGroups:
-          - kamaji.clastix.io
-        apiVersions:
-          - v1alpha1
-        operations:
-          - CREATE
-          - UPDATE
-          - DELETE
-        resources:
-          - datastores
-    sideEffects: None
-  - admissionReviewVersions:
-      - v1
-    clientConfig:
-      service:
-        name: {{ include "kamaji.webhookServiceName" . }}
-        namespace: {{ .Release.Namespace }}
-        path: /validate-kamaji-clastix-io-v1alpha1-tenantcontrolplane
-    failurePolicy: Fail
-    name: vtenantcontrolplane.kb.io
-    rules:
-      - apiGroups:
-          - kamaji.clastix.io
-        apiVersions:
-          - v1alpha1
-        operations:
-          - CREATE
-          - UPDATE
-        resources:
-          - tenantcontrolplanes
-    sideEffects: None
+{{ tpl (.Files.Get "controller-gen/validating-webhook.yaml") . }}

charts/kamaji/values.yaml

@@ -15,71 +15,10 @@ image:
 # -- A list of extra arguments to add to the kamaji controller default ones
 extraArgs: []
 
-
 serviceMonitor:
   # -- Toggle the ServiceMonitor true if you have Prometheus Operator installed and configured
   enabled: false
 
-etcd:
-  # -- Install an etcd with enabled multi-tenancy along with Kamaji
-  deploy: true
-
-  # -- The peer API port which servers are listening to.
-  peerApiPort: 2380
-
-  # -- The client request port.
-  port: 2379
-
-  # -- Install specific etcd image
-  image:
-    repository: quay.io/coreos/etcd
-    tag: "v3.5.6"
-    pullPolicy: IfNotPresent
-
-  # -- The livenessProbe for the etcd container
-  livenessProbe:
-    failureThreshold: 8
-    httpGet:
-      path: /health?serializable=true
-      port: 2381
-      scheme: HTTP
-    initialDelaySeconds: 10
-    periodSeconds: 10
-    timeoutSeconds: 15
-
-  serviceAccount:
-    # -- Create a ServiceAccount, required to install and provision the etcd backing storage (default: true)
-    create: true
-    # -- Define the ServiceAccount name to use during the setup and provision of the etcd backing storage (default: "")
-    name: ""
-  persistence:
-    size: 10Gi
-    storageClass: ""
-    accessModes:
-    - ReadWriteOnce
-    # -- The custom annotations to add to the PVC
-    customAnnotations: {}
-    #  volumeType: local
-
-  overrides:
-    caSecret:
-      # -- Name of the secret which contains CA's certificate and private key. (default: "etcd-certs")
-      name: etcd-certs
-      # -- Namespace of the secret which contains CA's certificate and private key. (default: "kamaji-system")
-      namespace: kamaji-system
-    clientSecret:
-      # -- Name of the secret which contains ETCD client certificates. (default: "root-client-certs")
-      name: root-client-certs
-      # -- Name of the namespace where the secret which contains ETCD client certificates is. (default: "kamaji-system")
-      namespace: kamaji-system
-    # -- (map) Dictionary of the endpoints for the etcd cluster's members, key is the name of the etcd server. Don't define the protocol (TLS is automatically inflected), or any port, inflected from .etcd.peerApiPort value.
-    endpoints:
-      etcd-0: etcd-0.etcd.kamaji-system.svc.cluster.local
-      etcd-1: etcd-1.etcd.kamaji-system.svc.cluster.local
-      etcd-2: etcd-2.etcd.kamaji-system.svc.cluster.local
-  # -- ETCD Compaction interval (e.g. "5m0s"). (default: "0" (disabled))
-  compactionInterval: 0
-
 # -- The address the probe endpoint binds to. (default ":8081")
 healthProbeBindAddress: ":8081"
 
@@ -99,7 +38,7 @@ readinessProbe:
   initialDelaySeconds: 5
   periodSeconds: 10
 
-# -- (string) The address the metric endpoint binds to. (default ":8080")
+# -- The address the metric endpoint binds to. (default ":8080")
 metricsBindAddress: ":8080"
 
 imagePullSecrets: []
@@ -153,59 +92,23 @@ affinity: {}
 temporaryDirectoryPath: "/tmp/kamaji"
 
 loggingDevel:
-  # -- (string) Development Mode defaults(encoder=consoleEncoder,logLevel=Debug,stackTraceLevel=Warn). Production Mode defaults(encoder=jsonEncoder,logLevel=Info,stackTraceLevel=Error) (default false)
+  # -- Development Mode defaults(encoder=consoleEncoder,logLevel=Debug,stackTraceLevel=Warn). Production Mode defaults(encoder=jsonEncoder,logLevel=Info,stackTraceLevel=Error) (default false)
   enable: false
 
-datastore:
-  # -- (string) The Datastore name override, if empty defaults to `default`
-  nameOverride:
-  # -- (string) The Kamaji Datastore driver, supported: etcd, MySQL, PostgreSQL (defaults=etcd).
-  driver: etcd
-  # -- (array) List of endpoints of the selected Datastore. When letting the Chart install the etcd datastore, this field is populated automatically.
-  endpoints: []
-  basicAuth:
-    usernameSecret:
-      # -- The name of the Secret containing the username used to connect to the relational database.
-      name:
-      # -- The namespace of the Secret containing the username used to connect to the relational database.
-      namespace:
-      # -- The Secret key where the data is stored.
-      keyPath:
-    passwordSecret:
-      # -- The name of the Secret containing the password used to connect to the relational database.
-      name:
-      # -- The namespace of the Secret containing the password used to connect to the relational database.
-      namespace:
-      # -- The Secret key where the data is stored.
-      keyPath:
-  tlsConfig:
-    certificateAuthority:
-      certificate:
-        # -- Name of the Secret containing the CA required to establish the mandatory SSL/TLS connection to the datastore.
-        name:
-        # -- Namespace of the Secret containing the CA required to establish the mandatory SSL/TLS connection to the datastore.
-        namespace:
-        # -- Key of the Secret which contains the content of the certificate.
-        keyPath:
-      privateKey:
-        # -- Name of the Secret containing the CA private key required to establish the mandatory SSL/TLS connection to the datastore.
-        name:
-        # -- Namespace of the Secret containing the CA private key required to establish the mandatory SSL/TLS connection to the datastore.
-        namespace:
-        # -- Key of the Secret which contains the content of the private key.
-        keyPath:
-    clientCertificate:
-      certificate:
-        # -- Name of the Secret containing the client certificate required to establish the mandatory SSL/TLS connection to the datastore.
-        name:
-        # -- Namespace of the Secret containing the client certificate required to establish the mandatory SSL/TLS connection to the datastore.
-        namespace:
-        # -- Key of the Secret which contains the content of the certificate.
-        keyPath:
-      privateKey:
-        # -- Name of the Secret containing the client certificate private key required to establish the mandatory SSL/TLS connection to the datastore.
-        name:
-        # -- Namespace of the Secret containing the client certificate private key required to establish the mandatory SSL/TLS connection to the datastore.
-        namespace:
-        # -- Key of the Secret which contains the content of the private key.
-        keyPath:
+# -- If specified, all the Kamaji instances with an unassigned DataStore will inherit this default value.
+defaultDatastoreName: default
+
+# -- Subchart: See https://github.com/clastix/kamaji-etcd/blob/master/charts/kamaji-etcd/values.yaml
+kamaji-etcd:
+  deploy: true
+  fullnameOverride: kamaji-etcd
+  ## -- Important, this must match your management cluster's clusterDomain, otherwise the init jobs will fail
+  clusterDomain: "cluster.local"
+  datastore:
+    enabled: true
+    name: default
+
+# -- Disable the analytics traces collection
+telemetry:
+  disabled: false
+  

cmd/manager/cmd.go

@@ -7,17 +7,24 @@ import (
 	"flag"
 	"fmt"
 	"io"
+	"net/http"
 	"os"
 	goRuntime "runtime"
 	"time"
 
+	telemetryclient "github.com/clastix/kamaji-telemetry/pkg/client"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/rest"
 	"k8s.io/klog/v2"
 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/cache"
+	"sigs.k8s.io/controller-runtime/pkg/event"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
+	ctrlwebhook "sigs.k8s.io/controller-runtime/pkg/webhook"
 
 	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
 	cmdutils "github.com/clastix/kamaji/cmd/utils"
@@ -31,22 +38,26 @@ import (
 	"github.com/clastix/kamaji/internal/webhook/routes"
 )
 
+//nolint:maintidx
 func NewCmd(scheme *runtime.Scheme) *cobra.Command {
 	// CLI flags
 	var (
-		metricsBindAddress         string
-		healthProbeBindAddress     string
-		leaderElect                bool
-		tmpDirectory               string
-		kineImage                  string
-		controllerReconcileTimeout time.Duration
-		datastore                  string
-		managerNamespace           string
-		managerServiceAccountName  string
-		managerServiceName         string
-		webhookCABundle            []byte
-		migrateJobImage            string
-		maxConcurrentReconciles    int
+		metricsBindAddress            string
+		healthProbeBindAddress        string
+		leaderElect                   bool
+		tmpDirectory                  string
+		kineImage                     string
+		controllerReconcileTimeout    time.Duration
+		cacheResyncPeriod             time.Duration
+		datastore                     string
+		managerNamespace              string
+		managerServiceAccountName     string
+		managerServiceName            string
+		webhookCABundle               []byte
+		migrateJobImage               string
+		maxConcurrentReconciles       int
+		disableTelemetry              bool
+		certificateExpirationDeadline time.Duration
 
 		webhookCAPath string
 	)
@@ -58,15 +69,19 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command {
 		Short:         "Start the Kamaji Kubernetes Operator",
 		SilenceErrors: false,
 		SilenceUsage:  true,
-		PreRunE: func(cmd *cobra.Command, args []string) (err error) {
+		PreRunE: func(cmd *cobra.Command, _ []string) (err error) {
 			// Avoid to pollute Kamaji stdout with useless details by the underlying klog implementations
 			klog.SetOutput(io.Discard)
 			klog.LogToStderr(false)
 
-			if err = cmdutils.CheckFlags(cmd.Flags(), []string{"kine-image", "datastore", "migrate-image", "tmp-directory", "pod-namespace", "webhook-service-name", "serviceaccount-name", "webhook-ca-path"}...); err != nil {
+			if err = cmdutils.CheckFlags(cmd.Flags(), []string{"kine-image", "migrate-image", "tmp-directory", "pod-namespace", "webhook-service-name", "serviceaccount-name", "webhook-ca-path"}...); err != nil {
 				return err
 			}
 
+			if certificateExpirationDeadline < 24*time.Hour {
+				return fmt.Errorf("certificate expiration deadline must be at least 24 hours")
+			}
+
 			if webhookCABundle, err = os.ReadFile(webhookCAPath); err != nil {
 				return fmt.Errorf("unable to read webhook CA: %w", err)
 			}
@@ -81,7 +96,7 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command {
 
 			return nil
 		},
-		RunE: func(cmd *cobra.Command, args []string) error {
+		RunE: func(*cobra.Command, []string) error {
 			setupLog := ctrl.Log.WithName("setup")
 
 			setupLog.Info(fmt.Sprintf("Kamaji version %s %s%s", internal.GitTag, internal.GitCommit, internal.GitDirty))
@@ -89,25 +104,42 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command {
 			setupLog.Info(fmt.Sprintf("Build date: %s", internal.BuildTime))
 			setupLog.Info(fmt.Sprintf("Go Version: %s", goRuntime.Version()))
 			setupLog.Info(fmt.Sprintf("Go OS/Arch: %s/%s", goRuntime.GOOS, goRuntime.GOARCH))
+			setupLog.Info(fmt.Sprintf("Telemetry enabled: %t", !disableTelemetry))
 
-			mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
-				Scheme:                  scheme,
-				MetricsBindAddress:      metricsBindAddress,
-				Port:                    9443,
+			telemetryClient := telemetryclient.New(http.Client{Timeout: 5 * time.Second}, "https://telemetry.clastix.io")
+			if disableTelemetry {
+				telemetryClient = telemetryclient.NewNewOp()
+			}
+
+			ctrlOpts := ctrl.Options{
+				Scheme: scheme,
+				Metrics: metricsserver.Options{
+					BindAddress: metricsBindAddress,
+				},
+				WebhookServer: ctrlwebhook.NewServer(ctrlwebhook.Options{
+					Port: 9443,
+				}),
 				HealthProbeBindAddress:  healthProbeBindAddress,
 				LeaderElection:          leaderElect,
 				LeaderElectionNamespace: managerNamespace,
-				LeaderElectionID:        "799b98bc.clastix.io",
-			})
+				LeaderElectionID:        "kamaji.clastix.io",
+				NewCache: func(config *rest.Config, opts cache.Options) (cache.Cache, error) {
+					opts.SyncPeriod = &cacheResyncPeriod
+
+					return cache.New(config, opts)
+				},
+			}
+
+			mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrlOpts)
 			if err != nil {
 				setupLog.Error(err, "unable to start manager")
 
 				return err
 			}
 
-			tcpChannel := make(controllers.TenantControlPlaneChannel)
+			tcpChannel, certChannel := make(chan event.GenericEvent), make(chan event.GenericEvent)
 
-			if err = (&controllers.DataStore{TenantControlPlaneTrigger: tcpChannel}).SetupWithManager(mgr); err != nil {
+			if err = (&controllers.DataStore{Client: mgr.GetClient(), TenantControlPlaneTrigger: tcpChannel}).SetupWithManager(mgr); err != nil {
 				setupLog.Error(err, "unable to create controller", "controller", "DataStore")
 
 				return err
@@ -117,11 +149,13 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command {
 				Client:    mgr.GetClient(),
 				APIReader: mgr.GetAPIReader(),
 				Config: controllers.TenantControlPlaneReconcilerConfig{
-					ReconcileTimeout:     controllerReconcileTimeout,
-					DefaultDataStoreName: datastore,
-					KineContainerImage:   kineImage,
-					TmpBaseDirectory:     tmpDirectory,
+					DefaultDataStoreName:    datastore,
+					KineContainerImage:      kineImage,
+					TmpBaseDirectory:        tmpDirectory,
+					CertExpirationThreshold: certificateExpirationDeadline,
 				},
+				ReconcileTimeout:        controllerReconcileTimeout,
+				CertificateChan:         certChannel,
 				TriggerChan:             tcpChannel,
 				KamajiNamespace:         managerNamespace,
 				KamajiServiceAccount:    managerServiceAccountName,
@@ -136,6 +170,35 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command {
 				return err
 			}
 
+			k8sVersion, versionErr := cmdutils.KubernetesVersion(mgr.GetConfig())
+			if versionErr != nil {
+				setupLog.Error(err, "unable to get kubernetes version")
+
+				k8sVersion = "Unknown"
+			}
+
+			if !disableTelemetry {
+				err = mgr.Add(&controllers.TelemetryController{
+					Client:                  mgr.GetClient(),
+					KubernetesVersion:       k8sVersion,
+					KamajiVersion:           internal.GitTag,
+					TelemetryClient:         telemetryClient,
+					LeaderElectionNamespace: ctrlOpts.LeaderElectionNamespace,
+					LeaderElectionID:        ctrlOpts.LeaderElectionID,
+				})
+				if err != nil {
+					setupLog.Error(err, "unable to create controller", "controller", "TelemetryController")
+
+					return err
+				}
+			}
+
+			if err = (&controllers.CertificateLifecycle{Channel: certChannel, Deadline: certificateExpirationDeadline}).SetupWithManager(mgr); err != nil {
+				setupLog.Error(err, "unable to create controller", "controller", "CertificateLifecycle")
+
+				return err
+			}
+
 			if err = (&kamajiv1alpha1.DatastoreUsedSecret{}).SetupWithManager(ctx, mgr); err != nil {
 				setupLog.Error(err, "unable to create indexer", "indexer", "DatastoreUsedSecret")
 
@@ -153,12 +216,14 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command {
 					handlers.Freeze{},
 				},
 				routes.TenantControlPlaneDefaults{}: {
-					handlers.TenantControlPlaneDefaults{DefaultDatastore: datastore},
+					handlers.TenantControlPlaneDefaults{
+						DefaultDatastore: datastore,
+					},
 				},
 				routes.TenantControlPlaneValidate{}: {
+					handlers.TenantControlPlaneCertSANs{},
 					handlers.TenantControlPlaneName{},
 					handlers.TenantControlPlaneVersion{},
-					handlers.TenantControlPlaneKubeletAddresses{},
 					handlers.TenantControlPlaneDataStore{Client: mgr.GetClient()},
 					handlers.TenantControlPlaneDeployment{
 						Client: mgr.GetClient(),
@@ -170,6 +235,16 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command {
 							Scheme: *mgr.GetScheme(),
 						},
 					},
+					handlers.TenantControlPlaneServiceCIDR{},
+					handlers.TenantControlPlaneLoadBalancerSourceRanges{},
+				},
+				routes.TenantControlPlaneTelemetry{}: {
+					handlers.TenantControlPlaneTelemetry{
+						Enabled:           !disableTelemetry,
+						TelemetryClient:   telemetryClient,
+						KamajiVersion:     internal.GitTag,
+						KubernetesVersion: k8sVersion,
+					},
 				},
 				routes.DataStoreValidate{}: {
 					handlers.DataStoreValidation{Client: mgr.GetClient()},
@@ -230,15 +305,18 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command {
 	cmd.Flags().StringVar(&healthProbeBindAddress, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
 	cmd.Flags().BoolVar(&leaderElect, "leader-elect", true, "Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.")
 	cmd.Flags().StringVar(&tmpDirectory, "tmp-directory", "/tmp/kamaji", "Directory which will be used to work with temporary files.")
-	cmd.Flags().StringVar(&kineImage, "kine-image", "rancher/kine:v0.9.2-amd64", "Container image along with tag to use for the Kine sidecar container (used only if etcd-storage-type is set to one of kine strategies).")
-	cmd.Flags().StringVar(&datastore, "datastore", "etcd", "The default DataStore that should be used by Kamaji to setup the required storage.")
-	cmd.Flags().StringVar(&migrateJobImage, "migrate-image", fmt.Sprintf("clastix/kamaji:v%s", internal.GitTag), "Specify the container image to launch when a TenantControlPlane is migrated to a new datastore.")
+	cmd.Flags().StringVar(&kineImage, "kine-image", "rancher/kine:v0.11.10-amd64", "Container image along with tag to use for the Kine sidecar container (used only if etcd-storage-type is set to one of kine strategies).")
+	cmd.Flags().StringVar(&datastore, "datastore", "", "Optional, the default DataStore that should be used by Kamaji to setup the required storage of Tenant Control Planes with undeclared DataStore.")
+	cmd.Flags().StringVar(&migrateJobImage, "migrate-image", fmt.Sprintf("%s/clastix/kamaji:%s", internal.ContainerRepository, internal.GitTag), "Specify the container image to launch when a TenantControlPlane is migrated to a new datastore.")
 	cmd.Flags().IntVar(&maxConcurrentReconciles, "max-concurrent-tcp-reconciles", 1, "Specify the number of workers for the Tenant Control Plane controller (beware of CPU consumption)")
 	cmd.Flags().StringVar(&managerNamespace, "pod-namespace", os.Getenv("POD_NAMESPACE"), "The Kubernetes Namespace on which the Operator is running in, required for the TenantControlPlane migration jobs.")
 	cmd.Flags().StringVar(&managerServiceName, "webhook-service-name", "kamaji-webhook-service", "The Kamaji webhook server Service name which is used to get validation webhooks, required for the TenantControlPlane migration jobs.")
 	cmd.Flags().StringVar(&managerServiceAccountName, "serviceaccount-name", os.Getenv("SERVICE_ACCOUNT"), "The Kubernetes Namespace on which the Operator is running in, required for the TenantControlPlane migration jobs.")
 	cmd.Flags().StringVar(&webhookCAPath, "webhook-ca-path", "/tmp/k8s-webhook-server/serving-certs/ca.crt", "Path to the Manager webhook server CA, required for the TenantControlPlane migration jobs.")
 	cmd.Flags().DurationVar(&controllerReconcileTimeout, "controller-reconcile-timeout", 30*time.Second, "The reconciliation request timeout before the controller withdraw the external resource calls, such as dealing with the Datastore, or the Tenant Control Plane API endpoint.")
+	cmd.Flags().DurationVar(&cacheResyncPeriod, "cache-resync-period", 10*time.Hour, "The controller-runtime.Manager cache resync period.")
+	cmd.Flags().BoolVar(&disableTelemetry, "disable-telemetry", false, "Disable the analytics traces collection.")
+	cmd.Flags().DurationVar(&certificateExpirationDeadline, "certificate-expiration-deadline", 24*time.Hour, "Define the deadline upon certificate expiration to start the renewal process, cannot be less than a 24 hours.")
 
 	cobra.OnInitialize(func() {
 		viper.AutomaticEnv()

cmd/migrate/cmd.go

@@ -22,16 +22,17 @@ import (
 func NewCmd(scheme *runtime.Scheme) *cobra.Command {
 	// CLI flags
 	var (
-		tenantControlPlane string
-		targetDataStore    string
-		timeout            time.Duration
+		tenantControlPlane    string
+		targetDataStore       string
+		cleanupPriorMigration bool
+		timeout               time.Duration
 	)
 
 	cmd := &cobra.Command{
 		Use:          "migrate",
 		Short:        "Migrate the data of a TenantControlPlane to another compatible DataStore",
 		SilenceUsage: true,
-		RunE: func(cmd *cobra.Command, args []string) error {
+		RunE: func(*cobra.Command, []string) error {
 			ctx, cancelFn := context.WithTimeout(context.Background(), timeout)
 			defer cancelFn()
 
@@ -95,6 +96,20 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command {
 				return err
 			}
 			defer targetConnection.Close()
+
+			if cleanupPriorMigration {
+				log.Info("Checking if target DataStore should be clean-up prior migration")
+
+				if exists, _ := targetConnection.DBExists(ctx, tcp.Status.Storage.Setup.Schema); exists {
+					log.Info("A colliding schema on target DataStore is present, cleaning up")
+
+					if dErr := targetConnection.DeleteDB(ctx, tcp.Status.Storage.Setup.Schema); dErr != nil {
+						return fmt.Errorf("error cleaning up prior migration: %s", dErr.Error())
+					}
+
+					log.Info("Cleaning up prior migration has been completed")
+				}
+			}
 			// Start migrating from the old Datastore to the new one
 			log.Info("migration from origin to target started")
 
@@ -110,6 +125,7 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command {
 
 	cmd.Flags().StringVar(&tenantControlPlane, "tenant-control-plane", "", "Namespaced-name of the TenantControlPlane that must be migrated (e.g.: default/test)")
 	cmd.Flags().StringVar(&targetDataStore, "target-datastore", "", "Name of the Datastore to which the TenantControlPlane will be migrated")
+	cmd.Flags().BoolVar(&cleanupPriorMigration, "cleanup-prior-migration", false, "When set to true, migration job will drop existing data in the target DataStore: useful to avoid stale data when migrating back and forth between DataStores.")
 	cmd.Flags().DurationVar(&timeout, "timeout", 5*time.Minute, "Amount of time for the context timeout")
 
 	_ = cmd.MarkFlagRequired("tenant-control-plane")

cmd/root.go

@@ -4,9 +4,6 @@
 package cmd
 
 import (
-	"math/rand"
-	"time"
-
 	"github.com/spf13/cobra"
 	_ "go.uber.org/automaxprocs" // Automatically set `GOMAXPROCS` to match Linux container CPU quota.
 	"k8s.io/apimachinery/pkg/runtime"
@@ -21,10 +18,7 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command {
 	return &cobra.Command{
 		Use:   "kamaji",
 		Short: "Build and operate Kubernetes at scale with a fraction of operational burden.",
-		PersistentPreRun: func(cmd *cobra.Command, args []string) {
-			// Seed is required to ensure non reproducibility for the certificates generate by Kamaji.
-			rand.Seed(time.Now().UnixNano())
-
+		PersistentPreRun: func(*cobra.Command, []string) {
 			utilruntime.Must(clientgoscheme.AddToScheme(scheme))
 			utilruntime.Must(kamajiv1alpha1.AddToScheme(scheme))
 			utilruntime.Must(appsv1.RegisterDefaults(scheme))

cmd/utils/k8s_version.go

@@ -0,0 +1,24 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package utils
+
+import (
+	"github.com/pkg/errors"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+)
+
+func KubernetesVersion(config *rest.Config) (string, error) {
+	cs, csErr := kubernetes.NewForConfig(config)
+	if csErr != nil {
+		return "", errors.Wrap(csErr, "cannot create kubernetes clientset")
+	}
+
+	sv, svErr := cs.ServerVersion()
+	if svErr != nil {
+		return "", errors.Wrap(svErr, "cannot get Kubernetes version")
+	}
+
+	return sv.GitVersion, nil
+}

config/certmanager/certificate.yaml

@@ -1,39 +0,0 @@
-# The following manifests contain a self-signed issuer CR and a certificate CR.
-# More document can be found at https://docs.cert-manager.io
-# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes.
-apiVersion: cert-manager.io/v1
-kind: Issuer
-metadata:
-  labels:
-    app.kubernetes.io/name: issuer
-    app.kubernetes.io/instance: selfsigned-issuer
-    app.kubernetes.io/component: certificate
-    app.kubernetes.io/created-by: operator
-    app.kubernetes.io/part-of: operator
-    app.kubernetes.io/managed-by: kustomize
-  name: selfsigned-issuer
-  namespace: system
-spec:
-  selfSigned: {}
----
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
-  labels:
-    app.kubernetes.io/name: certificate
-    app.kubernetes.io/instance: serving-cert
-    app.kubernetes.io/component: certificate
-    app.kubernetes.io/created-by: operator
-    app.kubernetes.io/part-of: operator
-    app.kubernetes.io/managed-by: kustomize
-  name: serving-cert  # this name should match the one appeared in kustomizeconfig.yaml
-  namespace: system
-spec:
-  # $(SERVICE_NAME) and $(SERVICE_NAMESPACE) will be substituted by kustomize
-  dnsNames:
-  - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc
-  - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc.cluster.local
-  issuerRef:
-    kind: Issuer
-    name: selfsigned-issuer
-  secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize

config/certmanager/kustomization.yaml

@@ -1,5 +0,0 @@
-resources:
-- certificate.yaml
-
-configurations:
-- kustomizeconfig.yaml

config/certmanager/kustomizeconfig.yaml

@@ -1,16 +0,0 @@
-# This configuration is for teaching kustomize how to update name ref and var substitution 
-nameReference:
-- kind: Issuer
-  group: cert-manager.io
-  fieldSpecs:
-  - kind: Certificate
-    group: cert-manager.io
-    path: spec/issuerRef/name
-
-varReference:
-- kind: Certificate
-  group: cert-manager.io
-  path: spec/commonName
-- kind: Certificate
-  group: cert-manager.io
-  path: spec/dnsNames

config/crd/bases/kamaji.clastix.io_datastores.yaml

@@ -1,278 +0,0 @@
----
-apiVersion: apiextensions.k8s.io/v1
-kind: CustomResourceDefinition
-metadata:
-  annotations:
-    controller-gen.kubebuilder.io/version: v0.11.4
-  name: datastores.kamaji.clastix.io
-spec:
-  group: kamaji.clastix.io
-  names:
-    kind: DataStore
-    listKind: DataStoreList
-    plural: datastores
-    singular: datastore
-  scope: Cluster
-  versions:
-  - additionalPrinterColumns:
-    - description: Kamaji data store driver
-      jsonPath: .spec.driver
-      name: Driver
-      type: string
-    - description: Age
-      jsonPath: .metadata.creationTimestamp
-      name: Age
-      type: date
-    name: v1alpha1
-    schema:
-      openAPIV3Schema:
-        description: DataStore is the Schema for the datastores API.
-        properties:
-          apiVersion:
-            description: 'APIVersion defines the versioned schema of this representation
-              of an object. Servers should convert recognized schemas to the latest
-              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
-            type: string
-          kind:
-            description: 'Kind is a string value representing the REST resource this
-              object represents. Servers may infer this from the endpoint the client
-              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
-            type: string
-          metadata:
-            type: object
-          spec:
-            description: DataStoreSpec defines the desired state of DataStore.
-            properties:
-              basicAuth:
-                description: In case of authentication enabled for the given data
-                  store, specifies the username and password pair. This value is optional.
-                properties:
-                  password:
-                    properties:
-                      content:
-                        description: Bare content of the file, base64 encoded. It
-                          has precedence over the SecretReference value.
-                        format: byte
-                        type: string
-                      secretReference:
-                        properties:
-                          keyPath:
-                            description: Name of the key for the given Secret reference
-                              where the content is stored. This value is mandatory.
-                            minLength: 1
-                            type: string
-                          name:
-                            description: name is unique within a namespace to reference
-                              a secret resource.
-                            type: string
-                          namespace:
-                            description: namespace defines the space within which
-                              the secret name must be unique.
-                            type: string
-                        required:
-                        - keyPath
-                        type: object
-                        x-kubernetes-map-type: atomic
-                    type: object
-                  username:
-                    properties:
-                      content:
-                        description: Bare content of the file, base64 encoded. It
-                          has precedence over the SecretReference value.
-                        format: byte
-                        type: string
-                      secretReference:
-                        properties:
-                          keyPath:
-                            description: Name of the key for the given Secret reference
-                              where the content is stored. This value is mandatory.
-                            minLength: 1
-                            type: string
-                          name:
-                            description: name is unique within a namespace to reference
-                              a secret resource.
-                            type: string
-                          namespace:
-                            description: namespace defines the space within which
-                              the secret name must be unique.
-                            type: string
-                        required:
-                        - keyPath
-                        type: object
-                        x-kubernetes-map-type: atomic
-                    type: object
-                required:
-                - password
-                - username
-                type: object
-              driver:
-                description: The driver to use to connect to the shared datastore.
-                enum:
-                - etcd
-                - MySQL
-                - PostgreSQL
-                type: string
-              endpoints:
-                description: List of the endpoints to connect to the shared datastore.
-                  No need for protocol, just bare IP/FQDN and port.
-                items:
-                  type: string
-                minItems: 1
-                type: array
-              tlsConfig:
-                description: Defines the TLS/SSL configuration required to connect
-                  to the data store in a secure way.
-                properties:
-                  certificateAuthority:
-                    description: Retrieve the Certificate Authority certificate and
-                      private key, such as bare content of the file, or a SecretReference.
-                      The key reference is required since etcd authentication is based
-                      on certificates, and Kamaji is responsible in creating this.
-                    properties:
-                      certificate:
-                        properties:
-                          content:
-                            description: Bare content of the file, base64 encoded.
-                              It has precedence over the SecretReference value.
-                            format: byte
-                            type: string
-                          secretReference:
-                            properties:
-                              keyPath:
-                                description: Name of the key for the given Secret
-                                  reference where the content is stored. This value
-                                  is mandatory.
-                                minLength: 1
-                                type: string
-                              name:
-                                description: name is unique within a namespace to
-                                  reference a secret resource.
-                                type: string
-                              namespace:
-                                description: namespace defines the space within which
-                                  the secret name must be unique.
-                                type: string
-                            required:
-                            - keyPath
-                            type: object
-                            x-kubernetes-map-type: atomic
-                        type: object
-                      privateKey:
-                        properties:
-                          content:
-                            description: Bare content of the file, base64 encoded.
-                              It has precedence over the SecretReference value.
-                            format: byte
-                            type: string
-                          secretReference:
-                            properties:
-                              keyPath:
-                                description: Name of the key for the given Secret
-                                  reference where the content is stored. This value
-                                  is mandatory.
-                                minLength: 1
-                                type: string
-                              name:
-                                description: name is unique within a namespace to
-                                  reference a secret resource.
-                                type: string
-                              namespace:
-                                description: namespace defines the space within which
-                                  the secret name must be unique.
-                                type: string
-                            required:
-                            - keyPath
-                            type: object
-                            x-kubernetes-map-type: atomic
-                        type: object
-                    required:
-                    - certificate
-                    type: object
-                  clientCertificate:
-                    description: Specifies the SSL/TLS key and private key pair used
-                      to connect to the data store.
-                    properties:
-                      certificate:
-                        properties:
-                          content:
-                            description: Bare content of the file, base64 encoded.
-                              It has precedence over the SecretReference value.
-                            format: byte
-                            type: string
-                          secretReference:
-                            properties:
-                              keyPath:
-                                description: Name of the key for the given Secret
-                                  reference where the content is stored. This value
-                                  is mandatory.
-                                minLength: 1
-                                type: string
-                              name:
-                                description: name is unique within a namespace to
-                                  reference a secret resource.
-                                type: string
-                              namespace:
-                                description: namespace defines the space within which
-                                  the secret name must be unique.
-                                type: string
-                            required:
-                            - keyPath
-                            type: object
-                            x-kubernetes-map-type: atomic
-                        type: object
-                      privateKey:
-                        properties:
-                          content:
-                            description: Bare content of the file, base64 encoded.
-                              It has precedence over the SecretReference value.
-                            format: byte
-                            type: string
-                          secretReference:
-                            properties:
-                              keyPath:
-                                description: Name of the key for the given Secret
-                                  reference where the content is stored. This value
-                                  is mandatory.
-                                minLength: 1
-                                type: string
-                              name:
-                                description: name is unique within a namespace to
-                                  reference a secret resource.
-                                type: string
-                              namespace:
-                                description: namespace defines the space within which
-                                  the secret name must be unique.
-                                type: string
-                            required:
-                            - keyPath
-                            type: object
-                            x-kubernetes-map-type: atomic
-                        type: object
-                    required:
-                    - certificate
-                    - privateKey
-                    type: object
-                required:
-                - certificateAuthority
-                - clientCertificate
-                type: object
-            required:
-            - driver
-            - endpoints
-            - tlsConfig
-            type: object
-          status:
-            description: DataStoreStatus defines the observed state of DataStore.
-            properties:
-              usedBy:
-                description: List of the Tenant Control Planes, namespaced named,
-                  using this data store.
-                items:
-                  type: string
-                type: array
-            type: object
-        type: object
-    served: true
-    storage: true
-    subresources:
-      status: {}

config/crd/kustomization.yaml

@@ -1,19 +0,0 @@
-# This kustomization.yaml is not intended to be run by itself,
-# since it depends on service name and namespace that are out of this kustomize package.
-# It should be run by config/default
-resources:
-- bases/kamaji.clastix.io_tenantcontrolplanes.yaml
-- bases/kamaji.clastix.io_datastores.yaml
-#+kubebuilder:scaffold:crdkustomizeresource
-
-patchesStrategicMerge:
-- patches/webhook_in_clusters.yaml
-#+kubebuilder:scaffold:crdkustomizewebhookpatch
-
-- patches/cainjection_in_clusters.yaml
-- patches/cainjection_in_datastores.yaml
-#+kubebuilder:scaffold:crdkustomizecainjectionpatch
-
-# the following config is for teaching kustomize how to do kustomization for CRDs.
-configurations:
-- kustomizeconfig.yaml

config/crd/kustomizeconfig.yaml

@@ -1,19 +0,0 @@
-# This file is for teaching kustomize how to substitute name and namespace reference in CRD
-nameReference:
-- kind: Service
-  version: v1
-  fieldSpecs:
-  - kind: CustomResourceDefinition
-    version: v1
-    group: apiextensions.k8s.io
-    path: spec/conversion/webhook/clientConfig/service/name
-
-namespace:
-- kind: CustomResourceDefinition
-  version: v1
-  group: apiextensions.k8s.io
-  path: spec/conversion/webhook/clientConfig/service/namespace
-  create: false
-
-varReference:
-- path: metadata/annotations

config/crd/patches/cainjection_in_clusters.yaml

@@ -1,7 +0,0 @@
-# The following patch adds a directive for certmanager to inject CA into the CRD
-apiVersion: apiextensions.k8s.io/v1
-kind: CustomResourceDefinition
-metadata:
-  annotations:
-    cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME)
-  name: tenantcontrolplanes.kamaji.clastix.io

config/crd/patches/cainjection_in_datastores.yaml

@@ -1,7 +0,0 @@
-# The following patch adds a directive for certmanager to inject CA into the CRD
-apiVersion: apiextensions.k8s.io/v1
-kind: CustomResourceDefinition
-metadata:
-  annotations:
-    cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME)
-  name: datastores.kamaji.clastix.io

config/crd/patches/webhook_in_clusters.yaml

@@ -1,16 +0,0 @@
-# The following patch enables a conversion webhook for the CRD
-apiVersion: apiextensions.k8s.io/v1
-kind: CustomResourceDefinition
-metadata:
-  name: tenantcontrolplanes.kamaji.clastix.io
-spec:
-  conversion:
-    strategy: Webhook
-    webhook:
-      clientConfig:
-        service:
-          namespace: system
-          name: webhook-service
-          path: /convert
-      conversionReviewVersions:
-      - v1

config/crd/patches/webhook_in_datastores.yaml

@@ -1,16 +0,0 @@
-# The following patch enables a conversion webhook for the CRD
-apiVersion: apiextensions.k8s.io/v1
-kind: CustomResourceDefinition
-metadata:
-  name: datastores.kamaji.clastix.io
-spec:
-  conversion:
-    strategy: Webhook
-    webhook:
-      clientConfig:
-        service:
-          namespace: system
-          name: webhook-service
-          path: /convert
-      conversionReviewVersions:
-      - v1

config/default/kustomization.yaml

@@ -1,58 +0,0 @@
-# Adds namespace to all resources.
-namespace: kamaji-system
-
-# Value of this field is prepended to the
-# names of all resources, e.g. a deployment named
-# "wordpress" becomes "alices-wordpress".
-# Note that it should also match with the prefix (text before '-') of the namespace
-# field above.
-namePrefix: kamaji-
-
-# Labels to add to all resources and selectors.
-#commonLabels:
-#  someName: someValue
-
-bases:
-- ../crd
-- ../rbac
-- ../manager
-- ../samples
-- ../webhook
-- ../certmanager
-- ../prometheus
-
-patchesStrategicMerge:
-# Mount the controller config file for loading manager configurations
-# through a ComponentConfig type
-#- manager_config_patch.yaml
-- manager_webhook_patch.yaml
-- webhookcainjection_patch.yaml
-
-# the following config is for teaching kustomize how to do var substitution
-vars:
-- name: CERTIFICATE_NAMESPACE # namespace of the certificate CR
-  objref:
-    kind: Certificate
-    group: cert-manager.io
-    version: v1
-    name: serving-cert # this name should match the one in certificate.yaml
-  fieldref:
-    fieldpath: metadata.namespace
-- name: CERTIFICATE_NAME
-  objref:
-    kind: Certificate
-    group: cert-manager.io
-    version: v1
-    name: serving-cert # this name should match the one in certificate.yaml
-- name: SERVICE_NAMESPACE # namespace of the service
-  objref:
-    kind: Service
-    version: v1
-    name: webhook-service
-  fieldref:
-    fieldpath: metadata.namespace
-- name: SERVICE_NAME
-  objref:
-    kind: Service
-    version: v1
-    name: webhook-service

config/default/manager_auth_proxy_patch.yaml

@@ -1,28 +0,0 @@
-# This patch inject a sidecar container which is a HTTP proxy for the
-# controller manager, it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews.
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: controller-manager
-  namespace: system
-spec:
-  template:
-    spec:
-      containers:
-      - name: kube-rbac-proxy
-        image: gcr.io/kubebuilder/kube-rbac-proxy:v0.8.0
-        args:
-        - "--secure-listen-address=0.0.0.0:8443"
-        - "--upstream=http://127.0.0.1:8080/"
-        - "--logtostderr=true"
-        - "--v=10"
-        ports:
-        - containerPort: 8443
-          protocol: TCP
-          name: https
-      - name: manager
-        args:
-        - "--health-probe-bind-address=:8081"
-        - "--metrics-bind-address=127.0.0.1:8080"
-        - "--leader-elect"
-        - "--datastore=kamaji-etcd"

config/default/manager_config_patch.yaml

@@ -1,20 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: controller-manager
-  namespace: system
-spec:
-  template:
-    spec:
-      containers:
-      - name: manager
-        args:
-        - "--config=controller_manager_config.yaml"
-        volumeMounts:
-        - name: manager-config
-          mountPath: /controller_manager_config.yaml
-          subPath: controller_manager_config.yaml
-      volumes:
-      - name: manager-config
-        configMap:
-          name: manager-config

config/default/manager_webhook_patch.yaml

@@ -1,23 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: controller-manager
-  namespace: system
-spec:
-  template:
-    spec:
-      containers:
-      - name: manager
-        ports:
-        - containerPort: 9443
-          name: webhook-server
-          protocol: TCP
-        volumeMounts:
-        - mountPath: /tmp/k8s-webhook-server/serving-certs
-          name: cert
-          readOnly: true
-      volumes:
-      - name: cert
-        secret:
-          defaultMode: 420
-          secretName: webhook-server-cert

config/default/webhookcainjection_patch.yaml

@@ -1,29 +0,0 @@
-# This patch add annotation to admission webhook config and
-# the variables $(CERTIFICATE_NAMESPACE) and $(CERTIFICATE_NAME) will be substituted by kustomize.
-apiVersion: admissionregistration.k8s.io/v1
-kind: MutatingWebhookConfiguration
-metadata:
-  labels:
-    app.kubernetes.io/name: mutatingwebhookconfiguration
-    app.kubernetes.io/instance: mutating-webhook-configuration
-    app.kubernetes.io/component: webhook
-    app.kubernetes.io/created-by: operator
-    app.kubernetes.io/part-of: operator
-    app.kubernetes.io/managed-by: kustomize
-  name: mutating-webhook-configuration
-  annotations:
-    cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME)
----
-apiVersion: admissionregistration.k8s.io/v1
-kind: ValidatingWebhookConfiguration
-metadata:
-  labels:
-    app.kubernetes.io/name: validatingwebhookconfiguration
-    app.kubernetes.io/instance: validating-webhook-configuration
-    app.kubernetes.io/component: webhook
-    app.kubernetes.io/created-by: operator
-    app.kubernetes.io/part-of: operator
-    app.kubernetes.io/managed-by: kustomize
-  name: validating-webhook-configuration
-  annotations:
-    cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME)

config/manager/controller_manager_config.yaml

@@ -1,11 +0,0 @@
-apiVersion: controller-runtime.sigs.k8s.io/v1alpha1
-kind: ControllerManagerConfig
-health:
-  healthProbeBindAddress: :8081
-metrics:
-  bindAddress: 127.0.0.1:8080
-webhook:
-  port: 9443
-leaderElection:
-  leaderElect: true
-  resourceName: 799b98bc.clastix.io

config/manager/kustomization.yaml

@@ -1,16 +0,0 @@
-resources:
-- manager.yaml
-
-generatorOptions:
-  disableNameSuffixHash: true
-
-configMapGenerator:
-- files:
-  - controller_manager_config.yaml
-  name: manager-config
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-images:
-- name: controller
-  newName: clastix/kamaji
-  newTag: v0.3.2

config/manager/manager.yaml

@@ -1,71 +0,0 @@
-apiVersion: v1
-kind: Namespace
-metadata:
-  labels:
-    control-plane: controller-manager
-  name: system
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: controller-manager
-  namespace: system
-  labels:
-    control-plane: controller-manager
-spec:
-  selector:
-    matchLabels:
-      control-plane: controller-manager
-  replicas: 1
-  template:
-    metadata:
-      labels:
-        control-plane: controller-manager
-    spec:
-      securityContext:
-        runAsNonRoot: true
-      containers:
-      - command:
-        - /kamaji
-        args:
-        - manager
-        - --leader-elect
-        env:
-          - name: POD_NAMESPACE
-            valueFrom:
-              fieldRef:
-                fieldPath: metadata.namespace
-          - name: SERVICE_ACCOUNT
-            valueFrom:
-              fieldRef:
-                fieldPath: spec.serviceAccountName
-        image: controller:latest
-        imagePullPolicy: Always
-        name: manager
-        ports:
-          - containerPort: 8080
-            name: metrics
-            protocol: TCP
-        securityContext:
-          allowPrivilegeEscalation: false
-        livenessProbe:
-          httpGet:
-            path: /healthz
-            port: 8081
-          initialDelaySeconds: 15
-          periodSeconds: 20
-        readinessProbe:
-          httpGet:
-            path: /readyz
-            port: 8081
-          initialDelaySeconds: 5
-          periodSeconds: 10
-        resources:
-          limits:
-            cpu: 200m
-            memory: 100Mi
-          requests:
-            cpu: 100m
-            memory: 20Mi
-      serviceAccountName: controller-manager
-      terminationGracePeriodSeconds: 10

config/manifests/kustomization.yaml

@@ -1,27 +0,0 @@
-# These resources constitute the fully configured set of manifests
-# used to generate the 'manifests/' directory in a bundle.
-resources:
-- bases/operator.clusterserviceversion.yaml
-- ../default
-- ../samples
-- ../scorecard
-
-# [WEBHOOK] To enable webhooks, uncomment all the sections with [WEBHOOK] prefix.
-# Do NOT uncomment sections with prefix [CERTMANAGER], as OLM does not support cert-manager.
-# These patches remove the unnecessary "cert" volume and its manager container volumeMount.
-#patchesJson6902:
-#- target:
-#    group: apps
-#    version: v1
-#    kind: Deployment
-#    name: controller-manager
-#    namespace: system
-#  patch: |-
-#    # Remove the manager container's "cert" volumeMount, since OLM will create and mount a set of certs.
-#    # Update the indices in this path if adding or removing containers/volumeMounts in the manager's Deployment.
-#    - op: remove
-#      path: /spec/template/spec/containers/1/volumeMounts/0
-#    # Remove the "cert" volume, since OLM will create and mount a set of certs.
-#    # Update the indices in this path if adding or removing volumes in the manager's Deployment.
-#    - op: remove
-#      path: /spec/template/spec/volumes/0

config/prometheus/kustomization.yaml

@@ -1,5 +0,0 @@
-resources:
-- monitor.yaml
-
-configurations:
-- kustomizeconfig.yaml

config/prometheus/kustomizeconfig.yaml

@@ -1,4 +0,0 @@
-varReference:
-  - kind: ServiceMonitor
-    group: monitoring.coreos.com
-    path: spec/namespaceSelector/matchNames

config/prometheus/monitor.yaml

@@ -1,19 +0,0 @@
-# Prometheus Monitor Service (Metrics)
-apiVersion: monitoring.coreos.com/v1
-kind: ServiceMonitor
-metadata:
-  labels:
-    control-plane: controller-manager
-  name: controller-manager-metrics-monitor
-  namespace: system
-spec:
-  endpoints:
-    - path: /metrics
-      port: metrics
-      scheme: http
-  namespaceSelector:
-    matchNames:
-      - $(SERVICE_NAMESPACE)
-  selector:
-    matchLabels:
-      control-plane: controller-manager

config/rbac/auth_proxy_client_clusterrole.yaml

@@ -1,9 +0,0 @@
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: metrics-reader
-rules:
-- nonResourceURLs:
-  - "/metrics"
-  verbs:
-  - get

config/rbac/auth_proxy_role.yaml

@@ -1,17 +0,0 @@
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: proxy-role
-rules:
-- apiGroups:
-  - authentication.k8s.io
-  resources:
-  - tokenreviews
-  verbs:
-  - create
-- apiGroups:
-  - authorization.k8s.io
-  resources:
-  - subjectaccessreviews
-  verbs:
-  - create

config/rbac/auth_proxy_role_binding.yaml

@@ -1,12 +0,0 @@
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: proxy-rolebinding
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: proxy-role
-subjects:
-- kind: ServiceAccount
-  name: controller-manager
-  namespace: system

config/rbac/auth_proxy_service.yaml

@@ -1,15 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  labels:
-    control-plane: controller-manager
-  name: controller-manager-metrics-service
-  namespace: system
-spec:
-  ports:
-  - name: https
-    port: 8443
-    protocol: TCP
-    targetPort: https
-  selector:
-    control-plane: controller-manager

config/rbac/datastore_editor_role.yaml

@@ -1,24 +0,0 @@
-# permissions for end users to edit datastores.
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: datastore-editor-role
-rules:
-- apiGroups:
-  - kamaji.clastix.io
-  resources:
-  - datastores
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch
-- apiGroups:
-  - kamaji.clastix.io
-  resources:
-  - datastores/status
-  verbs:
-  - get

config/rbac/datastore_viewer_role.yaml

@@ -1,20 +0,0 @@
-# permissions for end users to view datastores.
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: datastore-viewer-role
-rules:
-- apiGroups:
-  - kamaji.clastix.io
-  resources:
-  - datastores
-  verbs:
-  - get
-  - list
-  - watch
-- apiGroups:
-  - kamaji.clastix.io
-  resources:
-  - datastores/status
-  verbs:
-  - get

config/rbac/kustomization.yaml

@@ -1,18 +0,0 @@
-resources:
-# All RBAC will be applied under this service account in
-# the deployment namespace. You may comment out this resource
-# if your manager will use a service account that exists at
-# runtime. Be sure to update RoleBinding and ClusterRoleBinding
-# subjects if changing service account names.
-- service_account.yaml
-- role.yaml
-- role_binding.yaml
-- leader_election_role.yaml
-- leader_election_role_binding.yaml
-# Comment the following 4 lines if you want to disable
-# the auth proxy (https://github.com/brancz/kube-rbac-proxy)
-# which protects your /metrics endpoint.
-- auth_proxy_service.yaml
-- auth_proxy_role.yaml
-- auth_proxy_role_binding.yaml
-- auth_proxy_client_clusterrole.yaml

config/rbac/leader_election_role.yaml

@@ -1,37 +0,0 @@
-# permissions to do leader election.
-apiVersion: rbac.authorization.k8s.io/v1
-kind: Role
-metadata:
-  name: leader-election-role
-rules:
-- apiGroups:
-  - ""
-  resources:
-  - configmaps
-  verbs:
-  - get
-  - list
-  - watch
-  - create
-  - update
-  - patch
-  - delete
-- apiGroups:
-  - coordination.k8s.io
-  resources:
-  - leases
-  verbs:
-  - get
-  - list
-  - watch
-  - create
-  - update
-  - patch
-  - delete
-- apiGroups:
-  - ""
-  resources:
-  - events
-  verbs:
-  - create
-  - patch

config/rbac/leader_election_role_binding.yaml

@@ -1,12 +0,0 @@
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: leader-election-rolebinding
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: Role
-  name: leader-election-role
-subjects:
-- kind: ServiceAccount
-  name: controller-manager
-  namespace: system

config/rbac/role.yaml

@@ -1,122 +0,0 @@
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: manager-role
-rules:
-- apiGroups:
-  - apps
-  resources:
-  - deployments
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch
-- apiGroups:
-  - batch
-  resources:
-  - jobs
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - watch
-- apiGroups:
-  - ""
-  resources:
-  - configmaps
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch
-- apiGroups:
-  - ""
-  resources:
-  - secrets
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch
-- apiGroups:
-  - ""
-  resources:
-  - services
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch
-- apiGroups:
-  - kamaji.clastix.io
-  resources:
-  - datastores
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch
-- apiGroups:
-  - kamaji.clastix.io
-  resources:
-  - datastores/status
-  verbs:
-  - get
-  - patch
-  - update
-- apiGroups:
-  - kamaji.clastix.io
-  resources:
-  - tenantcontrolplanes
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch
-- apiGroups:
-  - kamaji.clastix.io
-  resources:
-  - tenantcontrolplanes/finalizers
-  verbs:
-  - update
-- apiGroups:
-  - kamaji.clastix.io
-  resources:
-  - tenantcontrolplanes/status
-  verbs:
-  - get
-  - patch
-  - update
-- apiGroups:
-  - networking.k8s.io
-  resources:
-  - ingresses
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch

config/rbac/role_binding.yaml

@@ -1,12 +0,0 @@
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: manager-rolebinding
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: manager-role
-subjects:
-- kind: ServiceAccount
-  name: controller-manager
-  namespace: system