docs: releasing v0.6.0

Signed-off-by: Dario Tranchitella <dario@tranchitella.eu>

.github/workflows/ci.yaml

@@ -14,7 +14,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-go@v3
         with:
-          go-version: '1.21'
+          go-version: '1.22'
           check-latest: true
       - name: Run golangci-lint
         uses: golangci/golangci-lint-action@v3.2.0
@@ -31,7 +31,7 @@ jobs:
           fetch-depth: 0
       - uses: actions/setup-go@v3
         with:
-          go-version: '1.21'
+          go-version: '1.22'
           check-latest: true
       - run: make yaml-installation-file
       - name: Checking if YAML installer file is not aligned

.github/workflows/e2e.yaml

@@ -38,7 +38,7 @@ jobs:
           fetch-depth: 0
       - uses: actions/setup-go@v3
         with:
-          go-version: '1.21'
+          go-version: '1.22'
           check-latest: true
       - run: |
           sudo apt-get update

.gitignore

@@ -32,5 +32,8 @@ bin
 **/*.key
 **/*.pem
 **/*.csr
-**/server-csr.json
 .DS_Store
+
+**/server-csr.json
+!deploy/kine/mysql/server-csr.json
+!deploy/kine/nats/server-csr.json

.golangci.yml

@@ -37,6 +37,8 @@ linters:
     - funlen
     - dupl
     - cyclop
+    - gocognit
+    - nestif
     # deprecated linters
     - deadcode
     - golint

ADOPTERS.md

@@ -7,10 +7,14 @@ Feel free to open a Pull-Request to get yours listed.
 
 | Type | Name | Since | Website | Use-Case |
 |:-|:-|:-|:-|:-|
+| End-user | Sicuro Tech Lab | 2024 | [link](https://sicurotechlab.it/) | Sicuro Tech Lab offers cloud infrastructure for Web Agencies and uses kamaji to provide managed k8s services. |
+| R&D | TIM | 2024 | [link](https://www.gruppotim.it) | TIM is an Italian telecommunications company using Kamaji for experimental research and development purposes. |
 | End-user | KINX | 2024 | [link](https://kinx.net/?lang=en) | KINX is an Internet infrastructure service provider and will use kamaji for its new [Managed Kubernetes Service](https://kinx.net/service/cloud/kubernetes/intro/?lang=en). |
 | End-user | sevensphere | 2023 | [link](https://www.sevensphere.io) | Sevensphere provides consulting services for end-user companies / cloud providers and uses Kamaji for designing cloud/on-premises Kubernetes-as-a-Service platform. |
 | Vendor | Ænix | 2023 | [link](https://aenix.io/) | Ænix provides consulting services for cloud providers and uses Kamaji for running Kubernetes-as-a-Service in free PaaS platform [Cozystack](https://cozystack.io). |
 | Vendor | Netsons | 2023 | [link](https://www.netsons.com) | Netsons is an Italian hosting and cloud provider and uses Kamaji in its [Managed Kubernetes](https://www.netsons.com/kubernetes) offering. |
+| Vendor | Aknostic | 2023 | [link](https://aknostic.com) | Aknostic is a cloud-native consultancy company using Kamaji to build a Kubernetes based PaaS. |
+
 
 ### Adopter Types
 
@@ -19,3 +23,5 @@ Feel free to open a Pull-Request to get yours listed.
 **Integration**: The organization has a product that integrates with Kamaji, but does not contain Kamaji.
 
 **Vendor**: The organization packages Kamaji in their product and sells it as part of their product.
+
+**R&D**: Company that exploring innovative technologies  and solutions for research and development purposes.

Dockerfile

@@ -1,5 +1,5 @@
 # Build the manager binary
-FROM golang:1.21 as builder
+FROM golang:1.22 as builder
 
 WORKDIR /workspace
 # Copy the Go Modules manifests

Makefile

@@ -3,7 +3,7 @@
 # To re-generate a bundle for another specific version without changing the standard setup, you can:
 # - use the VERSION as arg of the bundle target (e.g make bundle VERSION=0.0.2)
 # - use environment variables to overwrite this value (e.g export VERSION=0.0.2)
-VERSION ?= 0.4.2
+VERSION ?= 0.6.0
 
 # CHANNELS define the bundle channels used in the bundle.
 # Add a new line here if you would like to change its default config. (E.g CHANNELS = "candidate,fast,stable")
@@ -85,7 +85,7 @@ kind: ## Download kind locally if necessary.
 
 CONTROLLER_GEN = $(shell pwd)/bin/controller-gen
 controller-gen: ## Download controller-gen locally if necessary.
-	$(call go-install-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.11.4)
+	$(call go-install-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.14.0)
 
 GOLANGCI_LINT = $(shell pwd)/bin/golangci-lint
 golangci-lint: ## Download golangci-lint locally if necessary.
@@ -132,7 +132,11 @@ datastore-postgres:
 	$(MAKE) NAME=gold _datastore-postgres
 
 _datastore-etcd:
-	$(HELM) upgrade --install etcd-$(NAME) clastix/kamaji-etcd --create-namespace -n etcd-system --set datastore.enabled=true
+	$(HELM) upgrade --install etcd-$(NAME) clastix/kamaji-etcd --create-namespace -n etcd-system --set datastore.enabled=true --set fullnameOverride=etcd-$(NAME)
+
+_datastore-nats:
+	$(MAKE) NAME=$(NAME) NAMESPACE=nats-system -C deploy/kine/nats nats
+	kubectl apply -f $(shell pwd)/config/samples/kamaji_v1alpha1_datastore_nats_$(NAME).yaml
 
 datastore-etcd: helm
 	$(HELM) repo add clastix https://clastix.github.io/charts
@@ -141,7 +145,15 @@ datastore-etcd: helm
 	$(MAKE) NAME=silver _datastore-etcd
 	$(MAKE) NAME=gold _datastore-etcd
 
-datastores: datastore-mysql datastore-etcd datastore-postgres ## Install all Kamaji DataStores with multiple drivers, and different tiers.
+datastore-nats: helm
+	$(HELM) repo add nats https://nats-io.github.io/k8s/helm/charts/
+	$(HELM) repo update
+	$(MAKE) NAME=bronze _datastore-nats
+	$(MAKE) NAME=silver _datastore-nats
+	$(MAKE) NAME=gold _datastore-nats
+	$(MAKE) NAME=notls _datastore-nats
+
+datastores: datastore-mysql datastore-etcd datastore-postgres datastore-nats ## Install all Kamaji DataStores with multiple drivers, and different tiers.
 
 ##@ Build
 

README.md

@@ -53,7 +53,7 @@ The state is managed by the `Datastore` API, a cluster-scoped resource which can
 - **Managing core addons**: Kamaji allows configuring automatically `kube-proxy`, `CoreDNS`, and `konnectivity`, with automatic remediation in case of user errors (e.g.: deleting the `CoreDNS` deployment).
 - **Auto Healing**: the `TenantControlPlane` objects in the management cluster are tracked by Kamaji, in case of deletion of those, everything is created in an idempotent way.
 - **Datastore multi-tenancy**: optionally, Kamaji allows running multiple Control Planes on the same _Datastore_ instance leveraging on the multi-tenancy of each driver, decreasing operations and optimizing costs.
-- **Overcoming `etcd` limitations**: optionally, Kamaji allows using a different _Datastore_ thanks to [`kine`](https://github.com/k3s-io/kine) by supporting `MySQL` or `PostgreSQL` as an alternative.
+- **Overcoming `etcd` limitations**: optionally, Kamaji allows using a different _Datastore_ thanks to [`kine`](https://github.com/k3s-io/kine) by supporting `MySQL`, `PostgreSQL`, or `NATS` as an alternative.
 - **Simplifying mixed-networks setup**: thanks to [`Konnectivity`](https://kubernetes.io/docs/tasks/extend-kubernetes/setup-konnectivity/),
   the Tenant Control Plane is connected to the worker nodes hosted in a different network, overcoming the no-NAT availability when dealing with nodes with a non routable IP address
   (e.g.: worker nodes in a different infrastructure).
@@ -104,7 +104,7 @@ Since Kamaji is just focusing on the Control Plane a [Kamaji's Cluster API Contr
 - [x] Dynamic address on Load Balancer
 - [x] Zero Downtime Tenant Control Plane upgrade
 - [x] [Join worker nodes from anywhere thanks to Konnectivity](https://kamaji.clastix.io/concepts/#konnectivity)
-- [x] [Alternative datastore MySQL and PostgreSQL](https://kamaji.clastix.io/guides/alternative-datastore/)
+- [x] [Alternative datastore MySQL, PostgreSQL, NATS](https://kamaji.clastix.io/guides/alternative-datastore/)
 - [x] [Pool of multiple datastores](https://kamaji.clastix.io/concepts/#datastores)
 - [x] [Seamless migration between datastores](https://kamaji.clastix.io/guides/datastore-migration/)
 - [ ] Automatic assignment to a datastore

api/v1alpha1/datastore_types.go

@@ -8,7 +8,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
-// +kubebuilder:validation:Enum=etcd;MySQL;PostgreSQL
+// +kubebuilder:validation:Enum=etcd;MySQL;PostgreSQL;NATS
 
 type Driver string
 
@@ -16,6 +16,7 @@ var (
 	EtcdDriver           Driver = "etcd"
 	KineMySQLDriver      Driver = "MySQL"
 	KinePostgreSQLDriver Driver = "PostgreSQL"
+	KineNatsDriver       Driver = "NATS"
 )
 
 // +kubebuilder:validation:MinItems=1
@@ -33,7 +34,8 @@ type DataStoreSpec struct {
 	// This value is optional.
 	BasicAuth *BasicAuth `json:"basicAuth,omitempty"`
 	// Defines the TLS/SSL configuration required to connect to the data store in a secure way.
-	TLSConfig TLSConfig `json:"tlsConfig"`
+	// This value is optional.
+	TLSConfig *TLSConfig `json:"tlsConfig,omitempty"`
 }
 
 // TLSConfig contains the information used to connect to the data store using a secured connection.
@@ -42,7 +44,7 @@ type TLSConfig struct {
 	// The key reference is required since etcd authentication is based on certificates, and Kamaji is responsible in creating this.
 	CertificateAuthority CertKeyPair `json:"certificateAuthority"`
 	// Specifies the SSL/TLS key and private key pair used to connect to the data store.
-	ClientCertificate ClientCertificate `json:"clientCertificate"`
+	ClientCertificate *ClientCertificate `json:"clientCertificate,omitempty"`
 }
 
 type ClientCertificate struct {

api/v1alpha1/indexer_datastore_usedsecret.go

@@ -43,20 +43,22 @@ func (d *DatastoreUsedSecret) ExtractValue() client.IndexerFunc {
 			}
 		}
 
-		if ds.Spec.TLSConfig.CertificateAuthority.Certificate.SecretRef != nil {
-			res = append(res, d.namespacedName(*ds.Spec.TLSConfig.CertificateAuthority.Certificate.SecretRef))
-		}
+		if ds.Spec.TLSConfig != nil {
+			if ds.Spec.TLSConfig.CertificateAuthority.Certificate.SecretRef != nil {
+				res = append(res, d.namespacedName(*ds.Spec.TLSConfig.CertificateAuthority.Certificate.SecretRef))
+			}
 
-		if ds.Spec.TLSConfig.CertificateAuthority.PrivateKey != nil && ds.Spec.TLSConfig.CertificateAuthority.PrivateKey.SecretRef != nil {
-			res = append(res, d.namespacedName(*ds.Spec.TLSConfig.CertificateAuthority.PrivateKey.SecretRef))
-		}
+			if ds.Spec.TLSConfig.CertificateAuthority.PrivateKey != nil && ds.Spec.TLSConfig.CertificateAuthority.PrivateKey.SecretRef != nil {
+				res = append(res, d.namespacedName(*ds.Spec.TLSConfig.CertificateAuthority.PrivateKey.SecretRef))
+			}
 
-		if ds.Spec.TLSConfig.ClientCertificate.Certificate.SecretRef != nil {
-			res = append(res, d.namespacedName(*ds.Spec.TLSConfig.ClientCertificate.Certificate.SecretRef))
-		}
+			if ds.Spec.TLSConfig.ClientCertificate.Certificate.SecretRef != nil {
+				res = append(res, d.namespacedName(*ds.Spec.TLSConfig.ClientCertificate.Certificate.SecretRef))
+			}
 
-		if ds.Spec.TLSConfig.ClientCertificate.PrivateKey.SecretRef != nil {
-			res = append(res, d.namespacedName(*ds.Spec.TLSConfig.ClientCertificate.PrivateKey.SecretRef))
+			if ds.Spec.TLSConfig.ClientCertificate.PrivateKey.SecretRef != nil {
+				res = append(res, d.namespacedName(*ds.Spec.TLSConfig.ClientCertificate.PrivateKey.SecretRef))
+			}
 		}
 
 		return res

api/v1alpha1/tenantcontrolplane_types.go

@@ -141,8 +141,9 @@ type DeploymentSpec struct {
 	// such as kube-apiserver, controller-manager, and scheduler. WARNING - This option
 	// can override existing parameters and cause components to misbehave in unxpected ways.
 	// Only modify if you know what you are doing.
-	ExtraArgs          *ControlPlaneExtraArgs `json:"extraArgs,omitempty"`
-	AdditionalMetadata AdditionalMetadata     `json:"additionalMetadata,omitempty"`
+	ExtraArgs             *ControlPlaneExtraArgs `json:"extraArgs,omitempty"`
+	AdditionalMetadata    AdditionalMetadata     `json:"additionalMetadata,omitempty"`
+	PodAdditionalMetadata AdditionalMetadata     `json:"podAdditionalMetadata,omitempty"`
 	// AdditionalInitContainers allows adding additional init containers to the Control Plane deployment.
 	AdditionalInitContainers []corev1.Container `json:"additionalInitContainers,omitempty"`
 	// AdditionalContainers allows adding additional containers to the Control Plane deployment.
@@ -152,6 +153,9 @@ type DeploymentSpec struct {
 	// AdditionalVolumeMounts allows to mount an additional volume into each component of the Control Plane
 	// (kube-apiserver, controller-manager, and scheduler).
 	AdditionalVolumeMounts *AdditionalVolumeMounts `json:"additionalVolumeMounts,omitempty"`
+	// +kubebuilder:default="default"
+	// ServiceAccountName allows to specify the service account to be mounted to the pods of the Control plane deployment
+	ServiceAccountName string `json:"serviceAccountName,omitempty"`
 }
 
 // AdditionalVolumeMounts allows mounting additional volumes to the Control Plane components.
@@ -216,8 +220,12 @@ type KonnectivityAgentSpec struct {
 	Image string `json:"image,omitempty"`
 	// Version for Konnectivity agent.
 	// +kubebuilder:default=v0.0.32
-	Version   string    `json:"version,omitempty"`
-	ExtraArgs ExtraArgs `json:"extraArgs,omitempty"`
+	Version string `json:"version,omitempty"`
+	// Tolerations for the deployed agent.
+	// Can be customized to start the konnectivity-agent even if the nodes are not ready or tainted.
+	// +kubebuilder:default={{key: "CriticalAddonsOnly", operator: "Exists"}}
+	Tolerations []corev1.Toleration `json:"tolerations,omitempty"`
+	ExtraArgs   ExtraArgs           `json:"extraArgs,omitempty"`
 }
 
 // KonnectivitySpec defines the spec for Konnectivity.

api/v1alpha1/types.go

@@ -28,9 +28,10 @@ func (c CGroupDriver) String() string {
 }
 
 const (
-	ServiceTypeLoadBalancer = (ServiceType)(corev1.ServiceTypeLoadBalancer)
-	ServiceTypeClusterIP    = (ServiceType)(corev1.ServiceTypeClusterIP)
-	ServiceTypeNodePort     = (ServiceType)(corev1.ServiceTypeNodePort)
+	ServiceTypeLoadBalancer       = (ServiceType)(corev1.ServiceTypeLoadBalancer)
+	ServiceTypeClusterIP          = (ServiceType)(corev1.ServiceTypeClusterIP)
+	ServiceTypeNodePort           = (ServiceType)(corev1.ServiceTypeNodePort)
+	KubeconfigSecretKeyAnnotation = "kamaji.clastix.io/kubeconfig-secret-key"
 )
 
 // +kubebuilder:validation:Enum=ClusterIP;NodePort;LoadBalancer

api/v1alpha1/zz_generated.deepcopy.go

@@ -1,5 +1,4 @@
 //go:build !ignore_autogenerated
-// +build !ignore_autogenerated
 
 // Copyright 2022 Clastix Labs
 // SPDX-License-Identifier: Apache-2.0
@@ -526,7 +525,11 @@ func (in *DataStoreSpec) DeepCopyInto(out *DataStoreSpec) {
 		*out = new(BasicAuth)
 		(*in).DeepCopyInto(*out)
 	}
-	in.TLSConfig.DeepCopyInto(&out.TLSConfig)
+	if in.TLSConfig != nil {
+		in, out := &in.TLSConfig, &out.TLSConfig
+		*out = new(TLSConfig)
+		(*in).DeepCopyInto(*out)
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataStoreSpec.
@@ -578,6 +581,11 @@ func (in *DatastoreUsedSecret) DeepCopy() *DatastoreUsedSecret {
 func (in *DeploymentSpec) DeepCopyInto(out *DeploymentSpec) {
 	*out = *in
 	out.RegistrySettings = in.RegistrySettings
+	if in.Replicas != nil {
+		in, out := &in.Replicas, &out.Replicas
+		*out = new(int32)
+		**out = **in
+	}
 	if in.NodeSelector != nil {
 		in, out := &in.NodeSelector, &out.NodeSelector
 		*out = make(map[string]string, len(*in))
@@ -616,6 +624,7 @@ func (in *DeploymentSpec) DeepCopyInto(out *DeploymentSpec) {
 		(*in).DeepCopyInto(*out)
 	}
 	in.AdditionalMetadata.DeepCopyInto(&out.AdditionalMetadata)
+	in.PodAdditionalMetadata.DeepCopyInto(&out.PodAdditionalMetadata)
 	if in.AdditionalInitContainers != nil {
 		in, out := &in.AdditionalInitContainers, &out.AdditionalInitContainers
 		*out = make([]v1.Container, len(*in))
@@ -775,6 +784,13 @@ func (in *IngressSpec) DeepCopy() *IngressSpec {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *KonnectivityAgentSpec) DeepCopyInto(out *KonnectivityAgentSpec) {
 	*out = *in
+	if in.Tolerations != nil {
+		in, out := &in.Tolerations, &out.Tolerations
+		*out = make([]v1.Toleration, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
 	if in.ExtraArgs != nil {
 		in, out := &in.ExtraArgs, &out.ExtraArgs
 		*out = make(ExtraArgs, len(*in))
@@ -1196,7 +1212,11 @@ func (in *StorageStatus) DeepCopy() *StorageStatus {
 func (in *TLSConfig) DeepCopyInto(out *TLSConfig) {
 	*out = *in
 	in.CertificateAuthority.DeepCopyInto(&out.CertificateAuthority)
-	in.ClientCertificate.DeepCopyInto(&out.ClientCertificate)
+	if in.ClientCertificate != nil {
+		in, out := &in.ClientCertificate, &out.ClientCertificate
+		*out = new(ClientCertificate)
+		(*in).DeepCopyInto(*out)
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TLSConfig.

charts/kamaji/Chart.yaml

@@ -1,21 +1,23 @@
 apiVersion: v2
-appVersion: v0.4.2
-description: Kamaji is a Kubernetes Control Plane Manager.
+appVersion: v0.6.0
+description: Kamaji is the Hosted Control Plane Manager for Kubernetes.
 home: https://github.com/clastix/kamaji
 icon: https://github.com/clastix/kamaji/raw/master/assets/logo-colored.png
 kubeVersion: ">=1.21.0-0"
 maintainers:
 - email: dario@tranchitella.eu
   name: Dario Tranchitella
+  url: https://clastix.io
 - email: me@maxgio.it
   name: Massimiliano Giovagnoli
 - email: me@bsctl.io
   name: Adriano Pezzuto
+  url: https://clastix.io
 name: kamaji
 sources:
 - https://github.com/clastix/kamaji
 type: application
-version: 0.15.1
+version: 0.15.3
 annotations:
   catalog.cattle.io/certified: partner
   catalog.cattle.io/release-name: kamaji

charts/kamaji/README.md

@@ -1,16 +1,16 @@
 # kamaji
 
-![Version: 0.15.1](https://img.shields.io/badge/Version-0.15.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v0.4.2](https://img.shields.io/badge/AppVersion-v0.4.2-informational?style=flat-square)
+![Version: 0.15.3](https://img.shields.io/badge/Version-0.15.3-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v0.6.0](https://img.shields.io/badge/AppVersion-v0.6.0-informational?style=flat-square)
 
-Kamaji is a Kubernetes Control Plane Manager.
+Kamaji is the Hosted Control Plane Manager for Kubernetes.
 
 ## Maintainers
 
 | Name | Email | Url |
 | ---- | ------ | --- |
-| Dario Tranchitella | <dario@tranchitella.eu> |  |
+| Dario Tranchitella | <dario@tranchitella.eu> | <https://clastix.io> |
 | Massimiliano Giovagnoli | <me@maxgio.it> |  |
-| Adriano Pezzuto | <me@bsctl.io> |  |
+| Adriano Pezzuto | <me@bsctl.io> | <https://clastix.io> |
 
 ## Source Code
 
@@ -77,7 +77,7 @@ Here the values you can override:
 | datastore.driver | string | `"etcd"` | (string) The Kamaji Datastore driver, supported: etcd, MySQL, PostgreSQL (defaults=etcd). |
 | datastore.enabled | bool | `true` | (bool) Enable the Kamaji Datastore creation (default=true) |
 | datastore.endpoints | list | `[]` | (array) List of endpoints of the selected Datastore. When letting the Chart install the etcd datastore, this field is populated automatically. |
-| datastore.nameOverride | string | `nil` | The Datastore name override, if empty and enabled=true defaults to `default`, if enabled=false, this is the name of the Datastore to connect to.  |
+| datastore.nameOverride | string | `nil` | The Datastore name override, if empty and enabled=true defaults to `default`, if enabled=false, this is the name of the Datastore to connect to. |
 | datastore.tlsConfig.certificateAuthority.certificate.keyPath | string | `nil` | Key of the Secret which contains the content of the certificate. |
 | datastore.tlsConfig.certificateAuthority.certificate.name | string | `nil` | Name of the Secret containing the CA required to establish the mandatory SSL/TLS connection to the datastore. |
 | datastore.tlsConfig.certificateAuthority.certificate.namespace | string | `nil` | Namespace of the Secret containing the CA required to establish the mandatory SSL/TLS connection to the datastore. |
@@ -90,6 +90,7 @@ Here the values you can override:
 | datastore.tlsConfig.clientCertificate.privateKey.keyPath | string | `nil` | Key of the Secret which contains the content of the private key. |
 | datastore.tlsConfig.clientCertificate.privateKey.name | string | `nil` | Name of the Secret containing the client certificate private key required to establish the mandatory SSL/TLS connection to the datastore. |
 | datastore.tlsConfig.clientCertificate.privateKey.namespace | string | `nil` | Namespace of the Secret containing the client certificate private key required to establish the mandatory SSL/TLS connection to the datastore. |
+| datastore.tlsConfig.enabled | bool | `true` |  |
 | etcd.compactionInterval | int | `0` | ETCD Compaction interval (e.g. "5m0s"). (default: "0" (disabled)) |
 | etcd.deploy | bool | `true` | Install an etcd with enabled multi-tenancy along with Kamaji |
 | etcd.image | object | `{"pullPolicy":"IfNotPresent","repository":"quay.io/coreos/etcd","tag":"v3.5.6"}` | Install specific etcd image |

charts/kamaji/crds/datastore.yaml

@@ -30,10 +30,19 @@ spec:
           description: DataStore is the Schema for the datastores API.
           properties:
             apiVersion:
-              description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              description: |-
+                APIVersion defines the versioned schema of this representation of an object.
+                Servers should convert recognized schemas to the latest internal value, and
+                may reject unrecognized values.
+                More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
               type: string
             kind:
-              description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              description: |-
+                Kind is a string value representing the REST resource this object represents.
+                Servers may infer this from the endpoint the client submits requests to.
+                Cannot be updated.
+                In CamelCase.
+                More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
               type: string
             metadata:
               type: object
@@ -41,25 +50,33 @@ spec:
               description: DataStoreSpec defines the desired state of DataStore.
               properties:
                 basicAuth:
-                  description: In case of authentication enabled for the given data store, specifies the username and password pair. This value is optional.
+                  description: |-
+                    In case of authentication enabled for the given data store, specifies the username and password pair.
+                    This value is optional.
                   properties:
                     password:
                       properties:
                         content:
-                          description: Bare content of the file, base64 encoded. It has precedence over the SecretReference value.
+                          description: |-
+                            Bare content of the file, base64 encoded.
+                            It has precedence over the SecretReference value.
                           format: byte
                           type: string
                         secretReference:
                           properties:
                             keyPath:
-                              description: Name of the key for the given Secret reference where the content is stored. This value is mandatory.
+                              description: |-
+                                Name of the key for the given Secret reference where the content is stored.
+                                This value is mandatory.
                               minLength: 1
                               type: string
                             name:
-                              description: name is unique within a namespace to reference a secret resource.
+                              description: name is unique within a namespace to reference
+                                a secret resource.
                               type: string
                             namespace:
-                              description: namespace defines the space within which the secret name must be unique.
+                              description: namespace defines the space within which
+                                the secret name must be unique.
                               type: string
                           required:
                             - keyPath
@@ -69,20 +86,26 @@ spec:
                     username:
                       properties:
                         content:
-                          description: Bare content of the file, base64 encoded. It has precedence over the SecretReference value.
+                          description: |-
+                            Bare content of the file, base64 encoded.
+                            It has precedence over the SecretReference value.
                           format: byte
                           type: string
                         secretReference:
                           properties:
                             keyPath:
-                              description: Name of the key for the given Secret reference where the content is stored. This value is mandatory.
+                              description: |-
+                                Name of the key for the given Secret reference where the content is stored.
+                                This value is mandatory.
                               minLength: 1
                               type: string
                             name:
-                              description: name is unique within a namespace to reference a secret resource.
+                              description: name is unique within a namespace to reference
+                                a secret resource.
                               type: string
                             namespace:
-                              description: namespace defines the space within which the secret name must be unique.
+                              description: namespace defines the space within which
+                                the secret name must be unique.
                               type: string
                           required:
                             - keyPath
@@ -99,36 +122,49 @@ spec:
                     - etcd
                     - MySQL
                     - PostgreSQL
+                    - NATS
                   type: string
                 endpoints:
-                  description: List of the endpoints to connect to the shared datastore. No need for protocol, just bare IP/FQDN and port.
+                  description: |-
+                    List of the endpoints to connect to the shared datastore.
+                    No need for protocol, just bare IP/FQDN and port.
                   items:
                     type: string
                   minItems: 1
                   type: array
                 tlsConfig:
-                  description: Defines the TLS/SSL configuration required to connect to the data store in a secure way.
+                  description: |-
+                    Defines the TLS/SSL configuration required to connect to the data store in a secure way.
+                    This value is optional.
                   properties:
                     certificateAuthority:
-                      description: Retrieve the Certificate Authority certificate and private key, such as bare content of the file, or a SecretReference. The key reference is required since etcd authentication is based on certificates, and Kamaji is responsible in creating this.
+                      description: |-
+                        Retrieve the Certificate Authority certificate and private key, such as bare content of the file, or a SecretReference.
+                        The key reference is required since etcd authentication is based on certificates, and Kamaji is responsible in creating this.
                       properties:
                         certificate:
                           properties:
                             content:
-                              description: Bare content of the file, base64 encoded. It has precedence over the SecretReference value.
+                              description: |-
+                                Bare content of the file, base64 encoded.
+                                It has precedence over the SecretReference value.
                               format: byte
                               type: string
                             secretReference:
                               properties:
                                 keyPath:
-                                  description: Name of the key for the given Secret reference where the content is stored. This value is mandatory.
+                                  description: |-
+                                    Name of the key for the given Secret reference where the content is stored.
+                                    This value is mandatory.
                                   minLength: 1
                                   type: string
                                 name:
-                                  description: name is unique within a namespace to reference a secret resource.
+                                  description: name is unique within a namespace to
+                                    reference a secret resource.
                                   type: string
                                 namespace:
-                                  description: namespace defines the space within which the secret name must be unique.
+                                  description: namespace defines the space within which
+                                    the secret name must be unique.
                                   type: string
                               required:
                                 - keyPath
@@ -138,20 +174,26 @@ spec:
                         privateKey:
                           properties:
                             content:
-                              description: Bare content of the file, base64 encoded. It has precedence over the SecretReference value.
+                              description: |-
+                                Bare content of the file, base64 encoded.
+                                It has precedence over the SecretReference value.
                               format: byte
                               type: string
                             secretReference:
                               properties:
                                 keyPath:
-                                  description: Name of the key for the given Secret reference where the content is stored. This value is mandatory.
+                                  description: |-
+                                    Name of the key for the given Secret reference where the content is stored.
+                                    This value is mandatory.
                                   minLength: 1
                                   type: string
                                 name:
-                                  description: name is unique within a namespace to reference a secret resource.
+                                  description: name is unique within a namespace to
+                                    reference a secret resource.
                                   type: string
                                 namespace:
-                                  description: namespace defines the space within which the secret name must be unique.
+                                  description: namespace defines the space within which
+                                    the secret name must be unique.
                                   type: string
                               required:
                                 - keyPath
@@ -162,25 +204,32 @@ spec:
                         - certificate
                       type: object
                     clientCertificate:
-                      description: Specifies the SSL/TLS key and private key pair used to connect to the data store.
+                      description: Specifies the SSL/TLS key and private key pair used
+                        to connect to the data store.
                       properties:
                         certificate:
                           properties:
                             content:
-                              description: Bare content of the file, base64 encoded. It has precedence over the SecretReference value.
+                              description: |-
+                                Bare content of the file, base64 encoded.
+                                It has precedence over the SecretReference value.
                               format: byte
                               type: string
                             secretReference:
                               properties:
                                 keyPath:
-                                  description: Name of the key for the given Secret reference where the content is stored. This value is mandatory.
+                                  description: |-
+                                    Name of the key for the given Secret reference where the content is stored.
+                                    This value is mandatory.
                                   minLength: 1
                                   type: string
                                 name:
-                                  description: name is unique within a namespace to reference a secret resource.
+                                  description: name is unique within a namespace to
+                                    reference a secret resource.
                                   type: string
                                 namespace:
-                                  description: namespace defines the space within which the secret name must be unique.
+                                  description: namespace defines the space within which
+                                    the secret name must be unique.
                                   type: string
                               required:
                                 - keyPath
@@ -190,20 +239,26 @@ spec:
                         privateKey:
                           properties:
                             content:
-                              description: Bare content of the file, base64 encoded. It has precedence over the SecretReference value.
+                              description: |-
+                                Bare content of the file, base64 encoded.
+                                It has precedence over the SecretReference value.
                               format: byte
                               type: string
                             secretReference:
                               properties:
                                 keyPath:
-                                  description: Name of the key for the given Secret reference where the content is stored. This value is mandatory.
+                                  description: |-
+                                    Name of the key for the given Secret reference where the content is stored.
+                                    This value is mandatory.
                                   minLength: 1
                                   type: string
                                 name:
-                                  description: name is unique within a namespace to reference a secret resource.
+                                  description: name is unique within a namespace to
+                                    reference a secret resource.
                                   type: string
                                 namespace:
-                                  description: namespace defines the space within which the secret name must be unique.
+                                  description: namespace defines the space within which
+                                    the secret name must be unique.
                                   type: string
                               required:
                                 - keyPath
@@ -216,18 +271,17 @@ spec:
                       type: object
                   required:
                     - certificateAuthority
-                    - clientCertificate
                   type: object
               required:
                 - driver
                 - endpoints
-                - tlsConfig
               type: object
             status:
               description: DataStoreStatus defines the observed state of DataStore.
               properties:
                 usedBy:
-                  description: List of the Tenant Control Planes, namespaced named, using this data store.
+                  description: List of the Tenant Control Planes, namespaced named,
+                    using this data store.
                   items:
                     type: string
                   type: array

charts/kamaji/templates/datastore.yaml

@@ -20,9 +20,14 @@ spec:
       secretReference:
         {{- .Values.datastore.basicAuth.passwordSecret | toYaml | nindent 8 }}
 {{- end }}
+{{- if .Values.datastore.tlsConfig.enabled }}
   tlsConfig:
     certificateAuthority:
       {{- include "datastore.certificateAuthority" . | indent 6 }}
+
+    {{- if .Values.datastore.tlsConfig.clientCertificate }}
     clientCertificate:
       {{- include "datastore.clientCertificate" . | indent 6 }}
+    {{- end }}
+{{- end}}
 {{- end}}

charts/kamaji/values.yaml

@@ -60,7 +60,7 @@ etcd:
     # -- The custom annotations to add to the PVC
     customAnnotations: {}
     #  volumeType: local
-  
+
   # -- (array) Kubernetes affinity rules to apply to Kamaji etcd pods
   tolerations: []
 
@@ -162,7 +162,7 @@ loggingDevel:
 datastore:
   # -- (bool) Enable the Kamaji Datastore creation (default=true)
   enabled: true
-  # -- (string) The Datastore name override, if empty and enabled=true defaults to `default`, if enabled=false, this is the name of the Datastore to connect to. 
+  # -- (string) The Datastore name override, if empty and enabled=true defaults to `default`, if enabled=false, this is the name of the Datastore to connect to.
   nameOverride:
   # -- (string) The Kamaji Datastore driver, supported: etcd, MySQL, PostgreSQL (defaults=etcd).
   driver: etcd
@@ -184,6 +184,7 @@ datastore:
       # -- The Secret key where the data is stored.
       keyPath:
   tlsConfig:
+    enabled: true
     certificateAuthority:
       certificate:
         # -- Name of the Secret containing the CA required to establish the mandatory SSL/TLS connection to the datastore.

cmd/manager/cmd.go

@@ -175,7 +175,9 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command {
 					handlers.Freeze{},
 				},
 				routes.TenantControlPlaneDefaults{}: {
-					handlers.TenantControlPlaneDefaults{DefaultDatastore: datastore},
+					handlers.TenantControlPlaneDefaults{
+						DefaultDatastore: datastore,
+					},
 				},
 				routes.TenantControlPlaneValidate{}: {
 					handlers.TenantControlPlaneName{},

config/crd/bases/kamaji.clastix.io_datastores.yaml

@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.11.4
+    controller-gen.kubebuilder.io/version: v0.14.0
   name: datastores.kamaji.clastix.io
 spec:
   group: kamaji.clastix.io
@@ -29,14 +29,19 @@ spec:
         description: DataStore is the Schema for the datastores API.
         properties:
           apiVersion:
-            description: 'APIVersion defines the versioned schema of this representation
-              of an object. Servers should convert recognized schemas to the latest
-              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
             type: string
           kind:
-            description: 'Kind is a string value representing the REST resource this
-              object represents. Servers may infer this from the endpoint the client
-              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
             type: string
           metadata:
             type: object
@@ -44,21 +49,24 @@ spec:
             description: DataStoreSpec defines the desired state of DataStore.
             properties:
               basicAuth:
-                description: In case of authentication enabled for the given data
-                  store, specifies the username and password pair. This value is optional.
+                description: |-
+                  In case of authentication enabled for the given data store, specifies the username and password pair.
+                  This value is optional.
                 properties:
                   password:
                     properties:
                       content:
-                        description: Bare content of the file, base64 encoded. It
-                          has precedence over the SecretReference value.
+                        description: |-
+                          Bare content of the file, base64 encoded.
+                          It has precedence over the SecretReference value.
                         format: byte
                         type: string
                       secretReference:
                         properties:
                           keyPath:
-                            description: Name of the key for the given Secret reference
-                              where the content is stored. This value is mandatory.
+                            description: |-
+                              Name of the key for the given Secret reference where the content is stored.
+                              This value is mandatory.
                             minLength: 1
                             type: string
                           name:
@@ -77,15 +85,17 @@ spec:
                   username:
                     properties:
                       content:
-                        description: Bare content of the file, base64 encoded. It
-                          has precedence over the SecretReference value.
+                        description: |-
+                          Bare content of the file, base64 encoded.
+                          It has precedence over the SecretReference value.
                         format: byte
                         type: string
                       secretReference:
                         properties:
                           keyPath:
-                            description: Name of the key for the given Secret reference
-                              where the content is stored. This value is mandatory.
+                            description: |-
+                              Name of the key for the given Secret reference where the content is stored.
+                              This value is mandatory.
                             minLength: 1
                             type: string
                           name:
@@ -111,37 +121,40 @@ spec:
                 - etcd
                 - MySQL
                 - PostgreSQL
+                - NATS
                 type: string
               endpoints:
-                description: List of the endpoints to connect to the shared datastore.
+                description: |-
+                  List of the endpoints to connect to the shared datastore.
                   No need for protocol, just bare IP/FQDN and port.
                 items:
                   type: string
                 minItems: 1
                 type: array
               tlsConfig:
-                description: Defines the TLS/SSL configuration required to connect
-                  to the data store in a secure way.
+                description: |-
+                  Defines the TLS/SSL configuration required to connect to the data store in a secure way.
+                  This value is optional.
                 properties:
                   certificateAuthority:
-                    description: Retrieve the Certificate Authority certificate and
-                      private key, such as bare content of the file, or a SecretReference.
-                      The key reference is required since etcd authentication is based
-                      on certificates, and Kamaji is responsible in creating this.
+                    description: |-
+                      Retrieve the Certificate Authority certificate and private key, such as bare content of the file, or a SecretReference.
+                      The key reference is required since etcd authentication is based on certificates, and Kamaji is responsible in creating this.
                     properties:
                       certificate:
                         properties:
                           content:
-                            description: Bare content of the file, base64 encoded.
+                            description: |-
+                              Bare content of the file, base64 encoded.
                               It has precedence over the SecretReference value.
                             format: byte
                             type: string
                           secretReference:
                             properties:
                               keyPath:
-                                description: Name of the key for the given Secret
-                                  reference where the content is stored. This value
-                                  is mandatory.
+                                description: |-
+                                  Name of the key for the given Secret reference where the content is stored.
+                                  This value is mandatory.
                                 minLength: 1
                                 type: string
                               name:
@@ -160,16 +173,17 @@ spec:
                       privateKey:
                         properties:
                           content:
-                            description: Bare content of the file, base64 encoded.
+                            description: |-
+                              Bare content of the file, base64 encoded.
                               It has precedence over the SecretReference value.
                             format: byte
                             type: string
                           secretReference:
                             properties:
                               keyPath:
-                                description: Name of the key for the given Secret
-                                  reference where the content is stored. This value
-                                  is mandatory.
+                                description: |-
+                                  Name of the key for the given Secret reference where the content is stored.
+                                  This value is mandatory.
                                 minLength: 1
                                 type: string
                               name:
@@ -195,16 +209,17 @@ spec:
                       certificate:
                         properties:
                           content:
-                            description: Bare content of the file, base64 encoded.
+                            description: |-
+                              Bare content of the file, base64 encoded.
                               It has precedence over the SecretReference value.
                             format: byte
                             type: string
                           secretReference:
                             properties:
                               keyPath:
-                                description: Name of the key for the given Secret
-                                  reference where the content is stored. This value
-                                  is mandatory.
+                                description: |-
+                                  Name of the key for the given Secret reference where the content is stored.
+                                  This value is mandatory.
                                 minLength: 1
                                 type: string
                               name:
@@ -223,16 +238,17 @@ spec:
                       privateKey:
                         properties:
                           content:
-                            description: Bare content of the file, base64 encoded.
+                            description: |-
+                              Bare content of the file, base64 encoded.
                               It has precedence over the SecretReference value.
                             format: byte
                             type: string
                           secretReference:
                             properties:
                               keyPath:
-                                description: Name of the key for the given Secret
-                                  reference where the content is stored. This value
-                                  is mandatory.
+                                description: |-
+                                  Name of the key for the given Secret reference where the content is stored.
+                                  This value is mandatory.
                                 minLength: 1
                                 type: string
                               name:
@@ -254,12 +270,10 @@ spec:
                     type: object
                 required:
                 - certificateAuthority
-                - clientCertificate
                 type: object
             required:
             - driver
             - endpoints
-            - tlsConfig
             type: object
           status:
             description: DataStoreStatus defines the observed state of DataStore.

config/manager/kustomization.yaml

@@ -13,4 +13,4 @@ kind: Kustomization
 images:
 - name: controller
   newName: clastix/kamaji
-  newTag: v0.4.2
+  newTag: v0.6.0

config/samples/kamaji_v1alpha1_datastore_nats_bronze.yaml

@@ -0,0 +1,34 @@
+apiVersion: kamaji.clastix.io/v1alpha1
+kind: DataStore
+metadata:
+  name: nats-bronze
+spec:
+  driver: NATS
+  endpoints:
+    - bronze-nats.nats-system.svc:4222
+  basicAuth:
+    username:
+      content: YWRtaW4=
+    password:
+      secretReference:
+        name: nats-bronze-config
+        namespace: nats-system
+        keyPath: password
+  tlsConfig:
+    certificateAuthority:
+      certificate:
+        secretReference:
+          name: nats-bronze-config
+          namespace: nats-system
+          keyPath: ca.crt
+    clientCertificate:
+      certificate:
+        secretReference:
+          name: nats-bronze-config
+          namespace: nats-system
+          keyPath: server.crt
+      privateKey:
+        secretReference:
+          name: nats-bronze-config
+          namespace: nats-system
+          keyPath: server.key

config/samples/kamaji_v1alpha1_datastore_nats_gold.yaml

@@ -0,0 +1,34 @@
+apiVersion: kamaji.clastix.io/v1alpha1
+kind: DataStore
+metadata:
+  name: nats-gold
+spec:
+  driver: NATS
+  endpoints:
+    - nats-gold.nats-system.svc:4222
+  basicAuth:
+    username:
+      content: YWRtaW4=
+    password:
+      secretReference:
+        name: nats-gold-config
+        namespace: nats-system
+        keyPath: password
+  tlsConfig:
+    certificateAuthority:
+      certificate:
+        secretReference:
+          name: nats-gold-config
+          namespace: nats-system
+          keyPath: ca.crt
+    clientCertificate:
+      certificate:
+        secretReference:
+          name: nats-gold-config
+          namespace: nats-system
+          keyPath: server.crt
+      privateKey:
+        secretReference:
+          name: nats-gold-config
+          namespace: nats-system
+          keyPath: server.key

config/samples/kamaji_v1alpha1_datastore_nats_notls.yaml

@@ -0,0 +1,17 @@
+apiVersion: kamaji.clastix.io/v1alpha1
+kind: DataStore
+metadata:
+  name: nats-notls
+spec:
+  driver: NATS
+  endpoints:
+    - notls-nats.nats-system.svc:4222
+  basicAuth:
+    username:
+      content: YWRtaW4=
+    password:
+      secretReference:
+        name: nats-notls-config
+        namespace: nats-system
+        keyPath: password
+

config/samples/kamaji_v1alpha1_datastore_nats_silver.yaml

@@ -0,0 +1,34 @@
+apiVersion: kamaji.clastix.io/v1alpha1
+kind: DataStore
+metadata:
+  name: nats-silver
+spec:
+  driver: NATS
+  endpoints:
+    - nats-silver.nats-system.svc:4222
+  basicAuth:
+    username:
+      content: YWRtaW4=
+    password:
+      secretReference:
+        name: nats-silver-config
+        namespace: nats-system
+        keyPath: password
+  tlsConfig:
+    certificateAuthority:
+      certificate:
+        secretReference:
+          name: nats-silver-config
+          namespace: nats-system
+          keyPath: ca.crt
+    clientCertificate:
+      certificate:
+        secretReference:
+          name: nats-silver-config
+          namespace: nats-system
+          keyPath: server.crt
+      privateKey:
+        secretReference:
+          name: nats-silver-config
+          namespace: nats-system
+          keyPath: server.key

config/webhook/manifests.yaml

@@ -30,25 +30,6 @@ kind: ValidatingWebhookConfiguration
 metadata:
   name: validating-webhook-configuration
 webhooks:
-- admissionReviewVersions:
-  - v1
-  clientConfig:
-    service:
-      name: webhook-service
-      namespace: system
-      path: /validate--v1-secret
-  failurePolicy: Ignore
-  name: vdatastoresecrets.kb.io
-  rules:
-  - apiGroups:
-    - ""
-    apiVersions:
-    - v1
-    operations:
-    - DELETE
-    resources:
-    - secrets
-  sideEffects: None
 - admissionReviewVersions:
   - v1
   clientConfig:
@@ -70,6 +51,25 @@ webhooks:
     resources:
     - datastores
   sideEffects: None
+- admissionReviewVersions:
+  - v1
+  clientConfig:
+    service:
+      name: webhook-service
+      namespace: system
+      path: /validate--v1-secret
+  failurePolicy: Ignore
+  name: vdatastoresecrets.kb.io
+  rules:
+  - apiGroups:
+    - ""
+    apiVersions:
+    - v1
+    operations:
+    - DELETE
+    resources:
+    - secrets
+  sideEffects: None
 - admissionReviewVersions:
   - v1
   clientConfig:

controllers/resources.go

@@ -205,6 +205,9 @@ func getKubeconfigResources(c client.Client, tcpReconcilerConfig TenantControlPl
 
 func getKubernetesStorageResources(c client.Client, dbConnection datastore.Connection, datastore kamajiv1alpha1.DataStore) []resources.Resource {
 	return []resources.Resource{
+		&ds.MultiTenancy{
+			DataStore: datastore,
+		},
 		&ds.Config{
 			Client:     c,
 			ConnString: dbConnection.GetConnectionString(),

deploy/kind/kind-kamaji.yaml

@@ -26,7 +26,7 @@ nodes:
     ## expose port 31132 of the node to port 31132 on the host for konnectivity
   - containerPort: 31132
     hostPort: 31132
-    protocol: TCP   
+    protocol: TCP
     ## expose port 31443 of the node to port 31443 on the host
   - containerPort: 31443
     hostPort: 31443

deploy/kine/mysql/certs/bronze/server-csr.json

@@ -1,14 +0,0 @@
-{
-  "CN": "bronze.mysql-system.svc.cluster.local",
-  "key": {
-    "algo": "rsa",
-    "size": 2048
-  },
-  "hosts": [
-    "127.0.0.1",
-    "localhost",
-    "bronze",
-    "bronze.mysql-system.svc",
-    "bronze.mysql-system.svc.cluster.local"
-  ]
-}

deploy/kine/nats/Makefile

@@ -0,0 +1,40 @@
+ROOT_DIR:=$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))
+NAME:=default
+NAMESPACE:=nats-system
+
+.PHONY: helm
+HELM = $(shell pwd)/../../../bin/helm
+helm: ## Download helm locally if necessary.
+	$(call go-install-tool,$(HELM),helm.sh/helm/v3/cmd/helm@v3.9.0)
+
+nats: nats-certificates nats-secret nats-deployment
+
+nats-certificates:
+	rm -rf $(ROOT_DIR)/certs/$(NAME) && mkdir -p $(ROOT_DIR)/certs/$(NAME)
+	cfssl gencert -initca $(ROOT_DIR)/ca-csr.json | cfssljson -bare $(ROOT_DIR)/certs/$(NAME)/ca
+	@mv $(ROOT_DIR)/certs/$(NAME)/ca.pem $(ROOT_DIR)/certs/$(NAME)/ca.crt
+	@mv $(ROOT_DIR)/certs/$(NAME)/ca-key.pem $(ROOT_DIR)/certs/$(NAME)/ca.key
+	@NAME=$(NAME) NAMESPACE=$(NAMESPACE) envsubst < server-csr.json > $(ROOT_DIR)/certs/$(NAME)/server-csr.json
+	cfssl gencert -ca=$(ROOT_DIR)/certs/$(NAME)/ca.crt -ca-key=$(ROOT_DIR)/certs/$(NAME)/ca.key \
+		-config=$(ROOT_DIR)/config.json -profile=server \
+		$(ROOT_DIR)/certs/$(NAME)/server-csr.json | cfssljson -bare $(ROOT_DIR)/certs/$(NAME)/server
+	@mv $(ROOT_DIR)/certs/$(NAME)/server.pem $(ROOT_DIR)/certs/$(NAME)/server.crt
+	@mv $(ROOT_DIR)/certs/$(NAME)/server-key.pem $(ROOT_DIR)/certs/$(NAME)/server.key
+	chmod 644 $(ROOT_DIR)/certs/$(NAME)/*
+
+nats-secret:
+	@kubectl create namespace $(NAMESPACE) || true
+	@kubectl -n $(NAMESPACE) create secret generic nats-$(NAME)-config \
+		--from-file=$(ROOT_DIR)/certs/$(NAME)/ca.crt --from-file=$(ROOT_DIR)/certs/$(NAME)/ca.key \
+		--from-file=$(ROOT_DIR)/certs/$(NAME)/server.key --from-file=$(ROOT_DIR)/certs/$(NAME)/server.crt \
+		--from-literal=password=password \
+		--dry-run=client -o yaml | kubectl apply -f -
+
+nats-deployment:
+	@VALUES_FILE=$(if $(findstring notls,$(NAME)),values-notls.yaml,values.yaml); \
+	NAME=$(NAME) envsubst < $(ROOT_DIR)/$$VALUES_FILE | $(HELM) upgrade --install $(NAME) nats/nats --create-namespace -n nats-system -f -
+
+
+nats-destroy:
+	@NAME=$(NAME) envsubst < $(ROOT_DIR)/nats.yaml | kubectl -n $(NAMESPACE) delete --ignore-not-found -f -
+	@kubectl delete -n $(NAMESPACE) secret mysql-$(NAME)config --ignore-not-found

deploy/kine/nats/ca-csr.json

@@ -0,0 +1,18 @@
+{
+  "CN": "Clastix CA",
+  "key": {
+    "algo": "rsa",
+    "size": 2048
+  },
+  "names": [
+    {
+      "C": "IT",
+      "ST": "Italy",
+      "L": "Milan"
+    }
+  ],
+  "hosts": [
+    "127.0.0.1",
+    "localhost"
+  ]
+}

deploy/kine/nats/config.json

@@ -0,0 +1,18 @@
+{
+  "signing": {
+      "default": {
+          "expiry": "8760h"
+      },
+      "profiles": {
+          "server": {
+              "expiry": "8760h",
+              "usages": [
+                  "signing",
+                  "key encipherment",
+                  "server auth",
+                  "client auth"
+              ]
+          }
+      }
+  }
+}

deploy/kine/nats/server-csr.json

@@ -0,0 +1,14 @@
+{
+  "CN": "$NAME.$NAMESPACE.svc.cluster.local",
+  "key": {
+    "algo": "rsa",
+    "size": 2048
+  },
+  "hosts": [
+    "127.0.0.1",
+    "localhost",
+    "$NAME-nats",
+    "$NAME-nats.$NAMESPACE.svc",
+    "$NAME-nats.$NAMESPACE.svc.cluster.local"
+  ]
+}

deploy/kine/nats/values-notls.yaml

@@ -0,0 +1,14 @@
+config:
+  merge:
+    accounts:
+      private:
+        jetstream: enabled
+        users:
+        - {user: admin, password: "password", permissions: {subscribe: [">"], publish: [">"]}}
+  cluster:
+    enabled: no
+  jetstream:
+    enabled: true
+    fileStore:
+      pvc:
+        size: 32Mi

deploy/kine/nats/values.yaml

@@ -0,0 +1,20 @@
+config:
+  merge:
+    accounts:
+      private:
+        jetstream: enabled
+        users:
+        - {user: admin, password: "password", permissions: {subscribe: [">"], publish: [">"]}}
+  cluster:
+    enabled: no
+  nats:
+    tls:
+      enabled: true
+      secretName: nats-$NAME-config
+      cert: server.crt
+      key: server.key
+  jetstream:
+    enabled: true
+    fileStore:
+      pvc:
+        size: 32Mi

docs/content/guides/alternative-datastore.md

@@ -1,39 +1,28 @@
 # Use Alternative Datastores
 
-Kamaji offers the possibility of having a different storage system than `etcd` thanks to [kine](https://github.com/k3s-io/kine) integration. One of the implementations is [PostgreSQL](https://www.postgresql.org/).
+Kamaji offers the possibility of having a different storage system than `etcd` thanks to [kine](https://github.com/k3s-io/kine) integration.
 
-## Install the datastore
+## Installing Drivers
 
-On the Management Cluster, install one of the alternative supported datastore:
+The following `make` recipes help you to setup alternative `Datastore` resources.
 
-- **MySQL** install it with command:
+> The default settings are not production grade:
+> the following scripts are just used to test the Kamaji usage of different drivers.
 
-    `$ make -C deploy/kine/mysql mariadb`
+On the Management Cluster, you can use the following commands:
 
-- **PostgreSQL** install it with command:
+- **MySQL**: `$ make -C deploy/kine/mysql mariadb`
 
-    `$ make -C deploy/kine/postgresql postgresql`
+- **PostgreSQL**: `$ make -C deploy/kine/postgresql postgresql`
 
-## Install Cert Manager
+- **NATS**: `$ make -C deploy/kine/nats nats`
 
-As prerequisite for Kamaji, install the Cert Manager
+## Defining a default Datastore upon Kamaji installation
 
-```bash
-helm repo add jetstack https://charts.jetstack.io
-helm repo update
-helm install \
-  cert-manager jetstack/cert-manager \
-  --namespace cert-manager \
-  --create-namespace \
-  --version v1.11.0 \
-  --set installCRDs=true
-```
+Use Helm to install the Kamaji Operator and make sure it uses a datastore with the proper driver `datastore.driver=<MySQL|PostgreSQL|NATS>`.
+Please refer to the Chart available values for more information on supported options.
 
-## Install Kamaji
-
-Use Helm to install the Kamaji Operator and make sure it uses a datastore with the proper driver `datastore.driver=<MySQL|PostgreSQL>`.
-
-For example, with a PostreSQL datastore installed:
+For example, with a PostgreSQL datastore installed:
 
 ```bash
 helm install kamaji charts/kamaji -n kamaji-system --create-namespace \
@@ -60,4 +49,19 @@ helm install kamaji charts/kamaji -n kamaji-system --create-namespace \
   --set datastore.tlsConfig.clientCertificate.privateKey.keyPath=tls.key
 ```
 
-Once installed, you will able to create Tenant Control Planes using an alternative datastore.
+Once installed, you will be able to create Tenant Control Planes using an alternative datastore.
+
+## Defining specific Datastore per Tenant Control Plane
+
+Each `TenantControlPlane` can refer to a specific `Datastore` thanks to the `/spec/dataStore` field.
+This allows you to implement your preferred sharding or pooling strategy.
+
+When the said key is omitted, Kamaji will use the default datastore configured with its CLI argument `--datastore`.
+
+## NATS considerations
+
+The NATS support is still experimental, mostly because multi-tenancy is **NOT** supported.
+
+> A `NATS` based DataStore can host one and only one Tenant Control Plane.
+> When a `TenantControlPlane` is referring to a NATS `DataStore` already used by another instance,
+> reconciliation will fail and blocked.

docs/content/guides/certs-lifecycle.md

@@ -99,7 +99,7 @@ The rotation will occur the day before their expiration.
 > Nota Bene:
 >
 > Kamaji is responsible for creating the `etcd` client certificate, and the generation of a new one will occur.
-> For other Datastore drivers, such as MySQL or PostgreSQL, the referenced Secret will always be deleted by the Controller to trigger the rotation:
+> For other Datastore drivers, such as MySQL, PostgreSQL, or NATS, the referenced Secret will always be deleted by the Controller to trigger the rotation:
 > the PKI management, since it's offloaded externally, must provide the renewed certificates.
 
 ## Certificate Authority rotation

docs/content/guides/console.md

@@ -69,7 +69,9 @@ From the main view, clicking on a Tenant Control Plane row will bring you to the
 ![Console Tenant Control Plane View](../images/console-tcp-view.png)
 
 ### Working with Datastore
-From the menu bar on the left, clicking on the Datastores item, you can access the list of provisioned Datastores. It shows a summary about datastores, including name and the used driver, i.e. etcd, mysql, and postgresql.
+
+From the menu bar on the left, clicking on the Datastores item, you can access the list of provisioned Datastores.
+It shows a summary about datastores, including name and the used driver, i.e. `etcd`, `MySQL`, `PostgreSQL`, and `NATS`.
 
 ![Console Datastore List](../images/console-ds-list.png)
 

docs/content/guides/datastore-migration.md

@@ -1,6 +1,7 @@
 # Datastore Migration
 
-On the Management Cluster, you can deploy one or more multi-tenant datastores as `etcd`, `PostgreSQL`, and `MySQL` to save the state of the Tenant Clusters. A Tenant Control Plane can be migrated from a datastore to another one without service disruption or without complex and error prone backup & restore procedures.
+On the Management Cluster, you can deploy one or more multi-tenant datastores as `etcd`, `PostgreSQL`, `MySQL`, and `NATS` to save the state of the Tenant Clusters.
+A Tenant Control Plane can be migrated from a datastore to another one without service disruption or without complex and error-prone backup & restore procedures.
 
 This guide will assist you to live migrate Tenant's data from a datastore to another one having the same `etcd` driver.
 
@@ -80,7 +81,7 @@ helm install dedicated clastix/kamaji-etcd -n dedicated --create-namespace --set
 You should end up with a new datastore `dedicated` provided by an `etcd` cluster:
 
 ```yaml
-kubectl get datastore dedicated -o yaml
+# kubectl get datastore dedicated -o yaml
 apiVersion: kamaji.clastix.io/v1alpha1
 kind: DataStore
 metadata:

docs/content/guides/kamaji-azure-deployment.md

@@ -125,7 +125,7 @@ helm install kamaji clastix/kamaji -n kamaji-system --create-namespace
 ```
 
 !!! note "A managed datastore is highly recommended in production"
-     The [kamaji-etcd](https://github.com/clastix/kamaji-etcd) project provides the code to setup a multi-tenant `etcd` running as StatefulSet made of three replicas. Optionally, Kamaji offers support for a more robust storage system, as `MySQL` or `PostgreSQL` compatible database, thanks to the native [kine](https://github.com/k3s-io/kine) integration.
+     The [kamaji-etcd](https://github.com/clastix/kamaji-etcd) project provides the code to setup a multi-tenant `etcd` running as StatefulSet made of three replicas. Optionally, Kamaji offers support for a more robust storage system, as `MySQL`, `PostgreSQL`, or `NATS` compatible database, thanks to the native [kine](https://github.com/k3s-io/kine) integration.
 
 ## Create Tenant Cluster
 

docs/content/reference/versioning.md

@@ -17,3 +17,5 @@ In Kamaji, there are different components that might require independent version
 | v0.4.0 | v1.22+             | [v1.21.0 .. v1.29.0] |
 | v0.4.1 | v1.22+             | [v1.21.0 .. v1.29.1] |
 | v0.4.2 | v1.22+             | [v1.21.0 .. v1.29.1] |
+| v0.5.0 | v1.22+             | [v1.21.0 .. v1.30.0] |
+| v0.6.0 | v1.22+             | [v1.21.0 .. v1.30.1] |

e2e/tcp_custom_sa_test.go

@@ -0,0 +1,74 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package e2e
+
+import (
+	"context"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	pointer "k8s.io/utils/ptr"
+
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+)
+
+var _ = Describe("Deploy a TenantControlPlane with resource with custom service account", func() {
+	// service account object
+	sa := &corev1.ServiceAccount{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test",
+			Namespace: "default",
+		},
+	}
+	// Fill TenantControlPlane object
+	tcp := &kamajiv1alpha1.TenantControlPlane{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "tcp-clusterip-customsa",
+			Namespace: "default",
+		},
+		Spec: kamajiv1alpha1.TenantControlPlaneSpec{
+			ControlPlane: kamajiv1alpha1.ControlPlane{
+				Deployment: kamajiv1alpha1.DeploymentSpec{
+					Replicas:           pointer.To(int32(1)),
+					ServiceAccountName: sa.GetName(),
+				},
+				Service: kamajiv1alpha1.ServiceSpec{
+					ServiceType: "ClusterIP",
+				},
+			},
+			NetworkProfile: kamajiv1alpha1.NetworkProfileSpec{
+				Address: "172.18.0.2",
+			},
+			Kubernetes: kamajiv1alpha1.KubernetesSpec{
+				Version: "v1.23.6",
+				Kubelet: kamajiv1alpha1.KubeletSpec{
+					CGroupFS: "cgroupfs",
+				},
+				AdmissionControllers: kamajiv1alpha1.AdmissionControllers{
+					"LimitRanger",
+					"ResourceQuota",
+				},
+			},
+			Addons: kamajiv1alpha1.AddonsSpec{},
+		},
+	}
+
+	// Create service account and TenantControlPlane resources into the cluster
+	JustBeforeEach(func() {
+		Expect(k8sClient.Create(context.Background(), sa)).NotTo(HaveOccurred())
+		Expect(k8sClient.Create(context.Background(), tcp)).NotTo(HaveOccurred())
+	})
+
+	// Delete the service account and TenantControlPlane resources after test is finished
+	JustAfterEach(func() {
+		Expect(k8sClient.Delete(context.Background(), tcp)).Should(Succeed())
+	})
+	// Check if TenantControlPlane resource has been created and if its pods have the right service account
+	It("Should be Ready and have correct sa", func() {
+		StatusMustEqualTo(tcp, kamajiv1alpha1.VersionReady)
+		PodsServiceAccountMustEqualTo(tcp, sa)
+	})
+})

e2e/tcp_nats_ready_no_tls_test.go

@@ -0,0 +1,54 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package e2e
+
+import (
+	"context"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	pointer "k8s.io/utils/ptr"
+
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+)
+
+var _ = Describe("Deploy a TenantControlPlane resource with the NATS driver without TLS", func() {
+	// Fill TenantControlPlane object
+	tcp := &kamajiv1alpha1.TenantControlPlane{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "nats-notls",
+			Namespace: "default",
+		},
+		Spec: kamajiv1alpha1.TenantControlPlaneSpec{
+			DataStore: "nats-notls",
+			ControlPlane: kamajiv1alpha1.ControlPlane{
+				Deployment: kamajiv1alpha1.DeploymentSpec{
+					Replicas: pointer.To(int32(1)),
+				},
+				Service: kamajiv1alpha1.ServiceSpec{
+					ServiceType: "ClusterIP",
+				},
+			},
+			Kubernetes: kamajiv1alpha1.KubernetesSpec{
+				Version: "v1.23.6",
+				Kubelet: kamajiv1alpha1.KubeletSpec{
+					CGroupFS: "cgroupfs",
+				},
+			},
+		},
+	}
+	// Create a TenantControlPlane resource into the cluster
+	JustBeforeEach(func() {
+		Expect(k8sClient.Create(context.Background(), tcp)).NotTo(HaveOccurred())
+	})
+	// Delete the TenantControlPlane resource after test is finished
+	JustAfterEach(func() {
+		Expect(k8sClient.Delete(context.Background(), tcp)).Should(Succeed())
+	})
+	// Check if TenantControlPlane resource has been created
+	It("Should be Ready", func() {
+		StatusMustEqualTo(tcp, kamajiv1alpha1.VersionReady)
+	})
+})

e2e/tcp_nats_ready_test.go

@@ -0,0 +1,54 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package e2e
+
+import (
+	"context"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	pointer "k8s.io/utils/ptr"
+
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+)
+
+var _ = Describe("Deploy a TenantControlPlane resource with the NATS driver", func() {
+	// Fill TenantControlPlane object
+	tcp := &kamajiv1alpha1.TenantControlPlane{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "nats",
+			Namespace: "default",
+		},
+		Spec: kamajiv1alpha1.TenantControlPlaneSpec{
+			DataStore: "nats-bronze",
+			ControlPlane: kamajiv1alpha1.ControlPlane{
+				Deployment: kamajiv1alpha1.DeploymentSpec{
+					Replicas: pointer.To(int32(1)),
+				},
+				Service: kamajiv1alpha1.ServiceSpec{
+					ServiceType: "ClusterIP",
+				},
+			},
+			Kubernetes: kamajiv1alpha1.KubernetesSpec{
+				Version: "v1.23.6",
+				Kubelet: kamajiv1alpha1.KubeletSpec{
+					CGroupFS: "cgroupfs",
+				},
+			},
+		},
+	}
+	// Create a TenantControlPlane resource into the cluster
+	JustBeforeEach(func() {
+		Expect(k8sClient.Create(context.Background(), tcp)).NotTo(HaveOccurred())
+	})
+	// Delete the TenantControlPlane resource after test is finished
+	JustAfterEach(func() {
+		Expect(k8sClient.Delete(context.Background(), tcp)).Should(Succeed())
+	})
+	// Check if TenantControlPlane resource has been created
+	It("Should be Ready", func() {
+		StatusMustEqualTo(tcp, kamajiv1alpha1.VersionReady)
+	})
+})

e2e/tcp_pod_additional_metadata_test.go

@@ -0,0 +1,78 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package e2e
+
+import (
+	"context"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	pointer "k8s.io/utils/ptr"
+
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+)
+
+var _ = Describe("Deploy a TenantControlPlane resource with additional pod metadata", func() {
+	// Fill TenantControlPlane object
+	tcp := &kamajiv1alpha1.TenantControlPlane{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "tcp-clusterip-additional-metadata",
+			Namespace: "default",
+		},
+		Spec: kamajiv1alpha1.TenantControlPlaneSpec{
+			ControlPlane: kamajiv1alpha1.ControlPlane{
+				Deployment: kamajiv1alpha1.DeploymentSpec{
+					Replicas: pointer.To(int32(1)),
+					PodAdditionalMetadata: kamajiv1alpha1.AdditionalMetadata{
+						Labels: map[string]string{
+							"hello-label": "world",
+							"foo-label":   "bar",
+						},
+						Annotations: map[string]string{
+							"hello-ann": "world",
+							"foo-ann":   "bar",
+						},
+					},
+				},
+				Service: kamajiv1alpha1.ServiceSpec{
+					ServiceType: "ClusterIP",
+				},
+			},
+			NetworkProfile: kamajiv1alpha1.NetworkProfileSpec{
+				Address: "172.18.0.2",
+			},
+			Kubernetes: kamajiv1alpha1.KubernetesSpec{
+				Version: "v1.23.6",
+				Kubelet: kamajiv1alpha1.KubeletSpec{
+					CGroupFS: "cgroupfs",
+				},
+				AdmissionControllers: kamajiv1alpha1.AdmissionControllers{
+					"LimitRanger",
+					"ResourceQuota",
+				},
+			},
+			Addons: kamajiv1alpha1.AddonsSpec{},
+		},
+	}
+
+	// Create a TenantControlPlane resource into the cluster
+	JustBeforeEach(func() {
+		Expect(k8sClient.Create(context.Background(), tcp)).NotTo(HaveOccurred())
+	})
+
+	// Delete the TenantControlPlane resource after test is finished
+	JustAfterEach(func() {
+		Expect(k8sClient.Delete(context.Background(), tcp)).Should(Succeed())
+	})
+
+	// Check if TenantControlPlane resource has been created
+	It("Should be Ready with expected additional metadata", func() {
+		StatusMustEqualTo(tcp, kamajiv1alpha1.VersionReady)
+		AllPodsLabelMustEqualTo(tcp, "hello-label", "world")
+		AllPodsLabelMustEqualTo(tcp, "foo-label", "bar")
+		AllPodsAnnotationMustEqualTo(tcp, "hello-ann", "world")
+		AllPodsAnnotationMustEqualTo(tcp, "foo-ann", "bar")
+	})
+})

e2e/utils_test.go

@@ -17,6 +17,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/kubernetes"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
 )
@@ -136,3 +137,61 @@ func StatusMustEqualTo(tcp *kamajiv1alpha1.TenantControlPlane, status kamajiv1al
 		return *tcp.Status.Kubernetes.Version.Status
 	}, 5*time.Minute, time.Second).Should(Equal(status))
 }
+
+func AllPodsLabelMustEqualTo(tcp *kamajiv1alpha1.TenantControlPlane, label string, value string) {
+	Eventually(func() bool {
+		tcpPods := &corev1.PodList{}
+		err := k8sClient.List(context.Background(), tcpPods, client.MatchingLabels{
+			"kamaji.clastix.io/name": tcp.GetName(),
+		})
+		if err != nil {
+			return false
+		}
+		for _, pod := range tcpPods.Items {
+			if pod.Labels[label] != value {
+				return false
+			}
+		}
+
+		return true
+	}, 5*time.Minute, time.Second).Should(BeTrue())
+}
+
+func AllPodsAnnotationMustEqualTo(tcp *kamajiv1alpha1.TenantControlPlane, annotation string, value string) {
+	Eventually(func() bool {
+		tcpPods := &corev1.PodList{}
+		err := k8sClient.List(context.Background(), tcpPods, client.MatchingLabels{
+			"kamaji.clastix.io/name": tcp.GetName(),
+		})
+		if err != nil {
+			return false
+		}
+		for _, pod := range tcpPods.Items {
+			if pod.Annotations[annotation] != value {
+				return false
+			}
+		}
+
+		return true
+	}, 5*time.Minute, time.Second).Should(BeTrue())
+}
+
+func PodsServiceAccountMustEqualTo(tcp *kamajiv1alpha1.TenantControlPlane, sa *corev1.ServiceAccount) {
+	saName := sa.GetName()
+	Eventually(func() bool {
+		tcpPods := &corev1.PodList{}
+		err := k8sClient.List(context.Background(), tcpPods, client.MatchingLabels{
+			"kamaji.clastix.io/name": tcp.GetName(),
+		})
+		if err != nil {
+			return false
+		}
+		for _, pod := range tcpPods.Items {
+			if pod.Spec.ServiceAccountName != saName {
+				return false
+			}
+		}
+
+		return true
+	}, 5*time.Minute, time.Second).Should(BeTrue())
+}

go.mod

@@ -1,19 +1,22 @@
 module github.com/clastix/kamaji
 
-go 1.21
+go 1.22.0
+
+toolchain go1.22.1
 
 require (
 	github.com/JamesStewy/go-mysqldump v0.2.2
 	github.com/blang/semver v3.5.1+incompatible
-	github.com/go-logr/logr v1.3.0
+	github.com/go-logr/logr v1.4.1
 	github.com/go-pg/pg/v10 v10.10.6
 	github.com/go-sql-driver/mysql v1.6.0
 	github.com/google/go-cmp v0.6.0
 	github.com/google/uuid v1.3.0
 	github.com/json-iterator/go v1.1.12
 	github.com/juju/mutex/v2 v2.0.0
-	github.com/onsi/ginkgo/v2 v2.13.0
-	github.com/onsi/gomega v1.29.0
+	github.com/nats-io/nats.go v1.34.1
+	github.com/onsi/ginkgo/v2 v2.17.1
+	github.com/onsi/gomega v1.32.0
 	github.com/pkg/errors v0.9.1
 	github.com/spf13/cobra v1.7.0
 	github.com/spf13/pflag v1.0.5
@@ -23,16 +26,16 @@ require (
 	go.etcd.io/etcd/client/v3 v3.5.10
 	go.uber.org/automaxprocs v1.5.1
 	gomodules.xyz/jsonpatch/v2 v2.4.0
-	k8s.io/api v0.29.1
-	k8s.io/apimachinery v0.29.1
-	k8s.io/apiserver v0.29.1
-	k8s.io/client-go v0.29.1
+	k8s.io/api v0.30.1
+	k8s.io/apimachinery v0.30.1
+	k8s.io/apiserver v0.30.1
+	k8s.io/client-go v0.30.1
 	k8s.io/cluster-bootstrap v0.0.0
-	k8s.io/klog/v2 v2.110.1
+	k8s.io/klog/v2 v2.120.1
 	k8s.io/kubelet v0.0.0
-	k8s.io/kubernetes v1.29.1
+	k8s.io/kubernetes v1.30.1
 	k8s.io/utils v0.0.0-20230726121419-3b25d923346b
-	sigs.k8s.io/controller-runtime v0.16.3
+	sigs.k8s.io/controller-runtime v0.17.1-0.20240416095710-67b27f27e514
 )
 
 require (
@@ -58,10 +61,10 @@ require (
 	github.com/docker/go-units v0.5.0 // indirect
 	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
 	github.com/evanphx/json-patch v5.6.0+incompatible // indirect
-	github.com/evanphx/json-patch/v5 v5.6.0 // indirect
+	github.com/evanphx/json-patch/v5 v5.9.0 // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/go-errors/errors v1.4.2 // indirect
-	github.com/go-logr/zapr v1.2.4 // indirect
+	github.com/go-logr/zapr v1.3.0 // indirect
 	github.com/go-openapi/jsonpointer v0.19.6 // indirect
 	github.com/go-openapi/jsonreference v0.20.2 // indirect
 	github.com/go-openapi/swag v0.22.3 // indirect
@@ -69,7 +72,7 @@ require (
 	github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
-	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/google/btree v1.0.1 // indirect
 	github.com/google/gnostic-models v0.6.8 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
@@ -82,11 +85,12 @@ require (
 	github.com/jinzhu/inflection v1.0.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/juju/errors v0.0.0-20220203013757-bd733f3c86b9 // indirect
+	github.com/klauspost/compress v1.17.8 // indirect
 	github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
 	github.com/lithammer/dedent v1.1.0 // indirect
 	github.com/magiconair/properties v1.8.5 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
-	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
+	github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect
 	github.com/mitchellh/mapstructure v1.4.3 // indirect
 	github.com/moby/sys/mount v0.2.0 // indirect
 	github.com/moby/sys/mountinfo v0.6.2 // indirect
@@ -96,16 +100,18 @@ require (
 	github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
 	github.com/morikuni/aec v1.0.0 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/nats-io/nkeys v0.4.7 // indirect
+	github.com/nats-io/nuid v1.0.1 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.0.2 // indirect
-	github.com/opencontainers/runc v1.1.10 // indirect
+	github.com/opencontainers/runc v1.1.12 // indirect
 	github.com/pelletier/go-toml v1.9.4 // indirect
 	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/client_golang v1.16.0 // indirect
-	github.com/prometheus/client_model v0.4.0 // indirect
-	github.com/prometheus/common v0.44.0 // indirect
-	github.com/prometheus/procfs v0.10.1 // indirect
+	github.com/prometheus/client_golang v1.18.0 // indirect
+	github.com/prometheus/client_model v0.5.0 // indirect
+	github.com/prometheus/common v0.45.0 // indirect
+	github.com/prometheus/procfs v0.12.0 // indirect
 	github.com/sirupsen/logrus v1.9.0 // indirect
 	github.com/spf13/afero v1.9.2 // indirect
 	github.com/spf13/cast v1.4.1 // indirect
@@ -121,34 +127,34 @@ require (
 	go.opencensus.io v0.24.0 // indirect
 	go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect
 	go.uber.org/multierr v1.11.0 // indirect
-	go.uber.org/zap v1.25.0 // indirect
-	golang.org/x/crypto v0.16.0 // indirect
+	go.uber.org/zap v1.26.0 // indirect
+	golang.org/x/crypto v0.22.0 // indirect
 	golang.org/x/exp v0.0.0-20220827204233-334a2380cb91 // indirect
-	golang.org/x/mod v0.14.0 // indirect
-	golang.org/x/net v0.19.0 // indirect
-	golang.org/x/oauth2 v0.10.0 // indirect
-	golang.org/x/sync v0.5.0 // indirect
-	golang.org/x/sys v0.15.0 // indirect
-	golang.org/x/term v0.15.0 // indirect
+	golang.org/x/mod v0.15.0 // indirect
+	golang.org/x/net v0.23.0 // indirect
+	golang.org/x/oauth2 v0.12.0 // indirect
+	golang.org/x/sync v0.6.0 // indirect
+	golang.org/x/sys v0.19.0 // indirect
+	golang.org/x/term v0.19.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
-	golang.org/x/tools v0.16.1 // indirect
+	golang.org/x/tools v0.18.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20230726155614-23370e0ffb3e // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect
 	google.golang.org/grpc v1.58.3 // indirect
-	google.golang.org/protobuf v1.31.0 // indirect
+	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/ini.v1 v1.66.2 // indirect
 	gopkg.in/square/go-jose.v2 v2.6.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	k8s.io/apiextensions-apiserver v0.29.1 // indirect
-	k8s.io/cli-runtime v0.29.1 // indirect
-	k8s.io/component-base v0.29.1 // indirect
+	k8s.io/apiextensions-apiserver v0.30.1 // indirect
+	k8s.io/cli-runtime v0.30.1 // indirect
+	k8s.io/component-base v0.30.1 // indirect
 	k8s.io/component-helpers v0.0.0 // indirect
-	k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect
+	k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect
 	k8s.io/kube-proxy v0.0.0 // indirect
 	k8s.io/system-validators v1.8.0 // indirect
 	mellium.im/sasl v0.3.0 // indirect
@@ -156,37 +162,37 @@ require (
 	sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3 // indirect
 	sigs.k8s.io/kustomize/kyaml v0.14.3-0.20230601165947-6ce0bf390ce3 // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
-	sigs.k8s.io/yaml v1.3.0 // indirect
+	sigs.k8s.io/yaml v1.4.0 // indirect
 )
 
 replace (
-	k8s.io/api => k8s.io/api v0.29.1
-	k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.29.1
-	k8s.io/apimachinery => k8s.io/apimachinery v0.29.1
-	k8s.io/apiserver => k8s.io/apiserver v0.29.1
-	k8s.io/cli-runtime => k8s.io/cli-runtime v0.29.1
-	k8s.io/client-go => k8s.io/client-go v0.29.1
-	k8s.io/cloud-provider => k8s.io/cloud-provider v0.29.1
-	k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.29.1
-	k8s.io/code-generator => k8s.io/code-generator v0.29.1
-	k8s.io/component-base => k8s.io/component-base v0.29.1
-	k8s.io/component-helpers => k8s.io/component-helpers v0.29.1
-	k8s.io/controller-manager => k8s.io/controller-manager v0.29.1
-	k8s.io/cri-api => k8s.io/cri-api v0.29.1
-	k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.29.1
-	k8s.io/dynamic-resource-allocation => k8s.io/dynamic-resource-allocation v0.29.1
-	k8s.io/endpointslice => k8s.io/endpointslice v0.29.1
-	k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.29.1
-	k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.29.1
-	k8s.io/kube-proxy => k8s.io/kube-proxy v0.29.1
-	k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.29.1
-	k8s.io/kubectl => k8s.io/kubectl v0.29.1
-	k8s.io/kubelet => k8s.io/kubelet v0.29.1
-	k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.29.1
-	k8s.io/metrics => k8s.io/metrics v0.29.1
-	k8s.io/mount-utils => k8s.io/mount-utils v0.29.1
-	k8s.io/pod-security-admission => k8s.io/pod-security-admission v0.29.1
-	k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.29.1
+	k8s.io/api => k8s.io/api v0.30.1
+	k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.30.1
+	k8s.io/apimachinery => k8s.io/apimachinery v0.30.1
+	k8s.io/apiserver => k8s.io/apiserver v0.30.1
+	k8s.io/cli-runtime => k8s.io/cli-runtime v0.30.1
+	k8s.io/client-go => k8s.io/client-go v0.30.1
+	k8s.io/cloud-provider => k8s.io/cloud-provider v0.30.1
+	k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.30.1
+	k8s.io/code-generator => k8s.io/code-generator v0.30.1
+	k8s.io/component-base => k8s.io/component-base v0.30.1
+	k8s.io/component-helpers => k8s.io/component-helpers v0.30.1
+	k8s.io/controller-manager => k8s.io/controller-manager v0.30.1
+	k8s.io/cri-api => k8s.io/cri-api v0.30.1
+	k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.30.1
+	k8s.io/dynamic-resource-allocation => k8s.io/dynamic-resource-allocation v0.30.1
+	k8s.io/endpointslice => k8s.io/endpointslice v0.30.1
+	k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.30.1
+	k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.30.1
+	k8s.io/kube-proxy => k8s.io/kube-proxy v0.30.1
+	k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.30.1
+	k8s.io/kubectl => k8s.io/kubectl v0.30.1
+	k8s.io/kubelet => k8s.io/kubelet v0.30.1
+	k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.30.1
+	k8s.io/metrics => k8s.io/metrics v0.30.1
+	k8s.io/mount-utils => k8s.io/mount-utils v0.30.1
+	k8s.io/pod-security-admission => k8s.io/pod-security-admission v0.30.1
+	k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.30.1
 )
 
 replace github.com/JamesStewy/go-mysqldump => github.com/vtoma/go-mysqldump v1.0.0

go.sum

@@ -817,9 +817,6 @@ github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0=
-github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
-github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A=
-github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
 github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
@@ -1063,8 +1060,8 @@ github.com/envoyproxy/protoc-gen-validate v1.0.2/go.mod h1:GpiZQP3dDbg4JouG/NNS7
 github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U=
 github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
-github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww=
-github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4=
+github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg=
+github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
 github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
@@ -1077,6 +1074,7 @@ github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4
 github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
 github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
 github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa/go.mod h1:KnogPXtdwXqoenmZCw6S+25EAm2MkxbG0deNDu4cbSA=
+github.com/fxamacker/cbor/v2 v2.6.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ=
 github.com/garyburd/redigo v0.0.0-20150301180006-535138d7bcd7/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
@@ -1107,12 +1105,12 @@ github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbV
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
-github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY=
 github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ=
+github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
-github.com/go-logr/zapr v1.2.3/go.mod h1:eIauM6P8qSvTw5o2ez6UEAfGjQKrxQTl5EoK+Qa2oG4=
-github.com/go-logr/zapr v1.2.4 h1:QHVo+6stLbfJmYGkQ7uGHUCu5hnAFAj6mDe6Ea0SeOo=
-github.com/go-logr/zapr v1.2.4/go.mod h1:FyHWQIzQORZ0QVE1BtVHv3cKtNLuXsbNLtpuhNapBOA=
+github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ=
+github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg=
 github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE=
 github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs=
 github.com/go-openapi/jsonreference v0.20.1/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k=
@@ -1187,15 +1185,16 @@ github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
 github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM=
 github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
-github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
 github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
+github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
 github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4=
 github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA=
-github.com/google/cel-go v0.17.7/go.mod h1:HXZKzB0LXqer5lHHgfWAnlYwJaQBDKMjxjulNQzhwhY=
+github.com/google/cel-go v0.17.8/go.mod h1:HXZKzB0LXqer5lHHgfWAnlYwJaQBDKMjxjulNQzhwhY=
 github.com/google/flatbuffers v2.0.8+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
 github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I=
 github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U=
@@ -1328,7 +1327,6 @@ github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANyt
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
 github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA=
-github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
 github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
 github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
@@ -1380,6 +1378,8 @@ github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j
 github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
 github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
 github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
+github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU=
+github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
 github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
@@ -1387,7 +1387,6 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxv
 github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
-github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
@@ -1421,8 +1420,9 @@ github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vq
 github.com/mattn/go-sqlite3 v1.14.14/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
 github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
-github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
 github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
+github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg=
+github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k=
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
 github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs=
 github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY=
@@ -1471,6 +1471,12 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8m
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
+github.com/nats-io/nats.go v1.34.1 h1:syWey5xaNHZgicYBemv0nohUPPmaLteiBEUT6Q5+F/4=
+github.com/nats-io/nats.go v1.34.1/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8=
+github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI=
+github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc=
+github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
+github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
 github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
@@ -1499,8 +1505,11 @@ github.com/onsi/ginkgo/v2 v2.9.2/go.mod h1:WHcJJG2dIlcCqVfBAwUCrJxSPFb6v4azBwgxe
 github.com/onsi/ginkgo/v2 v2.9.5/go.mod h1:tvAoo1QUJwNEU2ITftXTpR7R1RbCzoZUOs3RonqW57k=
 github.com/onsi/ginkgo/v2 v2.9.7/go.mod h1:cxrmXWykAwTwhQsJOPfdIDiJ+l2RYq7U8hFU+M/1uw0=
 github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM=
-github.com/onsi/ginkgo/v2 v2.13.0 h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4=
 github.com/onsi/ginkgo/v2 v2.13.0/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o=
+github.com/onsi/ginkgo/v2 v2.13.2/go.mod h1:XStQ8QcGwLyF4HdfcZB8SFOS/MWCgDuXMSBe6zrvLgM=
+github.com/onsi/ginkgo/v2 v2.15.0/go.mod h1:HlxMHtYF57y6Dpf+mc5529KKmSq9h2FpCF+/ZkwUxKM=
+github.com/onsi/ginkgo/v2 v2.17.1 h1:V++EzdbhI4ZV4ev0UTIj0PzhzOcReJFyJaLjtSF55M8=
+github.com/onsi/ginkgo/v2 v2.17.1/go.mod h1:llBI3WDLL9Z6taip6f33H76YcWtJv+7R3HigUjbIBOs=
 github.com/onsi/gomega v0.0.0-20151007035656-2152b45fa28a/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
@@ -1521,8 +1530,11 @@ github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+q
 github.com/onsi/gomega v1.27.7/go.mod h1:1p8OOlwo2iUUDsHnOrjE5UKYJ+e3W8eQ3qSlRahPmr4=
 github.com/onsi/gomega v1.27.8/go.mod h1:2J8vzI/s+2shY9XHRApDkdgPo1TKT7P2u6fXeJKFnNQ=
 github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M=
-github.com/onsi/gomega v1.29.0 h1:KIA/t2t5UBzoirT4H9tsML45GEbo3ouUnBHsCfD2tVg=
 github.com/onsi/gomega v1.29.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ=
+github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ=
+github.com/onsi/gomega v1.31.0/go.mod h1:DW9aCi7U6Yi40wNVAvT6kzFnEVEI5n3DloYBiKiT6zk=
+github.com/onsi/gomega v1.32.0 h1:JRYU78fJ1LPxlckP6Txi/EYqJvjtMrDC04/MM5XRHPk=
+github.com/onsi/gomega v1.32.0/go.mod h1:a4x4gW6Pz2yK1MAmvluYme5lvYTn61afQ2ETw/8n4Lg=
 github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
 github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
 github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
@@ -1539,8 +1551,8 @@ github.com/opencontainers/runc v1.0.0-rc8.0.20190926000215-3e425f80a8c9/go.mod h
 github.com/opencontainers/runc v1.0.0-rc9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
 github.com/opencontainers/runc v1.0.0-rc93/go.mod h1:3NOsor4w32B2tC0Zbl8Knk4Wg84SM2ImC1fxBuqJ/H0=
 github.com/opencontainers/runc v1.0.2/go.mod h1:aTaHFFwQXuA71CiyxOdFFIorAoemI04suvGRQFzWTD0=
-github.com/opencontainers/runc v1.1.10 h1:EaL5WeO9lv9wmS6SASjszOeQdSctvpbu0DdBQBizE40=
-github.com/opencontainers/runc v1.1.10/go.mod h1:+/R6+KmDlh+hOO8NkjmgkG9Qzvypzk0yXxAPYYR65+M=
+github.com/opencontainers/runc v1.1.12 h1:BOIssBaW1La0/qbNZHXOOa71dZfZEQOzW7dqQf3phss=
+github.com/opencontainers/runc v1.1.12/go.mod h1:S+lQwSfncpBha7XTy/5lBwWgm5+y5Ma/O44Ekby9FK8=
 github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
 github.com/opencontainers/runtime-spec v1.0.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
 github.com/opencontainers/runtime-spec v1.0.2-0.20190207185410-29686dbc5559/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
@@ -1588,16 +1600,18 @@ github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqr
 github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
 github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y=
 github.com/prometheus/client_golang v1.15.1/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk=
-github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8=
 github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc=
+github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk=
+github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA=
 github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w=
-github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY=
 github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU=
+github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw=
+github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI=
 github.com/prometheus/common v0.0.0-20180110214958-89604d197083/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
 github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
 github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
@@ -1608,8 +1622,9 @@ github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9
 github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
 github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA=
 github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc=
-github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY=
 github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY=
+github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM=
+github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY=
 github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
@@ -1623,8 +1638,9 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1
 github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
 github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4=
 github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY=
-github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg=
 github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM=
+github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo=
+github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
 github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
@@ -1746,6 +1762,7 @@ github.com/vtoma/go-mysqldump v1.0.0 h1:TNQXlsGD4r1+P9cMyzeWnOgHy0QbP0R+XfXqu5Is
 github.com/vtoma/go-mysqldump v1.0.0/go.mod h1:i9PUM5mb4MH+4D4tJktCTLWDy6CX5rDCL/nS4+f3N5w=
 github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
 github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI=
+github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
 github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
 github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
 github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs=
@@ -1821,24 +1838,21 @@ go.starlark.net v0.0.0-20230525235612-a134d8f9ddca h1:VdD38733bfYv5tUZwEIskMM93V
 go.starlark.net v0.0.0-20230525235612-a134d8f9ddca/go.mod h1:jxU+3+j+71eXOW14274+SmmuW82qJzl6iZSeqEtTGds=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
-go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
 go.uber.org/automaxprocs v1.5.1 h1:e1YG66Lrk73dn4qhg8WFSvhF0JuFQF0ERIp4rpuV8Qk=
 go.uber.org/automaxprocs v1.5.1/go.mod h1:BF4eumQw0P9GtnuxxovUd06vwm1o18oMzFtK66vU6XU=
-go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=
-go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
 go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo=
-go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A=
 go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4=
+go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
+go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
 go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
+go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
 go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
 go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo=
-go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI=
-go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg=
-go.uber.org/zap v1.25.0 h1:4Hvk6GtkucQ790dqmj7l1eEnRdKm3k3ZUrUMS2d5+5c=
-go.uber.org/zap v1.25.0/go.mod h1:JIAUzQIH94IC4fOJQm7gMmBJP5k7wQfdcnYdPoEXJYk=
+go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo=
+go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so=
 golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20180910181607-0e37d006457b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
@@ -1865,8 +1879,11 @@ golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIi
 golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw=
 golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
 golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
-golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY=
 golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
+golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
+golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
+golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30=
+golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
 golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -1931,8 +1948,10 @@ golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
-golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0=
+golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/mod v0.15.0 h1:SernR4v+D55NyBH2QiEQrlBAnj1ECL6AGrA5+dPaMY8=
+golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -2015,9 +2034,12 @@ golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ=
 golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA=
 golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI=
 golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
+golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
 golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
-golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c=
 golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U=
+golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
+golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
+golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -2048,8 +2070,9 @@ golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I
 golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw=
 golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4=
 golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE=
-golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8=
 golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI=
+golang.org/x/oauth2 v0.12.0 h1:smVPGxink+n1ZI5pkQa8y6fZT0RW0MgCO5bFpepy4B4=
+golang.org/x/oauth2 v0.12.0/go.mod h1:A74bZ3aGXgCY0qaIC9Ahg6Lglin4AMAco8cIv9baba4=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -2068,8 +2091,10 @@ golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
-golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE=
+golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
 golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ=
+golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -2203,8 +2228,13 @@ golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
+golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
+golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/telemetry v0.0.0-20240208230135-b75ee8823808/go.mod h1:KG1lNk5ZFNssSZLrpVb4sMXKMpGwGXOxSG3rnu2gZQQ=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
@@ -2221,8 +2251,11 @@ golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o=
 golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU=
 golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
 golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
-golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4=
 golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0=
+golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
+golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
+golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q=
+golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -2275,7 +2308,6 @@ golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtn
 golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20190927191325-030b2cf1153e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
@@ -2296,7 +2328,6 @@ golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjs
 golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
 golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
 golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
@@ -2333,8 +2364,10 @@ golang.org/x/tools v0.9.3/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc=
 golang.org/x/tools v0.10.0/go.mod h1:UJwyiVBsOA2uwvK/e5OY3GTpDUJriEd+/YlqAwLPmyM=
 golang.org/x/tools v0.12.0/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM=
 golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
-golang.org/x/tools v0.16.1 h1:TLyB3WofjdOEepBHAU20JdNC1Zbg87elYofWYAY5oZA=
+golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg=
 golang.org/x/tools v0.16.1/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0=
+golang.org/x/tools v0.18.0 h1:k8NLag8AGHnn+PHbl7g43CtqZAwG60vZkLqgyZgIHgQ=
+golang.org/x/tools v0.18.0/go.mod h1:GL7B4CwcLLeo59yx/9UWWuNOW1n3VZ4f5axWfML7Lcg=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -2638,6 +2671,7 @@ google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwS
 google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g=
 google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8=
 google.golang.org/grpc v1.56.2/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s=
+google.golang.org/grpc v1.56.3/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s=
 google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo=
 google.golang.org/grpc v1.58.2/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0=
 google.golang.org/grpc v1.58.3 h1:BjnpXut1btbtgN/6sp+brB2Kbm2LjNXnidYujAVbSoQ=
@@ -2660,8 +2694,9 @@ google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqw
 google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 google.golang.org/protobuf v1.29.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
-google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
 google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
+google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
 gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -2716,46 +2751,45 @@ honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt
 honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las=
-k8s.io/api v0.29.1 h1:DAjwWX/9YT7NQD4INu49ROJuZAAAP/Ijki48GUPzxqw=
-k8s.io/api v0.29.1/go.mod h1:7Kl10vBRUXhnQQI8YR/R327zXC8eJ7887/+Ybta+RoQ=
-k8s.io/apiextensions-apiserver v0.29.1 h1:S9xOtyk9M3Sk1tIpQMu9wXHm5O2MX6Y1kIpPMimZBZw=
-k8s.io/apiextensions-apiserver v0.29.1/go.mod h1:zZECpujY5yTW58co8V2EQR4BD6A9pktVgHhvc0uLfeU=
-k8s.io/apimachinery v0.29.1 h1:KY4/E6km/wLBguvCZv8cKTeOwwOBqFNjwJIdMkMbbRc=
-k8s.io/apimachinery v0.29.1/go.mod h1:6HVkd1FwxIagpYrHSwJlQqZI3G9LfYWRPAkUvLnXTKU=
-k8s.io/apiserver v0.29.1 h1:e2wwHUfEmMsa8+cuft8MT56+16EONIEK8A/gpBSco+g=
-k8s.io/apiserver v0.29.1/go.mod h1:V0EpkTRrJymyVT3M49we8uh2RvXf7fWC5XLB0P3SwRw=
-k8s.io/cli-runtime v0.29.1 h1:By3WVOlEWYfyxhGko0f/IuAOLQcbBSMzwSaDren2JUs=
-k8s.io/cli-runtime v0.29.1/go.mod h1:vjEY9slFp8j8UoMhV5AlO8uulX9xk6ogfIesHobyBDU=
-k8s.io/client-go v0.29.1 h1:19B/+2NGEwnFLzt0uB5kNJnfTsbV8w6TgQRz9l7ti7A=
-k8s.io/client-go v0.29.1/go.mod h1:TDG/psL9hdet0TI9mGyHJSgRkW3H9JZk2dNEUS7bRks=
-k8s.io/cluster-bootstrap v0.29.1 h1:YZ7fAY9DT1ZQVU6JFPWjpgm7DzSoTs50r6cI/OP+vLA=
-k8s.io/cluster-bootstrap v0.29.1/go.mod h1:T8sLsyIaQg2aIRdWxrBPYm8pyk+9Xxy+DJxJarv2MnM=
-k8s.io/component-base v0.29.1 h1:MUimqJPCRnnHsskTTjKD+IC1EHBbRCVyi37IoFBrkYw=
-k8s.io/component-base v0.29.1/go.mod h1:fP9GFjxYrLERq1GcWWZAE3bqbNcDKDytn2srWuHTtKc=
-k8s.io/component-helpers v0.29.1 h1:54MMEDu6xeJmMtAKztsPwu0kJKr4+jCUzaEIn2UXRoc=
-k8s.io/component-helpers v0.29.1/go.mod h1:+I7xz4kfUgxWAPJIVKrqe4ml4rb9UGpazlOmhXYo+cY=
-k8s.io/cri-api v0.29.1/go.mod h1:9fQTFm+wi4FLyqrkVUoMJiUB3mE74XrVvHz8uFY/sSw=
-k8s.io/gengo v0.0.0-20230829151522-9cce18d56c01/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E=
+k8s.io/api v0.30.1 h1:kCm/6mADMdbAxmIh0LBjS54nQBE+U4KmbCfIkF5CpJY=
+k8s.io/api v0.30.1/go.mod h1:ddbN2C0+0DIiPntan/bye3SW3PdwLa11/0yqwvuRrJM=
+k8s.io/apiextensions-apiserver v0.30.1 h1:4fAJZ9985BmpJG6PkoxVRpXv9vmPUOVzl614xarePws=
+k8s.io/apiextensions-apiserver v0.30.1/go.mod h1:R4GuSrlhgq43oRY9sF2IToFh7PVlF1JjfWdoG3pixk4=
+k8s.io/apimachinery v0.30.1 h1:ZQStsEfo4n65yAdlGTfP/uSHMQSoYzU/oeEbkmF7P2U=
+k8s.io/apimachinery v0.30.1/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc=
+k8s.io/apiserver v0.30.1 h1:BEWEe8bzS12nMtDKXzCF5Q5ovp6LjjYkSp8qOPk8LZ8=
+k8s.io/apiserver v0.30.1/go.mod h1:i87ZnQ+/PGAmSbD/iEKM68bm1D5reX8fO4Ito4B01mo=
+k8s.io/cli-runtime v0.30.1 h1:kSBBpfrJGS6lllc24KeniI9JN7ckOOJKnmFYH1RpTOw=
+k8s.io/cli-runtime v0.30.1/go.mod h1:zhHgbqI4J00pxb6gM3gJPVf2ysDjhQmQtnTxnMScab8=
+k8s.io/client-go v0.30.1 h1:uC/Ir6A3R46wdkgCV3vbLyNOYyCJ8oZnjtJGKfytl/Q=
+k8s.io/client-go v0.30.1/go.mod h1:wrAqLNs2trwiCH/wxxmT/x3hKVH9PuV0GGW0oDoHVqc=
+k8s.io/cluster-bootstrap v0.30.1 h1:WHh04Oh0YAWMsJ5TXXEF+LGu3g/2ymQOYsH8IopUHlQ=
+k8s.io/cluster-bootstrap v0.30.1/go.mod h1:GcLD4Z4GFY+CsNYcaqlSAmxFIHgSMGZHNT4SjA0A6ao=
+k8s.io/component-base v0.30.1 h1:bvAtlPh1UrdaZL20D9+sWxsJljMi0QZ3Lmw+kmZAaxQ=
+k8s.io/component-base v0.30.1/go.mod h1:e/X9kDiOebwlI41AvBHuWdqFriSRrX50CdwA9TFaHLI=
+k8s.io/component-helpers v0.30.1 h1:/UcxSLzZ0owluTE2WMDrFfZl2L+WVXKdYYYm68qnH7U=
+k8s.io/component-helpers v0.30.1/go.mod h1:b1Xk27UJ3p/AmPqDx7khrnSxrdwQy9gTP7O1y6MZ6rg=
+k8s.io/cri-api v0.30.1/go.mod h1://4/umPJSW1ISNSNng4OwjpkvswJOQwU8rnkvO8P+xg=
+k8s.io/gengo/v2 v2.0.0-20240228010128-51d4e06bde70/go.mod h1:VH3AT8AaQOqiGjMF9p0/IM1Dj+82ZwjfxUP1IxaHE+8=
 k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
 k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
 k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
 k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
-k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0=
-k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo=
-k8s.io/kms v0.29.1/go.mod h1:Hqkx3zEGWThUTbcSkK508DUv4c1HOJOB5qihSoLBWgU=
-k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780=
-k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA=
-k8s.io/kube-proxy v0.29.1 h1:UArwF7uYSFBSftOHfISjygkfIZ4j30FWUQ1TKySZIe8=
-k8s.io/kube-proxy v0.29.1/go.mod h1:VrsEJg4mLKNWiQkewom0uJUfY8XFtf3ZA6nMH2Vx3MM=
-k8s.io/kubelet v0.29.1 h1:cso8Dk8dymkj8q+EvW/aCbIYU2aOkH27gho48tYza/8=
-k8s.io/kubelet v0.29.1/go.mod h1:hTl/naFcCVG1Ku17fMgj/krbheBwBkf3gnFhaboMx7E=
+k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw=
+k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE=
+k8s.io/kms v0.30.1/go.mod h1:GrMurD0qk3G4yNgGcsCEmepqf9KyyIrTXYR2lyUOJC4=
+k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag=
+k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98=
+k8s.io/kube-proxy v0.30.1 h1:lDE510UHimfyRfh/c4Erlgg9OEDmHnZce3KKERV1JtI=
+k8s.io/kube-proxy v0.30.1/go.mod h1:5aRCUPUowbrps6o8oTJ0oRNVLIvcZvnesGZFFMEWZ08=
+k8s.io/kubelet v0.30.1 h1:6gS1gWjrefUGfC/9n0ITOzxnKyt89FfkIhom70Bola4=
+k8s.io/kubelet v0.30.1/go.mod h1:5IUeAt3YlIfLNdT/YfRuCCONfEefm7qfcqz81b002Z8=
 k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk=
-k8s.io/kubernetes v1.29.1 h1:fxJFVb8uqbYZDYHpwIsAndBQs360cQGb0xa1gYFh3fo=
-k8s.io/kubernetes v1.29.1/go.mod h1:xZPKU0yO0CBbLTnbd+XGyRmmtmaVuJykDb8gNCkeeUE=
+k8s.io/kubernetes v1.30.1 h1:XlqS6KslLEA5mQzLK2AJrhr4Z1m8oJfkhHiWJ5lue+I=
+k8s.io/kubernetes v1.30.1/go.mod h1:yPbIk3MhmhGigX62FLJm+CphNtjxqCvAIFQXup6RKS0=
 k8s.io/system-validators v1.8.0 h1:tq05tdO9zdJZnNF3SXrq6LE7Knc/KfJm5wk68467JDg=
 k8s.io/system-validators v1.8.0/go.mod h1:gP1Ky+R9wtrSiFbrpEPwWMeYz9yqyy1S/KOh0Vci7WI=
 k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
-k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI=
 k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
 lukechampine.com/uint128 v1.1.1/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk=
@@ -2816,9 +2850,9 @@ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8
 rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
-sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.28.0/go.mod h1:VHVDI/KrK4fjnV61bE2g3sA7tiETLn8sooImelsCx3Y=
-sigs.k8s.io/controller-runtime v0.16.3 h1:2TuvuokmfXvDUamSx1SuAOO3eTyye+47mJCigwG62c4=
-sigs.k8s.io/controller-runtime v0.16.3/go.mod h1:j7bialYoSn142nv9sCOJmQgDXQXxnroFU4VnX/brVJ0=
+sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.29.0/go.mod h1:z7+wmGM2dfIiLRfrC6jb5kV2Mq/sK1ZP303cxzkV5Y4=
+sigs.k8s.io/controller-runtime v0.17.1-0.20240416095710-67b27f27e514 h1:jHkHzzQ/HXBagSiTfTnfGVdPhE7O0jC7QRXDZ9MK4Xg=
+sigs.k8s.io/controller-runtime v0.17.1-0.20240416095710-67b27f27e514/go.mod h1:TLM3OvUJgcqHVBLVRlNylmfbOlOukMLFHtc6jo3EtIQ=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
 sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3 h1:XX3Ajgzov2RKUdc5jW3t5jwY7Bo7dcRm+tFxT+NfgY0=
@@ -2829,5 +2863,6 @@ sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ih
 sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4=
 sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08=
 sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
-sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=
 sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=
+sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E=
+sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY=

internal/builders/controlplane/deployment.go

@@ -61,7 +61,8 @@ func (d Deployment) Build(ctx context.Context, deployment *appsv1.Deployment, te
 
 	d.setLabels(deployment, utilities.MergeMaps(utilities.KamajiLabels(tenantControlPlane.GetName(), "deployment"), tenantControlPlane.Spec.ControlPlane.Deployment.AdditionalMetadata.Labels))
 	d.setAnnotations(deployment, utilities.MergeMaps(deployment.Annotations, tenantControlPlane.Spec.ControlPlane.Deployment.AdditionalMetadata.Annotations))
-	d.setTemplateLabels(&deployment.Spec.Template, d.templateLabels(ctx, &tenantControlPlane))
+	d.setTemplateLabels(&deployment.Spec.Template, utilities.MergeMaps(d.templateLabels(ctx, &tenantControlPlane), tenantControlPlane.Spec.ControlPlane.Deployment.PodAdditionalMetadata.Labels))
+	d.setTemplateAnnotations(&deployment.Spec.Template, tenantControlPlane.Spec.ControlPlane.Deployment.PodAdditionalMetadata.Annotations)
 	d.setNodeSelector(&deployment.Spec.Template.Spec, tenantControlPlane)
 	d.setToleration(&deployment.Spec.Template.Spec, tenantControlPlane)
 	d.setAffinity(&deployment.Spec.Template.Spec, tenantControlPlane)
@@ -76,6 +77,7 @@ func (d Deployment) Build(ctx context.Context, deployment *appsv1.Deployment, te
 	d.setContainers(&deployment.Spec.Template.Spec, tenantControlPlane, address)
 	d.setAdditionalVolumes(&deployment.Spec.Template.Spec, tenantControlPlane)
 	d.setVolumes(&deployment.Spec.Template.Spec, tenantControlPlane)
+	d.setServiceAccount(&deployment.Spec.Template.Spec, tenantControlPlane)
 	d.Client.Scheme().Default(deployment)
 }
 
@@ -708,7 +710,7 @@ func (d Deployment) buildKubeAPIServerCommand(tenantControlPlane kamajiv1alpha1.
 	}
 
 	switch d.DataStore.Spec.Driver {
-	case kamajiv1alpha1.KineMySQLDriver, kamajiv1alpha1.KinePostgreSQLDriver:
+	case kamajiv1alpha1.KineMySQLDriver, kamajiv1alpha1.KinePostgreSQLDriver, kamajiv1alpha1.KineNatsDriver:
 		desiredArgs["--etcd-servers"] = "http://127.0.0.1:2379"
 	case kamajiv1alpha1.EtcdDriver:
 		httpsEndpoints := make([]string, 0, len(d.DataStore.Spec.Endpoints))
@@ -803,7 +805,10 @@ func (d Deployment) removeKineContainers(podSpec *corev1.PodSpec) {
 
 		podSpec.Containers = containers
 	}
-	// Removing the kine init-container, if present
+	d.removeKineInitContainers(podSpec)
+}
+
+func (d Deployment) removeKineInitContainers(podSpec *corev1.PodSpec) {
 	if found, index := utilities.HasNamedContainer(podSpec.InitContainers, kineInitContainerName); found {
 		var initContainers []corev1.Container
 
@@ -821,45 +826,67 @@ func (d Deployment) buildKine(podSpec *corev1.PodSpec, tcp kamajiv1alpha1.Tenant
 
 		return
 	}
-	// Ensuring the init container required for kine is present:
-	// a chmod is required for kine in order to read the certificates to connect to the secured datastore.
-	found, index := utilities.HasNamedContainer(podSpec.InitContainers, kineInitContainerName)
-	if !found {
-		index = len(podSpec.InitContainers)
-		podSpec.InitContainers = append(podSpec.InitContainers, corev1.Container{})
+
+	// Building kine arguments, taking in consideration the user-space ones if provided.
+	args := map[string]string{}
+
+	if d.DataStore.Spec.TLSConfig != nil {
+		// Ensuring the init container required for kine is present:
+		// a chmod is required for kine in order to read the certificates to connect to the secured datastore.
+		found, index := utilities.HasNamedContainer(podSpec.InitContainers, kineInitContainerName)
+		if !found {
+			index = len(podSpec.InitContainers)
+			podSpec.InitContainers = append(podSpec.InitContainers, corev1.Container{})
+		}
+
+		podSpec.InitContainers[index].Name = kineInitContainerName
+		podSpec.InitContainers[index].Image = d.KineContainerImage
+		podSpec.InitContainers[index].Command = []string{"sh"}
+
+		podSpec.InitContainers[index].Args = []string{
+			"-c",
+			"cp /kine/*.* /certs && chmod -R 600 /certs/*.*",
+		}
+
+		podSpec.InitContainers[index].VolumeMounts = []corev1.VolumeMount{
+			{
+				Name:      dataStoreCertsVolumeName,
+				ReadOnly:  true,
+				MountPath: "/kine",
+			},
+			{
+				Name:      kineVolumeCertName,
+				MountPath: "/certs",
+				ReadOnly:  false,
+			},
+		}
+
+		args["--ca-file"] = "/certs/ca.crt"
+
+		if d.DataStore.Spec.TLSConfig.ClientCertificate != nil {
+			args["--cert-file"] = "/certs/server.crt"
+			args["--key-file"] = "/certs/server.key"
+		}
+	} else {
+		// if no TLS configuration is provided, the kine initContainer must be removed.
+		d.removeKineInitContainers(podSpec)
 	}
 
-	podSpec.InitContainers[index].Name = kineInitContainerName
-	podSpec.InitContainers[index].Image = d.KineContainerImage
-	podSpec.InitContainers[index].Command = []string{"sh"}
-	podSpec.InitContainers[index].Args = []string{
-		"-c",
-		"cp /kine/*.* /certs && chmod -R 600 /certs/*.*",
-	}
-	podSpec.InitContainers[index].VolumeMounts = []corev1.VolumeMount{
-		{
-			Name:      dataStoreCertsVolumeName,
-			ReadOnly:  true,
-			MountPath: "/kine",
-		},
-		{
-			Name:      kineVolumeCertName,
-			MountPath: "/certs",
-			ReadOnly:  false,
-		},
-	}
 	// Kine is expecting an additional container, and it must be removed before proceeding with the additional one
 	// in order to make this function idempotent.
-	found, index = utilities.HasNamedContainer(podSpec.Containers, kineContainerName)
+	found, index := utilities.HasNamedContainer(podSpec.Containers, kineContainerName)
 	if !found {
 		index = len(podSpec.Containers)
 		podSpec.Containers = append(podSpec.Containers, corev1.Container{})
 	}
-	// Building kine arguments, taking in consideration the user-space ones if provided.
-	args := map[string]string{}
 
 	if tcp.Spec.ControlPlane.Deployment.ExtraArgs != nil {
-		args = utilities.ArgsFromSliceToMap(tcp.Spec.ControlPlane.Deployment.ExtraArgs.Kine)
+		utilArgs := utilities.ArgsFromSliceToMap(tcp.Spec.ControlPlane.Deployment.ExtraArgs.Kine)
+
+		// Merging the user-space arguments with the Kamaji ones.
+		for k, v := range utilArgs {
+			args[k] = v
+		}
 	}
 
 	switch d.DataStore.Spec.Driver {
@@ -867,12 +894,10 @@ func (d Deployment) buildKine(podSpec *corev1.PodSpec, tcp kamajiv1alpha1.Tenant
 		args["--endpoint"] = "mysql://$(DB_USER):$(DB_PASSWORD)@tcp($(DB_CONNECTION_STRING))/$(DB_SCHEMA)"
 	case kamajiv1alpha1.KinePostgreSQLDriver:
 		args["--endpoint"] = "postgres://$(DB_USER):$(DB_PASSWORD)@$(DB_CONNECTION_STRING)/$(DB_SCHEMA)"
+	case kamajiv1alpha1.KineNatsDriver:
+		args["--endpoint"] = "nats://$(DB_USER):$(DB_PASSWORD)@$(DB_CONNECTION_STRING)?bucket=$(DB_SCHEMA)&noEmbed"
 	}
 
-	args["--ca-file"] = "/certs/ca.crt"
-	args["--cert-file"] = "/certs/server.crt"
-	args["--key-file"] = "/certs/server.key"
-
 	podSpec.Containers[index].Name = kineContainerName
 	podSpec.Containers[index].Image = d.KineContainerImage
 	podSpec.Containers[index].Command = []string{"/bin/kine"}
@@ -999,6 +1024,10 @@ func (d Deployment) setTemplateLabels(template *corev1.PodTemplateSpec, labels m
 	template.SetLabels(labels)
 }
 
+func (d Deployment) setTemplateAnnotations(template *corev1.PodTemplateSpec, annotations map[string]string) {
+	template.SetAnnotations(annotations)
+}
+
 func (d Deployment) setLabels(resource *appsv1.Deployment, labels map[string]string) {
 	resource.SetLabels(labels)
 }
@@ -1064,3 +1093,13 @@ func (d Deployment) setToleration(spec *corev1.PodSpec, tcp kamajiv1alpha1.Tenan
 func (d Deployment) setAffinity(spec *corev1.PodSpec, tcp kamajiv1alpha1.TenantControlPlane) {
 	spec.Affinity = tcp.Spec.ControlPlane.Deployment.Affinity
 }
+
+func (d Deployment) setServiceAccount(spec *corev1.PodSpec, tcp kamajiv1alpha1.TenantControlPlane) {
+	if len(tcp.Spec.ControlPlane.Deployment.ServiceAccountName) > 0 {
+		spec.ServiceAccountName = tcp.Spec.ControlPlane.Deployment.ServiceAccountName
+
+		return
+	}
+
+	spec.ServiceAccountName = "default"
+}

internal/datastore/connection.go

@@ -21,18 +21,26 @@ func NewStorageConnection(ctx context.Context, client client.Client, ds kamajiv1
 
 	switch ds.Spec.Driver {
 	case kamajiv1alpha1.KineMySQLDriver:
-		cc.TLSConfig.ServerName = cc.Endpoints[0].Host
+
+		if ds.Spec.TLSConfig != nil {
+			cc.TLSConfig.ServerName = cc.Endpoints[0].Host
+		}
+
 		cc.Parameters = map[string][]string{
 			"multiStatements": {"true"},
 		}
 
 		return NewMySQLConnection(*cc)
 	case kamajiv1alpha1.KinePostgreSQLDriver:
-		cc.TLSConfig.ServerName = cc.Endpoints[0].Host
+		if ds.Spec.TLSConfig != nil {
+			cc.TLSConfig.ServerName = cc.Endpoints[0].Host
+		}
 
 		return NewPostgreSQLConnection(*cc)
 	case kamajiv1alpha1.EtcdDriver:
 		return NewETCDConnection(*cc)
+	case kamajiv1alpha1.KineNatsDriver:
+		return NewNATSConnection(*cc)
 	default:
 		return nil, fmt.Errorf("%s is not a valid driver", ds.Spec.Driver)
 	}

internal/datastore/datastore.go

@@ -36,29 +36,41 @@ type ConnectionConfig struct {
 }
 
 func NewConnectionConfig(ctx context.Context, client client.Client, ds kamajiv1alpha1.DataStore) (*ConnectionConfig, error) {
-	ca, err := ds.Spec.TLSConfig.CertificateAuthority.Certificate.GetContent(ctx, client)
-	if err != nil {
-		return nil, err
+	var tlsConfig *tls.Config
+
+	if ds.Spec.TLSConfig != nil {
+		ca, err := ds.Spec.TLSConfig.CertificateAuthority.Certificate.GetContent(ctx, client)
+		if err != nil {
+			return nil, err
+		}
+
+		rootCAs := x509.NewCertPool()
+		if ok := rootCAs.AppendCertsFromPEM(ca); !ok {
+			return nil, fmt.Errorf("error create root CA for the DB connector")
+		}
+
+		tlsConfig = &tls.Config{
+			RootCAs: rootCAs,
+		}
 	}
 
-	crt, err := ds.Spec.TLSConfig.ClientCertificate.Certificate.GetContent(ctx, client)
-	if err != nil {
-		return nil, err
-	}
+	if ds.Spec.TLSConfig != nil && ds.Spec.TLSConfig.ClientCertificate != nil {
+		crt, err := ds.Spec.TLSConfig.ClientCertificate.Certificate.GetContent(ctx, client)
+		if err != nil {
+			return nil, err
+		}
 
-	key, err := ds.Spec.TLSConfig.ClientCertificate.PrivateKey.GetContent(ctx, client)
-	if err != nil {
-		return nil, err
-	}
+		key, err := ds.Spec.TLSConfig.ClientCertificate.PrivateKey.GetContent(ctx, client)
+		if err != nil {
+			return nil, err
+		}
 
-	rootCAs := x509.NewCertPool()
-	if ok := rootCAs.AppendCertsFromPEM(ca); !ok {
-		return nil, fmt.Errorf("error create root CA for the DB connector")
-	}
+		certificate, err := tls.X509KeyPair(crt, key)
+		if err != nil {
+			return nil, errors.Wrap(err, "cannot retrieve x.509 key pair from the Kine Secret")
+		}
 
-	certificate, err := tls.X509KeyPair(crt, key)
-	if err != nil {
-		return nil, errors.Wrap(err, "cannot retrieve x.509 key pair from the Kine Secret")
+		tlsConfig.Certificates = []tls.Certificate{certificate}
 	}
 
 	var user, password string
@@ -99,10 +111,7 @@ func NewConnectionConfig(ctx context.Context, client client.Client, ds kamajiv1a
 		User:      user,
 		Password:  password,
 		Endpoints: eps,
-		TLSConfig: &tls.Config{
-			RootCAs:      rootCAs,
-			Certificates: []tls.Certificate{certificate},
-		},
+		TLSConfig: tlsConfig,
 	}, nil
 }
 

internal/datastore/mysql.go

@@ -112,12 +112,14 @@ func NewMySQLConnection(config ConnectionConfig) (Connection, error) {
 
 	tlsKey := "mysql"
 
-	if err = mysql.RegisterTLSConfig(tlsKey, config.TLSConfig); err != nil {
-		return nil, err
+	if config.TLSConfig != nil {
+		if err = mysql.RegisterTLSConfig(tlsKey, config.TLSConfig); err != nil {
+			return nil, err
+		}
+		mysqlConfig.TLSConfig = tlsKey
 	}
 
 	mysqlConfig.DBName = config.DBName
-	mysqlConfig.TLSConfig = tlsKey
 	parsedDSN := mysqlConfig.FormatDSN()
 
 	db, err := sql.Open("mysql", parsedDSN)

internal/datastore/nats.go

@@ -0,0 +1,184 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package datastore
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/nats-io/nats.go"
+	"github.com/pkg/errors"
+
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+)
+
+// NATSConnection represents a connection to a NATS KV store.
+type NATSConnection struct {
+	js     nats.JetStreamContext
+	conn   *nats.Conn
+	config ConnectionConfig
+}
+
+// NewNATSConnection initializes a connection to NATS and sets up the KV store.
+func NewNATSConnection(config ConnectionConfig) (*NATSConnection, error) {
+	var endpoints string
+
+	if len(config.Endpoints) > 1 {
+		// comma separated list of endpoints
+		var ep []string
+		for _, e := range config.Endpoints {
+			ep = append(ep, fmt.Sprintf("nats://%s", e.String()))
+		}
+
+		endpoints = strings.Join(ep, ",")
+	} else {
+		endpoints = fmt.Sprintf("nats://%s", config.Endpoints[0].String())
+	}
+
+	var conn *nats.Conn
+	var err error
+	var natsOpts []nats.Option
+
+	if config.TLSConfig != nil {
+		natsOpts = append(natsOpts, nats.Secure(config.TLSConfig))
+	}
+
+	if config.User != "" && config.Password != "" {
+		natsOpts = append(natsOpts, nats.UserInfo(config.User, config.Password))
+	}
+
+	conn, err = nats.Connect(endpoints, natsOpts...)
+	if err != nil {
+		return nil, err
+	}
+
+	js, err := conn.JetStream()
+	if err != nil {
+		return nil, err
+	}
+
+	return &NATSConnection{
+		js:     js,
+		conn:   conn,
+		config: config,
+	}, nil
+}
+
+func (nc *NATSConnection) CreateUser(_ context.Context, _, _ string) error {
+	return nil
+}
+
+func (nc *NATSConnection) CreateDB(_ context.Context, dbName string) error {
+	_, err := nc.js.CreateKeyValue(&nats.KeyValueConfig{Bucket: dbName})
+	if err != nil {
+		return errors.Wrap(err, "unable to create the datastore")
+	}
+
+	return nil
+}
+
+func (nc *NATSConnection) GrantPrivileges(_ context.Context, _, _ string) error {
+	return nil
+}
+
+func (nc *NATSConnection) UserExists(_ context.Context, _ string) (bool, error) {
+	return true, nil
+}
+
+func (nc *NATSConnection) DBExists(_ context.Context, dbName string) (bool, error) {
+	_, err := nc.js.KeyValue(dbName)
+	if err != nil {
+		if errors.Is(err, nats.ErrBucketNotFound) {
+			return false, nil
+		}
+
+		return false, err
+	}
+
+	return true, nil
+}
+
+func (nc *NATSConnection) GrantPrivilegesExists(_ context.Context, _, _ string) (bool, error) {
+	return true, nil
+}
+
+func (nc *NATSConnection) DeleteUser(_ context.Context, _ string) error {
+	return nil
+}
+
+func (nc *NATSConnection) DeleteDB(_ context.Context, dbName string) error {
+	err := nc.js.DeleteKeyValue(dbName)
+
+	return err
+}
+
+func (nc *NATSConnection) RevokePrivileges(_ context.Context, _, _ string) error {
+	return nil
+}
+
+func (nc *NATSConnection) GetConnectionString() string {
+	return nc.config.Endpoints[0].String()
+}
+
+func (nc *NATSConnection) Close() error {
+	return nc.conn.Drain()
+}
+
+func (nc *NATSConnection) Check(_ context.Context) error {
+	status := nc.conn.Status()
+
+	if status != nats.CONNECTED {
+		return errors.New("connection to NATS is not established")
+	}
+
+	return nil
+}
+
+func (nc *NATSConnection) Driver() string {
+	return string(kamajiv1alpha1.KineNatsDriver)
+}
+
+func (nc *NATSConnection) GetConfig() ConnectionConfig {
+	return nc.config
+}
+
+func (nc *NATSConnection) Migrate(ctx context.Context, tcp kamajiv1alpha1.TenantControlPlane, target Connection) error {
+	targetClient := target.(*NATSConnection) //nolint:forcetypeassert
+	dbName := tcp.Status.Storage.Setup.Schema
+
+	targetKv, err := targetClient.js.KeyValue(dbName)
+	if err != nil {
+		return err
+	}
+
+	sourceKv, err := nc.js.KeyValue(dbName)
+	if err != nil {
+		return err
+	}
+
+	if err := target.Check(ctx); err != nil {
+		return err
+	}
+
+	// copy all keys from source to target
+	keys, err := sourceKv.Keys()
+	if err != nil {
+		return err
+	}
+
+	for _, key := range keys {
+		entry, err := sourceKv.Get(key)
+		if err != nil {
+			return err
+		}
+
+		_, err = targetKv.Put(key, entry.Value())
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

internal/datastore/postgresql.go

@@ -10,6 +10,7 @@ import (
 	"strings"
 
 	"github.com/go-pg/pg/v10"
+	goerrors "github.com/pkg/errors"
 
 	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
 	"github.com/clastix/kamaji/internal/datastore/errors"
@@ -34,6 +35,7 @@ const (
 type PostgreSQLConnection struct {
 	db               *pg.DB
 	connection       ConnectionEndpoint
+	rootUser         string
 	switchDatabaseFn func(dbName string) *pg.DB
 }
 
@@ -114,6 +116,7 @@ func NewPostgreSQLConnection(config ConnectionConfig) (Connection, error) {
 	return &PostgreSQLConnection{
 		db:               pg.Connect(opt),
 		switchDatabaseFn: fn,
+		rootUser:         config.User,
 		connection:       config.Endpoints[0],
 	}, nil
 }
@@ -232,6 +235,10 @@ func (r *PostgreSQLConnection) DeleteUser(ctx context.Context, user string) erro
 }
 
 func (r *PostgreSQLConnection) DeleteDB(ctx context.Context, dbName string) error {
+	if err := r.GrantPrivileges(ctx, r.rootUser, dbName); err != nil {
+		return errors.NewCannotDeleteDatabaseError(goerrors.Wrap(err, "cannot grant privileges to root user"))
+	}
+
 	if _, err := r.db.ExecContext(ctx, fmt.Sprintf(postgresqlDropDBStatement, dbName)); err != nil {
 		return errors.NewCannotDeleteDatabaseError(err)
 	}

internal/resources/ca_certificate.go

@@ -105,7 +105,7 @@ func (r *CACertificate) mutate(ctx context.Context, tenantControlPlane *kamajiv1
 			}
 
 			if isValid {
-				return nil
+				return ctrl.SetControllerReference(tenantControlPlane, r.resource, r.Client.Scheme())
 			}
 		}
 

internal/resources/datastore/datastore_certificate.go

@@ -80,79 +80,89 @@ func (r *Certificate) mutate(ctx context.Context, tenantControlPlane *kamajiv1al
 	return func() error {
 		logger := log.FromContext(ctx, "resource", r.GetName())
 
-		ca, err := r.DataStore.Spec.TLSConfig.CertificateAuthority.Certificate.GetContent(ctx, r.Client)
-		if err != nil {
-			logger.Error(err, "cannot retrieve CA certificate content")
+		if r.DataStore.Spec.TLSConfig != nil {
+			ca, err := r.DataStore.Spec.TLSConfig.CertificateAuthority.Certificate.GetContent(ctx, r.Client)
+			if err != nil {
+				logger.Error(err, "cannot retrieve CA certificate content")
 
-			return err
-		}
+				return err
+			}
 
-		r.resource.Data["ca.crt"] = ca
+			r.resource.Data["ca.crt"] = ca
 
-		r.resource.SetLabels(utilities.MergeMaps(
-			utilities.KamajiLabels(tenantControlPlane.GetName(), r.GetName()),
-			map[string]string{
-				constants.ControllerLabelResource: "x509",
-			},
-		))
+			r.resource.SetLabels(utilities.MergeMaps(
+				utilities.KamajiLabels(tenantControlPlane.GetName(), r.GetName()),
+				map[string]string{
+					constants.ControllerLabelResource: "x509",
+				},
+			))
 
-		if err = ctrl.SetControllerReference(tenantControlPlane, r.resource, r.Client.Scheme()); err != nil {
-			logger.Error(err, "cannot set controller reference", "resource", r.GetName())
+			if err = ctrl.SetControllerReference(tenantControlPlane, r.resource, r.Client.Scheme()); err != nil {
+				logger.Error(err, "cannot set controller reference", "resource", r.GetName())
 
-			return err
-		}
+				return err
+			}
 
-		if utilities.GetObjectChecksum(r.resource) == utilities.CalculateMapChecksum(r.resource.Data) {
-			if r.DataStore.Spec.Driver == kamajiv1alpha1.EtcdDriver {
-				if isValid, _ := crypto.IsValidCertificateKeyPairBytes(r.resource.Data["server.crt"], r.resource.Data["server.key"]); isValid {
-					return nil
+			if utilities.GetObjectChecksum(r.resource) == utilities.CalculateMapChecksum(r.resource.Data) {
+				if r.DataStore.Spec.Driver == kamajiv1alpha1.EtcdDriver {
+					if isValid, _ := crypto.IsValidCertificateKeyPairBytes(r.resource.Data["server.crt"], r.resource.Data["server.key"]); isValid {
+						return nil
+					}
 				}
 			}
+
+			var crt, key *bytes.Buffer
+
+			switch r.DataStore.Spec.Driver {
+			case kamajiv1alpha1.EtcdDriver:
+				var privateKey []byte
+				// When dealing with the etcd storage we cannot use the basic authentication, thus the generation of a
+				// certificate used for authentication is mandatory, along with the CA private key.
+				if privateKey, err = r.DataStore.Spec.TLSConfig.CertificateAuthority.PrivateKey.GetContent(ctx, r.Client); err != nil {
+					logger.Error(err, "unable to retrieve CA private key content")
+
+					return err
+				}
+
+				if crt, key, err = crypto.GenerateCertificatePrivateKeyPair(crypto.NewCertificateTemplate(tenantControlPlane.Status.Storage.Setup.User), ca, privateKey); err != nil {
+					logger.Error(err, "unable to generate certificate and private key")
+
+					return err
+				}
+			case kamajiv1alpha1.KineMySQLDriver, kamajiv1alpha1.KinePostgreSQLDriver, kamajiv1alpha1.KineNatsDriver:
+				var crtBytes, keyBytes []byte
+				// For the SQL drivers we just need to copy the certificate, since the basic authentication is used
+				// to connect to the desired schema and database.
+
+				if r.DataStore.Spec.TLSConfig.ClientCertificate != nil {
+					if crtBytes, err = r.DataStore.Spec.TLSConfig.ClientCertificate.Certificate.GetContent(ctx, r.Client); err != nil {
+						logger.Error(err, "unable to retrieve certificate content")
+
+						return err
+					}
+
+					crt = bytes.NewBuffer(crtBytes)
+
+					if keyBytes, err = r.DataStore.Spec.TLSConfig.ClientCertificate.PrivateKey.GetContent(ctx, r.Client); err != nil {
+						logger.Error(err, "unable to retrieve private key content")
+
+						return err
+					}
+					key = bytes.NewBuffer(keyBytes)
+				}
+			default:
+				return fmt.Errorf("unrecognized driver for Certificate generation")
+			}
+
+			if r.DataStore.Spec.TLSConfig.ClientCertificate != nil {
+				r.resource.Data["server.crt"] = crt.Bytes()
+				r.resource.Data["server.key"] = key.Bytes()
+			}
+		} else {
+			// set r.resource.Data to empty to allow switching from TLS to non-tls
+			r.resource.Data = map[string][]byte{}
 		}
 
-		var crt, key *bytes.Buffer
-
-		switch r.DataStore.Spec.Driver {
-		case kamajiv1alpha1.EtcdDriver:
-			var privateKey []byte
-			// When dealing with the etcd storage we cannot use the basic authentication, thus the generation of a
-			// certificate used for authentication is mandatory, along with the CA private key.
-			if privateKey, err = r.DataStore.Spec.TLSConfig.CertificateAuthority.PrivateKey.GetContent(ctx, r.Client); err != nil {
-				logger.Error(err, "unable to retrieve CA private key content")
-
-				return err
-			}
-
-			if crt, key, err = crypto.GenerateCertificatePrivateKeyPair(crypto.NewCertificateTemplate(tenantControlPlane.Status.Storage.Setup.User), ca, privateKey); err != nil {
-				logger.Error(err, "unable to generate certificate and private key")
-
-				return err
-			}
-		case kamajiv1alpha1.KineMySQLDriver, kamajiv1alpha1.KinePostgreSQLDriver:
-			var crtBytes, keyBytes []byte
-			// For the SQL drivers we just need to copy the certificate, since the basic authentication is used
-			// to connect to the desired schema and database.
-			if crtBytes, err = r.DataStore.Spec.TLSConfig.ClientCertificate.Certificate.GetContent(ctx, r.Client); err != nil {
-				logger.Error(err, "unable to retrieve certificate content")
-
-				return err
-			}
-
-			crt = bytes.NewBuffer(crtBytes)
-
-			if keyBytes, err = r.DataStore.Spec.TLSConfig.ClientCertificate.PrivateKey.GetContent(ctx, r.Client); err != nil {
-				logger.Error(err, "unable to retrieve private key content")
-
-				return err
-			}
-			key = bytes.NewBuffer(keyBytes)
-		default:
-			return fmt.Errorf("unrecognized driver for Certificate generation")
-		}
-
-		r.resource.Data["server.crt"] = crt.Bytes()
-		r.resource.Data["server.key"] = key.Bytes()
-
 		utilities.SetObjectChecksum(r.resource, r.resource.Data)
 
 		return nil

internal/resources/datastore/datastore_multitenancy.go

@@ -0,0 +1,62 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package datastore
+
+import (
+	"context"
+
+	"github.com/pkg/errors"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+)
+
+type MultiTenancy struct {
+	DataStore kamajiv1alpha1.DataStore
+}
+
+func (m MultiTenancy) Define(context.Context, *kamajiv1alpha1.TenantControlPlane) error {
+	return nil
+}
+
+func (m MultiTenancy) ShouldCleanup(*kamajiv1alpha1.TenantControlPlane) bool {
+	return false
+}
+
+func (m MultiTenancy) CleanUp(context.Context, *kamajiv1alpha1.TenantControlPlane) (bool, error) {
+	return false, nil
+}
+
+func (m MultiTenancy) CreateOrUpdate(_ context.Context, tcp *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) {
+	// If the NATS Datastore is already used by a Tenant Control Plane
+	// and a new one is reclaiming it, we need to stop it, since it's not allowed.
+	// TODO(prometherion): remove this after multi-tenancy is implemented for NATS.
+	if m.DataStore.Spec.Driver != kamajiv1alpha1.KineNatsDriver {
+		return controllerutil.OperationResultNone, nil
+	}
+
+	usedBy := sets.New[string](m.DataStore.Status.UsedBy...)
+
+	switch {
+	case usedBy.Has(tcp.Namespace + "/" + tcp.Name):
+		return controllerutil.OperationResultNone, nil
+	case usedBy.Len() == 0:
+		return controllerutil.OperationResultNone, nil
+	default:
+		return controllerutil.OperationResultNone, errors.New("NATS doesn't support multi-tenancy, the current datastore is already in use")
+	}
+}
+
+func (m MultiTenancy) GetName() string {
+	return "ds.multitenancy"
+}
+
+func (m MultiTenancy) ShouldStatusBeUpdated(context.Context, *kamajiv1alpha1.TenantControlPlane) bool {
+	return false
+}
+
+func (m MultiTenancy) UpdateTenantControlPlaneStatus(context.Context, *kamajiv1alpha1.TenantControlPlane) error {
+	return nil
+}

internal/resources/datastore/datastore_storage_config.go

@@ -104,9 +104,10 @@ func (r *Config) UpdateTenantControlPlaneStatus(_ context.Context, tenantControl
 	return nil
 }
 
-func (r *Config) mutate(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) controllerutil.MutateFn {
+func (r *Config) mutate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) controllerutil.MutateFn {
 	return func() error {
 		var password []byte
+		var username []byte
 
 		hash := utilities.GetObjectChecksum(r.resource)
 		switch {
@@ -133,10 +134,31 @@ func (r *Config) mutate(_ context.Context, tenantControlPlane *kamajiv1alpha1.Te
 		finalizersList.Insert(finalizers.DatastoreSecretFinalizer)
 		r.resource.SetFinalizers(finalizersList.UnsortedList())
 
+		// TODO(thecodeassassin): remove this after multi-tenancy is implemented for NATS.
+		// Due to NATS is missing a programmatic approach to create users and password,
+		// we're using the Datastore root password.
+		if r.DataStore.Spec.Driver == kamajiv1alpha1.KineNatsDriver {
+			// set username and password to the basicAuth values of the NATS datastore
+			u, err := r.DataStore.Spec.BasicAuth.Username.GetContent(ctx, r.Client)
+			if err != nil {
+				return errors.Wrap(err, "failed to retrieve the username for the NATS datastore")
+			}
+
+			p, err := r.DataStore.Spec.BasicAuth.Password.GetContent(ctx, r.Client)
+			if err != nil {
+				return errors.Wrap(err, "failed to retrieve the password for the NATS datastore")
+			}
+
+			username = u
+			password = p
+		} else {
+			username = coalesceFn(tenantControlPlane.Status.Storage.Setup.User)
+		}
+
 		r.resource.Data = map[string][]byte{
 			"DB_CONNECTION_STRING": []byte(r.ConnString),
 			"DB_SCHEMA":            coalesceFn(tenantControlPlane.Status.Storage.Setup.Schema),
-			"DB_USER":              coalesceFn(tenantControlPlane.Status.Storage.Setup.User),
+			"DB_USER":              username,
 			"DB_PASSWORD":          password,
 		}
 

internal/resources/front_proxy_ca_certificate.go

@@ -91,7 +91,7 @@ func (r *FrontProxyCACertificate) mutate(ctx context.Context, tenantControlPlane
 				logger.Info(fmt.Sprintf("%s certificate-private_key pair is not valid: %s", kubeadmconstants.FrontProxyCACertAndKeyBaseName, err.Error()))
 			}
 			if isValid {
-				return nil
+				return ctrl.SetControllerReference(tenantControlPlane, r.resource, r.Client.Scheme())
 			}
 		}
 

internal/resources/konnectivity/agent.go

@@ -126,12 +126,7 @@ func (r *Agent) mutate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.T
 		))
 
 		r.resource.Spec.Template.Spec.PriorityClassName = "system-cluster-critical"
-		r.resource.Spec.Template.Spec.Tolerations = []corev1.Toleration{
-			{
-				Key:      "CriticalAddonsOnly",
-				Operator: "Exists",
-			},
-		}
+		r.resource.Spec.Template.Spec.Tolerations = tenantControlPlane.Spec.Addons.Konnectivity.KonnectivityAgentSpec.Tolerations
 		r.resource.Spec.Template.Spec.NodeSelector = map[string]string{
 			"kubernetes.io/os": "linux",
 		}

internal/resources/kubeadm_upgrade.go

@@ -70,9 +70,9 @@ func (k *KubernetesUpgrade) CreateOrUpdate(ctx context.Context, tenantControlPla
 		return controllerutil.OperationResultNone, errors.Wrap(err, "cannot create REST client required for Kubernetes upgrade plan")
 	}
 
-	versionGetter := kamajiupgrade.NewKamajiKubeVersionGetter(clientSet)
+	versionGetter := kamajiupgrade.NewKamajiKubeVersionGetter(clientSet, tenantControlPlane.Status.Kubernetes.Version.Version)
 
-	if _, err = upgrade.GetAvailableUpgrades(versionGetter, false, false, true, clientSet, "", &printers.Discard{}); err != nil {
+	if _, err = upgrade.GetAvailableUpgrades(versionGetter, false, false, clientSet, &printers.Discard{}); err != nil {
 		return controllerutil.OperationResultNone, errors.Wrap(err, "cannot retrieve available Upgrades for Kubernetes upgrade plan")
 	}
 

internal/resources/sa_certificate.go

@@ -90,7 +90,7 @@ func (r *SACertificate) mutate(ctx context.Context, tenantControlPlane *kamajiv1
 				logger.Info(fmt.Sprintf("%s public_key-private_key pair is not valid: %s", kubeadmconstants.ServiceAccountKeyBaseName, err.Error()))
 			}
 			if isValid {
-				return nil
+				return ctrl.SetControllerReference(tenantControlPlane, r.resource, r.Client.Scheme())
 			}
 		}
 

internal/upgrade/kube_version_getter.go

@@ -16,12 +16,13 @@ import (
 
 type kamajiKubeVersionGetter struct {
 	upgrade.VersionGetter
+	Version string
 }
 
-func NewKamajiKubeVersionGetter(restClient kubernetes.Interface) upgrade.VersionGetter {
+func NewKamajiKubeVersionGetter(restClient kubernetes.Interface, version string) upgrade.VersionGetter {
 	kubeVersionGetter := upgrade.NewOfflineVersionGetter(upgrade.NewKubeVersionGetter(restClient), KubeadmVersion)
 
-	return &kamajiKubeVersionGetter{VersionGetter: kubeVersionGetter}
+	return &kamajiKubeVersionGetter{VersionGetter: kubeVersionGetter, Version: version}
 }
 
 func (k kamajiKubeVersionGetter) ClusterVersion() (string, *versionutil.Version, error) {
@@ -48,6 +49,12 @@ func (k kamajiKubeVersionGetter) VersionFromCILabel(ciVersionLabel, description
 	return k.VersionGetter.VersionFromCILabel(ciVersionLabel, description)
 }
 
-func (k kamajiKubeVersionGetter) KubeletVersions() (map[string]uint16, error) {
+func (k kamajiKubeVersionGetter) KubeletVersions() (map[string][]string, error) {
 	return k.VersionGetter.KubeletVersions()
 }
+
+func (k kamajiKubeVersionGetter) ComponentVersions(string) (map[string][]string, error) {
+	return map[string][]string{
+		k.Version: {"kamaji"},
+	}, nil
+}

internal/upgrade/kubeadm_version.go

@@ -4,5 +4,5 @@
 package upgrade
 
 const (
-	KubeadmVersion = "v1.29.1"
+	KubeadmVersion = "v1.30.1"
 )

internal/utilities/args.go

@@ -14,13 +14,7 @@ func ArgsFromSliceToMap(args []string) (m map[string]string) {
 	m = make(map[string]string)
 
 	for _, arg := range args {
-		parts := strings.SplitN(arg, "=", 2)
-
-		flag, value := parts[0], ""
-
-		if len(parts) > 1 {
-			value = parts[1]
-		}
+		flag, value, _ := strings.Cut(arg, "=")
 
 		m[flag] = value
 	}

internal/utilities/args_test.go

@@ -0,0 +1,30 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package utilities
+
+import (
+	"maps"
+	"testing"
+)
+
+func TestArgsFromSliceToMap(t *testing.T) {
+	tests := map[string]map[string]string{
+		"--a":     {"--a": ""},
+		"--a=":    {"--a": ""},
+		"--a=b":   {"--a": "b"},
+		"--a=b=c": {"--a": "b=c"},
+	}
+
+	got := ArgsFromSliceToMap([]string{})
+	if len(got) != 0 {
+		t.Errorf("expected empty input to result in empty map, but got %+v", got)
+	}
+
+	for arg, expect := range tests {
+		got := ArgsFromSliceToMap([]string{arg})
+		if !maps.Equal(expect, got) {
+			t.Errorf("expected input %q to result in %+v, but got %+v", arg, expect, got)
+		}
+	}
+}

internal/utilities/tenant_client.go

@@ -44,7 +44,13 @@ func GetTenantKubeconfig(ctx context.Context, client client.Client, tenantContro
 		return nil, err
 	}
 
-	return DecodeKubeconfig(*secretKubeconfig, kubeadmconstants.SuperAdminKubeConfigFileName)
+	secretKey := kubeadmconstants.SuperAdminKubeConfigFileName
+	v, ok := tenantControlPlane.GetAnnotations()[kamajiv1alpha1.KubeconfigSecretKeyAnnotation]
+	if ok && v != "" {
+		secretKey = v
+	}
+
+	return DecodeKubeconfig(*secretKubeconfig, secretKey)
 }
 
 func GetRESTClientConfig(ctx context.Context, client client.Client, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (*restclient.Config, error) {

internal/webhook/chainer.go

@@ -19,7 +19,7 @@ import (
 )
 
 type handlersChainer struct {
-	decoder *admission.Decoder
+	decoder admission.Decoder
 }
 
 //nolint:gocognit

internal/webhook/handlers/ds_validate.go

@@ -84,6 +84,10 @@ func (d DataStoreValidation) validateBasicAuth(ctx context.Context, ds kamajiv1a
 }
 
 func (d DataStoreValidation) validateTLSConfig(ctx context.Context, ds kamajiv1alpha1.DataStore) error {
+	if ds.Spec.TLSConfig == nil && ds.Spec.Driver != kamajiv1alpha1.EtcdDriver {
+		return nil
+	}
+
 	if err := d.validateContentReference(ctx, ds.Spec.TLSConfig.CertificateAuthority.Certificate); err != nil {
 		return fmt.Errorf("CA certificate is not valid, %w", err)
 	}

internal/webhook/routes/ds_secrets.go

@@ -13,7 +13,7 @@ import (
 type DataStoreSecrets struct{}
 
 func (d DataStoreSecrets) GetPath() string {
-	return "validate--v1-secret"
+	return "/validate--v1-secret"
 }
 
 func (d DataStoreSecrets) GetObject() runtime.Object {