chore(gh): trigger helm release on tag

.github/workflows/ci.yaml

@@ -18,7 +18,7 @@ jobs:
       - name: Run golangci-lint
         uses: golangci/golangci-lint-action@v2.3.0
         with:
-          version: v1.49.0
+          version: v1.45.2
           only-new-issues: false
           args: --timeout 5m --config .golangci.yml
   diff:

.golangci.yml

@@ -27,10 +27,6 @@ linters:
     - exhaustivestruct
     - wsl
     - exhaustive
-    - nosprintfhostport
-    - nonamedreturns
-    - interfacebloat
-    - exhaustruct
     - lll
     - gosec
     - gomoddirectives

Dockerfile

@@ -22,7 +22,6 @@ COPY main.go main.go
 COPY api/ api/
 COPY controllers/ controllers/
 COPY internal/ internal/
-COPY indexers/ indexers/
 
 # Build
 RUN CGO_ENABLED=0 GOOS=linux GOARCH=$TARGETARCH go build \

Makefile

@@ -37,6 +37,8 @@ BUNDLE_IMG ?= $(IMAGE_TAG_BASE)-bundle:v$(VERSION)
 
 # Image URL to use all building/pushing image targets
 IMG ?= clastix/kamaji:latest
+# Produce CRDs that work back to Kubernetes 1.11 (no version conversion)
+CRD_OPTIONS ?= "crd:trivialVersions=true,preserveUnknownFields=false"
 
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
 ifeq (,$(shell go env GOBIN))
@@ -85,7 +87,7 @@ kind: ## Download kind locally if necessary.
 
 CONTROLLER_GEN = $(shell pwd)/bin/controller-gen
 controller-gen: ## Download controller-gen locally if necessary.
-	$(call go-install-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.9.2)
+	$(call go-install-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.6.1)
 
 KUSTOMIZE = $(shell pwd)/bin/kustomize
 kustomize: ## Download kustomize locally if necessary.
@@ -94,9 +96,8 @@ kustomize: ## Download kustomize locally if necessary.
 ##@ Development
 
 manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects.
-	$(CONTROLLER_GEN) rbac:roleName=manager-role crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases
+	$(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=manager-role webhook paths="./..." output:crd:artifacts:config=config/crd/bases
 	cp config/crd/bases/kamaji.clastix.io_tenantcontrolplanes.yaml charts/kamaji/crds/tenantcontrolplane.yaml
-	cp config/crd/bases/kamaji.clastix.io_datastores.yaml charts/kamaji/crds/datastore.yaml
 
 generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations.
 	$(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..."

PROJECT

@@ -16,12 +16,4 @@ resources:
   kind: TenantControlPlane
   path: github.com/clastix/kamaji/api/v1alpha1
   version: v1alpha1
-- api:
-    crdVersion: v1
-    namespaced: false
-  domain: clastix.io
-  group: kamaji
-  kind: DataStore
-  path: github.com/clastix/kamaji/api/v1alpha1
-  version: v1alpha1
 version: "3"

api/v1alpha1/datastore_funcs.go

@@ -1,39 +0,0 @@
-// Copyright 2022 Clastix Labs
-// SPDX-License-Identifier: Apache-2.0
-
-package v1alpha1
-
-import (
-	"context"
-	"fmt"
-
-	corev1 "k8s.io/api/core/v1"
-	"k8s.io/apimachinery/pkg/types"
-	"sigs.k8s.io/controller-runtime/pkg/client"
-)
-
-// GetContent is the resolver for the container of the Secret.
-// The bare content has priority over the external reference.
-func (in *ContentRef) GetContent(ctx context.Context, client client.Client) ([]byte, error) {
-	if content := in.Content; len(content) > 0 {
-		return content, nil
-	}
-
-	secretRef := in.SecretRef
-
-	if secretRef == nil {
-		return nil, fmt.Errorf("no bare content and no external Secret reference")
-	}
-
-	secret, namespacedName := &corev1.Secret{}, types.NamespacedName{Name: secretRef.Name, Namespace: secretRef.Namespace}
-	if err := client.Get(ctx, namespacedName, secret); err != nil {
-		return nil, err
-	}
-
-	v, ok := secret.Data[secretRef.KeyPath]
-	if !ok {
-		return nil, fmt.Errorf("secret %s does not have key %s", namespacedName.String(), secretRef.KeyPath)
-	}
-
-	return v, nil
-}

api/v1alpha1/datastore_types.go

@@ -1,104 +0,0 @@
-// Copyright 2022 Clastix Labs
-// SPDX-License-Identifier: Apache-2.0
-
-package v1alpha1
-
-import (
-	corev1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-)
-
-type Driver string //+kubebuilder:validation:Enum=etcd;MySQL;PostgreSQL
-
-var (
-	EtcdDriver           Driver = "etcd"
-	KineMySQLDriver      Driver = "MySQL"
-	KinePostgreSQLDriver Driver = "PostgreSQL"
-)
-
-// DataStoreSpec defines the desired state of DataStore.
-type DataStoreSpec struct {
-	// The driver to use to connect to the shared datastore.
-	Driver Driver `json:"driver"`
-	// List of the endpoints to connect to the shared datastore.
-	// No need for protocol, just bare IP/FQDN and port.
-	Endpoints []string `json:"endpoints"` //+kubebuilder:validation:MinLength=1
-	// In case of authentication enabled for the given data store, specifies the username and password pair.
-	// This value is optional.
-	BasicAuth *BasicAuth `json:"basicAuth,omitempty"`
-	// Defines the TLS/SSL configuration required to connect to the data store in a secure way.
-	TLSConfig TLSConfig `json:"tlsConfig"`
-}
-
-// TLSConfig contains the information used to connect to the data store using a secured connection.
-type TLSConfig struct {
-	// Retrieve the Certificate Authority certificate and private key, such as bare content of the file, or a SecretReference.
-	// The key reference is required since etcd authentication is based on certificates, and Kamaji is responsible in creating this.
-	CertificateAuthority CertKeyPair `json:"certificateAuthority"`
-	// Specifies the SSL/TLS key and private key pair used to connect to the data store.
-	ClientCertificate ClientCertificate `json:"clientCertificate"`
-}
-
-type ClientCertificate struct {
-	Certificate ContentRef `json:"certificate"`
-	PrivateKey  ContentRef `json:"privateKey"`
-}
-
-type CertKeyPair struct {
-	Certificate ContentRef  `json:"certificate"`
-	PrivateKey  *ContentRef `json:"privateKey,omitempty"`
-}
-
-// BasicAuth contains the required information to perform the connection using user credentials to the data store.
-type BasicAuth struct {
-	Username ContentRef `json:"username"`
-	Password ContentRef `json:"password"`
-}
-
-type ContentRef struct {
-	// Bare content of the file, base64 encoded.
-	// It has precedence over the SecretReference value.
-	Content   []byte           `json:"content,omitempty"`
-	SecretRef *SecretReference `json:"secretReference,omitempty"`
-}
-
-type SecretReference struct {
-	corev1.SecretReference `json:",inline"`
-	// Name of the key for the given Secret reference where the content is stored.
-	// This value is mandatory.
-	KeyPath string `json:"keyPath"`
-}
-
-// DataStoreStatus defines the observed state of DataStore.
-type DataStoreStatus struct {
-	// List of the Tenant Control Planes, namespaced named, using this data store.
-	UsedBy []string `json:"usedBy,omitempty"`
-}
-
-//+kubebuilder:object:root=true
-//+kubebuilder:subresource:status
-//+kubebuilder:resource:scope=Cluster
-//+kubebuilder:printcolumn:name="Driver",type="string",JSONPath=".spec.driver",description="Kamaji data store driver"
-//+kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="Age"
-
-// DataStore is the Schema for the datastores API.
-type DataStore struct {
-	metav1.TypeMeta   `json:",inline"`
-	metav1.ObjectMeta `json:"metadata,omitempty"`
-
-	Spec   DataStoreSpec   `json:"spec,omitempty"`
-	Status DataStoreStatus `json:"status,omitempty"`
-}
-
-//+kubebuilder:object:root=true
-
-// DataStoreList contains a list of DataStore.
-type DataStoreList struct {
-	metav1.TypeMeta `json:",inline"`
-	metav1.ListMeta `json:"metadata,omitempty"`
-	Items           []DataStore `json:"items"`
-}
-
-func init() {
-	SchemeBuilder.Register(&DataStore{}, &DataStoreList{})
-}

api/v1alpha1/groupversion_info.go

@@ -2,9 +2,8 @@
 // SPDX-License-Identifier: Apache-2.0
 
 // Package v1alpha1 contains API Schema definitions for the kamaji v1alpha1 API group
-// +kubebuilder:object:generate=true
-// +groupName=kamaji.clastix.io
-//nolint
+//+kubebuilder:object:generate=true
+//+groupName=kamaji.clastix.io
 package v1alpha1
 
 import (

api/v1alpha1/tenantcontrolplane_status.go

@@ -8,6 +8,8 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	networkingv1 "k8s.io/api/networking/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/clastix/kamaji/internal/etcd"
 )
 
 // APIServerCertificatesStatus defines the observed state of ETCD Certificate for API server.
@@ -44,7 +46,7 @@ type PublicKeyPrivateKeyPairStatus struct {
 	Checksum   string      `json:"checksum,omitempty"`
 }
 
-// CertificatesStatus defines the observed state of ETCD TLSConfig.
+// CertificatesStatus defines the observed state of ETCD Certificates.
 type CertificatesStatus struct {
 	CA                     CertificatePrivateKeyPairStatus `json:"ca,omitempty"`
 	APIServer              CertificatePrivateKeyPairStatus `json:"apiServer,omitempty"`
@@ -55,31 +57,41 @@ type CertificatesStatus struct {
 	ETCD                   *ETCDCertificatesStatus         `json:"etcd,omitempty"`
 }
 
-type DataStoreCertificateStatus struct {
+// ETCDStatus defines the observed state of ETCDStatus.
+type ETCDStatus struct {
+	Role etcd.Role `json:"role,omitempty"`
+	User etcd.User `json:"user,omitempty"`
+}
+
+type SQLCertificateStatus struct {
 	SecretName string      `json:"secretName,omitempty"`
 	Checksum   string      `json:"checksum,omitempty"`
 	LastUpdate metav1.Time `json:"lastUpdate,omitempty"`
 }
 
-type DataStoreConfigStatus struct {
+type SQLConfigStatus struct {
 	SecretName string `json:"secretName,omitempty"`
 	Checksum   string `json:"checksum,omitempty"`
 }
 
-type DataStoreSetupStatus struct {
+type SQLSetupStatus struct {
 	Schema     string      `json:"schema,omitempty"`
 	User       string      `json:"user,omitempty"`
 	LastUpdate metav1.Time `json:"lastUpdate,omitempty"`
 	Checksum   string      `json:"checksum,omitempty"`
 }
 
+type KineStatus struct {
+	Driver      string               `json:"driver,omitempty"`
+	Config      SQLConfigStatus      `json:"config,omitempty"`
+	Setup       SQLSetupStatus       `json:"setup,omitempty"`
+	Certificate SQLCertificateStatus `json:"certificate,omitempty"`
+}
+
 // StorageStatus defines the observed state of StorageStatus.
 type StorageStatus struct {
-	Driver        string                     `json:"driver,omitempty"`
-	DataStoreName string                     `json:"dataStoreName,omitempty"`
-	Config        DataStoreConfigStatus      `json:"config,omitempty"`
-	Setup         DataStoreSetupStatus       `json:"setup,omitempty"`
-	Certificate   DataStoreCertificateStatus `json:"certificate,omitempty"`
+	ETCD *ETCDStatus `json:"etcd,omitempty"`
+	Kine *KineStatus `json:"kine,omitempty"`
 }
 
 // KubeconfigStatus contains information about the generated kubeconfig.
@@ -151,8 +163,9 @@ type AddonStatus struct {
 
 // AddonsStatus defines the observed state of the different Addons.
 type AddonsStatus struct {
-	CoreDNS      AddonStatus        `json:"coreDNS,omitempty"`
-	KubeProxy    AddonStatus        `json:"kubeProxy,omitempty"`
+	CoreDNS   AddonStatus `json:"coreDNS,omitempty"`
+	KubeProxy AddonStatus `json:"kubeProxy,omitempty"`
+
 	Konnectivity KonnectivityStatus `json:"konnectivity,omitempty"`
 }
 

api/v1alpha1/tenantcontrolplane_types.go

@@ -85,11 +85,6 @@ type ControlPlaneComponentsResources struct {
 type DeploymentSpec struct {
 	// +kubebuilder:default=2
 	Replicas int32 `json:"replicas,omitempty"`
-	// TopologySpreadConstraints describes how the Tenant Control Plane pods ought to spread across topology
-	// domains. Scheduler will schedule pods in a way which abides by the constraints.
-	// In case of nil underlying LabelSelector, the Kamaji one for the given Tenant Control Plane will be used.
-	// All topologySpreadConstraints are ANDed.
-	TopologySpreadConstraints []corev1.TopologySpreadConstraint `json:"topologySpreadConstraints,omitempty"`
 	// Resources defines the amount of memory and CPU to allocate to each component of the Control Plane
 	// (kube-apiserver, controller-manager, and scheduler).
 	Resources *ControlPlaneComponentsResources `json:"resources,omitempty"`
@@ -115,31 +110,20 @@ type ServiceSpec struct {
 }
 
 // AddonSpec defines the spec for every addon.
-type AddonSpec struct {
-	ImageOverrideTrait `json:",inline"`
-}
-
-type ImageOverrideTrait struct {
-	// ImageRepository sets the container registry to pull images from.
-	// if not set, the default ImageRepository will be used instead.
-	ImageRepository string `json:"imageRepository,omitempty"`
-	// ImageTag allows to specify a tag for the image.
-	// In case this value is set, kubeadm does not change automatically the version of the above components during upgrades.
-	ImageTag string `json:"imageTag,omitempty"`
-}
+type AddonSpec struct{}
 
 // KonnectivitySpec defines the spec for Konnectivity.
 type KonnectivitySpec struct {
 	// Port of Konnectivity proxy server.
 	ProxyPort int32 `json:"proxyPort"`
 	// Version for Konnectivity server and agent.
-	// +kubebuilder:default=v0.0.32
+	// +kubebuilder:default=v0.0.31
 	Version string `json:"version,omitempty"`
 	// ServerImage defines the container image for Konnectivity's server.
-	// +kubebuilder:default=registry.k8s.io/kas-network-proxy/proxy-server
+	// +kubebuilder:default=us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-server
 	ServerImage string `json:"serverImage,omitempty"`
 	// AgentImage defines the container image for Konnectivity's agent.
-	// +kubebuilder:default=registry.k8s.io/kas-network-proxy/proxy-agent
+	// +kubebuilder:default=us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-agent
 	AgentImage string `json:"agentImage,omitempty"`
 	// Resources define the amount of CPU and memory to allocate to the Konnectivity server.
 	Resources *corev1.ResourceRequirements `json:"resources,omitempty"`
@@ -147,22 +131,13 @@ type KonnectivitySpec struct {
 
 // AddonsSpec defines the enabled addons and their features.
 type AddonsSpec struct {
-	// Enables the DNS addon in the Tenant Cluster.
-	// The registry and the tag are configurable, the image is hard-coded to `coredns`.
-	CoreDNS *AddonSpec `json:"coreDNS,omitempty"`
-	// Enables the Konnectivity addon in the Tenant Cluster, required if the worker nodes are in a different network.
+	CoreDNS      *AddonSpec        `json:"coreDNS,omitempty"`
 	Konnectivity *KonnectivitySpec `json:"konnectivity,omitempty"`
-	// Enables the kube-proxy addon in the Tenant Cluster.
-	// The registry and the tag are configurable, the image is hard-coded to `kube-proxy`.
-	KubeProxy *AddonSpec `json:"kubeProxy,omitempty"`
+	KubeProxy    *AddonSpec        `json:"kubeProxy,omitempty"`
 }
 
 // TenantControlPlaneSpec defines the desired state of TenantControlPlane.
 type TenantControlPlaneSpec struct {
-	// DataStore allows to specify a DataStore that should be used to store the Kubernetes data for the given Tenant Control Plane.
-	// This parameter is optional and acts as an override over the default one which is used by the Kamaji Operator.
-	// Migration from a different DataStore to another one is not yet supported and the reconciliation will be blocked.
-	DataStore    string       `json:"dataStore,omitempty"`
 	ControlPlane ControlPlane `json:"controlPlane"`
 	// Kubernetes specification for tenant control plane
 	Kubernetes KubernetesSpec `json:"kubernetes"`

api/v1alpha1/zz_generated.deepcopy.go

@@ -61,7 +61,6 @@ func (in *AdditionalMetadata) DeepCopy() *AdditionalMetadata {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *AddonSpec) DeepCopyInto(out *AddonSpec) {
 	*out = *in
-	out.ImageOverrideTrait = in.ImageOverrideTrait
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AddonSpec.
@@ -157,44 +156,6 @@ func (in AdmissionControllers) DeepCopy() AdmissionControllers {
 	return *out
 }
 
-// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *BasicAuth) DeepCopyInto(out *BasicAuth) {
-	*out = *in
-	in.Username.DeepCopyInto(&out.Username)
-	in.Password.DeepCopyInto(&out.Password)
-}
-
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BasicAuth.
-func (in *BasicAuth) DeepCopy() *BasicAuth {
-	if in == nil {
-		return nil
-	}
-	out := new(BasicAuth)
-	in.DeepCopyInto(out)
-	return out
-}
-
-// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *CertKeyPair) DeepCopyInto(out *CertKeyPair) {
-	*out = *in
-	in.Certificate.DeepCopyInto(&out.Certificate)
-	if in.PrivateKey != nil {
-		in, out := &in.PrivateKey, &out.PrivateKey
-		*out = new(ContentRef)
-		(*in).DeepCopyInto(*out)
-	}
-}
-
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CertKeyPair.
-func (in *CertKeyPair) DeepCopy() *CertKeyPair {
-	if in == nil {
-		return nil
-	}
-	out := new(CertKeyPair)
-	in.DeepCopyInto(out)
-	return out
-}
-
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *CertificatePrivateKeyPairStatus) DeepCopyInto(out *CertificatePrivateKeyPairStatus) {
 	*out = *in
@@ -237,48 +198,6 @@ func (in *CertificatesStatus) DeepCopy() *CertificatesStatus {
 	return out
 }
 
-// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *ClientCertificate) DeepCopyInto(out *ClientCertificate) {
-	*out = *in
-	in.Certificate.DeepCopyInto(&out.Certificate)
-	in.PrivateKey.DeepCopyInto(&out.PrivateKey)
-}
-
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClientCertificate.
-func (in *ClientCertificate) DeepCopy() *ClientCertificate {
-	if in == nil {
-		return nil
-	}
-	out := new(ClientCertificate)
-	in.DeepCopyInto(out)
-	return out
-}
-
-// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *ContentRef) DeepCopyInto(out *ContentRef) {
-	*out = *in
-	if in.Content != nil {
-		in, out := &in.Content, &out.Content
-		*out = make([]byte, len(*in))
-		copy(*out, *in)
-	}
-	if in.SecretRef != nil {
-		in, out := &in.SecretRef, &out.SecretRef
-		*out = new(SecretReference)
-		**out = **in
-	}
-}
-
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ContentRef.
-func (in *ContentRef) DeepCopy() *ContentRef {
-	if in == nil {
-		return nil
-	}
-	out := new(ContentRef)
-	in.DeepCopyInto(out)
-	return out
-}
-
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ControlPlane) DeepCopyInto(out *ControlPlane) {
 	*out = *in
@@ -366,168 +285,9 @@ func (in *ControlPlaneExtraArgs) DeepCopy() *ControlPlaneExtraArgs {
 	return out
 }
 
-// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *DataStore) DeepCopyInto(out *DataStore) {
-	*out = *in
-	out.TypeMeta = in.TypeMeta
-	in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
-	in.Spec.DeepCopyInto(&out.Spec)
-	in.Status.DeepCopyInto(&out.Status)
-}
-
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataStore.
-func (in *DataStore) DeepCopy() *DataStore {
-	if in == nil {
-		return nil
-	}
-	out := new(DataStore)
-	in.DeepCopyInto(out)
-	return out
-}
-
-// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
-func (in *DataStore) DeepCopyObject() runtime.Object {
-	if c := in.DeepCopy(); c != nil {
-		return c
-	}
-	return nil
-}
-
-// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *DataStoreCertificateStatus) DeepCopyInto(out *DataStoreCertificateStatus) {
-	*out = *in
-	in.LastUpdate.DeepCopyInto(&out.LastUpdate)
-}
-
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataStoreCertificateStatus.
-func (in *DataStoreCertificateStatus) DeepCopy() *DataStoreCertificateStatus {
-	if in == nil {
-		return nil
-	}
-	out := new(DataStoreCertificateStatus)
-	in.DeepCopyInto(out)
-	return out
-}
-
-// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *DataStoreConfigStatus) DeepCopyInto(out *DataStoreConfigStatus) {
-	*out = *in
-}
-
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataStoreConfigStatus.
-func (in *DataStoreConfigStatus) DeepCopy() *DataStoreConfigStatus {
-	if in == nil {
-		return nil
-	}
-	out := new(DataStoreConfigStatus)
-	in.DeepCopyInto(out)
-	return out
-}
-
-// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *DataStoreList) DeepCopyInto(out *DataStoreList) {
-	*out = *in
-	out.TypeMeta = in.TypeMeta
-	in.ListMeta.DeepCopyInto(&out.ListMeta)
-	if in.Items != nil {
-		in, out := &in.Items, &out.Items
-		*out = make([]DataStore, len(*in))
-		for i := range *in {
-			(*in)[i].DeepCopyInto(&(*out)[i])
-		}
-	}
-}
-
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataStoreList.
-func (in *DataStoreList) DeepCopy() *DataStoreList {
-	if in == nil {
-		return nil
-	}
-	out := new(DataStoreList)
-	in.DeepCopyInto(out)
-	return out
-}
-
-// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
-func (in *DataStoreList) DeepCopyObject() runtime.Object {
-	if c := in.DeepCopy(); c != nil {
-		return c
-	}
-	return nil
-}
-
-// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *DataStoreSetupStatus) DeepCopyInto(out *DataStoreSetupStatus) {
-	*out = *in
-	in.LastUpdate.DeepCopyInto(&out.LastUpdate)
-}
-
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataStoreSetupStatus.
-func (in *DataStoreSetupStatus) DeepCopy() *DataStoreSetupStatus {
-	if in == nil {
-		return nil
-	}
-	out := new(DataStoreSetupStatus)
-	in.DeepCopyInto(out)
-	return out
-}
-
-// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *DataStoreSpec) DeepCopyInto(out *DataStoreSpec) {
-	*out = *in
-	if in.Endpoints != nil {
-		in, out := &in.Endpoints, &out.Endpoints
-		*out = make([]string, len(*in))
-		copy(*out, *in)
-	}
-	if in.BasicAuth != nil {
-		in, out := &in.BasicAuth, &out.BasicAuth
-		*out = new(BasicAuth)
-		(*in).DeepCopyInto(*out)
-	}
-	in.TLSConfig.DeepCopyInto(&out.TLSConfig)
-}
-
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataStoreSpec.
-func (in *DataStoreSpec) DeepCopy() *DataStoreSpec {
-	if in == nil {
-		return nil
-	}
-	out := new(DataStoreSpec)
-	in.DeepCopyInto(out)
-	return out
-}
-
-// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *DataStoreStatus) DeepCopyInto(out *DataStoreStatus) {
-	*out = *in
-	if in.UsedBy != nil {
-		in, out := &in.UsedBy, &out.UsedBy
-		*out = make([]string, len(*in))
-		copy(*out, *in)
-	}
-}
-
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataStoreStatus.
-func (in *DataStoreStatus) DeepCopy() *DataStoreStatus {
-	if in == nil {
-		return nil
-	}
-	out := new(DataStoreStatus)
-	in.DeepCopyInto(out)
-	return out
-}
-
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *DeploymentSpec) DeepCopyInto(out *DeploymentSpec) {
 	*out = *in
-	if in.TopologySpreadConstraints != nil {
-		in, out := &in.TopologySpreadConstraints, &out.TopologySpreadConstraints
-		*out = make([]v1.TopologySpreadConstraint, len(*in))
-		for i := range *in {
-			(*in)[i].DeepCopyInto(&(*out)[i])
-		}
-	}
 	if in.Resources != nil {
 		in, out := &in.Resources, &out.Resources
 		*out = new(ControlPlaneComponentsResources)
@@ -584,6 +344,23 @@ func (in *ETCDCertificatesStatus) DeepCopy() *ETCDCertificatesStatus {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ETCDStatus) DeepCopyInto(out *ETCDStatus) {
+	*out = *in
+	in.Role.DeepCopyInto(&out.Role)
+	in.User.DeepCopyInto(&out.User)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ETCDStatus.
+func (in *ETCDStatus) DeepCopy() *ETCDStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(ETCDStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ExternalKubernetesObjectStatus) DeepCopyInto(out *ExternalKubernetesObjectStatus) {
 	*out = *in
@@ -600,21 +377,6 @@ func (in *ExternalKubernetesObjectStatus) DeepCopy() *ExternalKubernetesObjectSt
 	return out
 }
 
-// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *ImageOverrideTrait) DeepCopyInto(out *ImageOverrideTrait) {
-	*out = *in
-}
-
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ImageOverrideTrait.
-func (in *ImageOverrideTrait) DeepCopy() *ImageOverrideTrait {
-	if in == nil {
-		return nil
-	}
-	out := new(ImageOverrideTrait)
-	in.DeepCopyInto(out)
-	return out
-}
-
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *IngressSpec) DeepCopyInto(out *IngressSpec) {
 	*out = *in
@@ -631,6 +393,24 @@ func (in *IngressSpec) DeepCopy() *IngressSpec {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *KineStatus) DeepCopyInto(out *KineStatus) {
+	*out = *in
+	out.Config = in.Config
+	in.Setup.DeepCopyInto(&out.Setup)
+	in.Certificate.DeepCopyInto(&out.Certificate)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KineStatus.
+func (in *KineStatus) DeepCopy() *KineStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(KineStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *KonnectivityConfigMap) DeepCopyInto(out *KonnectivityConfigMap) {
 	*out = *in
@@ -942,17 +722,48 @@ func (in *PublicKeyPrivateKeyPairStatus) DeepCopy() *PublicKeyPrivateKeyPairStat
 }
 
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *SecretReference) DeepCopyInto(out *SecretReference) {
+func (in *SQLCertificateStatus) DeepCopyInto(out *SQLCertificateStatus) {
 	*out = *in
-	out.SecretReference = in.SecretReference
+	in.LastUpdate.DeepCopyInto(&out.LastUpdate)
 }
 
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecretReference.
-func (in *SecretReference) DeepCopy() *SecretReference {
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SQLCertificateStatus.
+func (in *SQLCertificateStatus) DeepCopy() *SQLCertificateStatus {
 	if in == nil {
 		return nil
 	}
-	out := new(SecretReference)
+	out := new(SQLCertificateStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *SQLConfigStatus) DeepCopyInto(out *SQLConfigStatus) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SQLConfigStatus.
+func (in *SQLConfigStatus) DeepCopy() *SQLConfigStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(SQLConfigStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *SQLSetupStatus) DeepCopyInto(out *SQLSetupStatus) {
+	*out = *in
+	in.LastUpdate.DeepCopyInto(&out.LastUpdate)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SQLSetupStatus.
+func (in *SQLSetupStatus) DeepCopy() *SQLSetupStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(SQLSetupStatus)
 	in.DeepCopyInto(out)
 	return out
 }
@@ -976,9 +787,16 @@ func (in *ServiceSpec) DeepCopy() *ServiceSpec {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *StorageStatus) DeepCopyInto(out *StorageStatus) {
 	*out = *in
-	out.Config = in.Config
-	in.Setup.DeepCopyInto(&out.Setup)
-	in.Certificate.DeepCopyInto(&out.Certificate)
+	if in.ETCD != nil {
+		in, out := &in.ETCD, &out.ETCD
+		*out = new(ETCDStatus)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.Kine != nil {
+		in, out := &in.Kine, &out.Kine
+		*out = new(KineStatus)
+		(*in).DeepCopyInto(*out)
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StorageStatus.
@@ -991,23 +809,6 @@ func (in *StorageStatus) DeepCopy() *StorageStatus {
 	return out
 }
 
-// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *TLSConfig) DeepCopyInto(out *TLSConfig) {
-	*out = *in
-	in.CertificateAuthority.DeepCopyInto(&out.CertificateAuthority)
-	in.ClientCertificate.DeepCopyInto(&out.ClientCertificate)
-}
-
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TLSConfig.
-func (in *TLSConfig) DeepCopy() *TLSConfig {
-	if in == nil {
-		return nil
-	}
-	out := new(TLSConfig)
-	in.DeepCopyInto(out)
-	return out
-}
-
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *TenantControlPlane) DeepCopyInto(out *TenantControlPlane) {
 	*out = *in

charts/kamaji/Chart.yaml

@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.7.0
+version: 0.1.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

charts/kamaji/README.md

@@ -1,6 +1,6 @@
 # kamaji
 
-![Version: 0.1.0](https://img.shields.io/badge/Version-0.1.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.1.0](https://img.shields.io/badge/AppVersion-0.1.0-informational?style=flat-square)
+![Version: 0.1.1](https://img.shields.io/badge/Version-0.1.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.1.0](https://img.shields.io/badge/AppVersion-0.1.0-informational?style=flat-square)
 
 Kamaji is a tool aimed to build and operate a Managed Kubernetes Service with a fraction of the operational burden. With Kamaji, you can deploy and operate hundreds of Kubernetes clusters as a hyper-scaler.
 
@@ -68,27 +68,6 @@ Here the values you can override:
 |-----|------|---------|-------------|
 | affinity | object | `{}` | Kubernetes affinity rules to apply to Kamaji controller pods |
 | configPath | string | `"./kamaji.yaml"` | Configuration file path alternative. (default "./kamaji.yaml") |
-| datastore.basicAuth.passwordSecret.keyPath | string | `nil` | The Secret key where the data is stored. |
-| datastore.basicAuth.passwordSecret.name | string | `nil` | The name of the Secret containing the password used to connect to the relational database. |
-| datastore.basicAuth.passwordSecret.namespace | string | `nil` | The namespace of the Secret containing the password used to connect to the relational database. |
-| datastore.basicAuth.usernameSecret.keyPath | string | `nil` | The Secret key where the data is stored. |
-| datastore.basicAuth.usernameSecret.name | string | `nil` | The name of the Secret containing the username used to connect to the relational database. |
-| datastore.basicAuth.usernameSecret.namespace | string | `nil` | The namespace of the Secret containing the username used to connect to the relational database. |
-| datastore.driver | string | `"etcd"` | (string) The Kamaji Datastore driver, supported: etcd, MySQL, PostgreSQL (defaults=etcd). |
-| datastore.endpoints | list | `[]` | (array) List of endpoints of the selected Datastore. When letting the Chart install the etcd datastore, this field is populated automatically. |
-| datastore.nameOverride | string | `nil` | The Datastore name override, if empty defaults to `default` |
-| datastore.tlsConfig.certificateAuthority.certificate.keyPath | string | `nil` | Key of the Secret which contains the content of the certificate. |
-| datastore.tlsConfig.certificateAuthority.certificate.name | string | `nil` | Name of the Secret containing the CA required to establish the mandatory SSL/TLS connection to the datastore. |
-| datastore.tlsConfig.certificateAuthority.certificate.namespace | string | `nil` | Namespace of the Secret containing the CA required to establish the mandatory SSL/TLS connection to the datastore. |
-| datastore.tlsConfig.certificateAuthority.privateKey.keyPath | string | `nil` | Key of the Secret which contains the content of the private key. |
-| datastore.tlsConfig.certificateAuthority.privateKey.name | string | `nil` | Name of the Secret containing the CA private key required to establish the mandatory SSL/TLS connection to the datastore. |
-| datastore.tlsConfig.certificateAuthority.privateKey.namespace | string | `nil` | Namespace of the Secret containing the CA private key required to establish the mandatory SSL/TLS connection to the datastore. |
-| datastore.tlsConfig.clientCertificate.certificate.keyPath | string | `nil` | Key of the Secret which contains the content of the certificate. |
-| datastore.tlsConfig.clientCertificate.certificate.name | string | `nil` | Name of the Secret containing the client certificate required to establish the mandatory SSL/TLS connection to the datastore. |
-| datastore.tlsConfig.clientCertificate.certificate.namespace | string | `nil` | Namespace of the Secret containing the client certificate required to establish the mandatory SSL/TLS connection to the datastore. |
-| datastore.tlsConfig.clientCertificate.privateKey.keyPath | string | `nil` | Key of the Secret which contains the content of the private key. |
-| datastore.tlsConfig.clientCertificate.privateKey.name | string | `nil` | Name of the Secret containing the client certificate private key required to establish the mandatory SSL/TLS connection to the datastore. |
-| datastore.tlsConfig.clientCertificate.privateKey.namespace | string | `nil` | Namespace of the Secret containing the client certificate private key required to establish the mandatory SSL/TLS connection to the datastore. |
 | etcd.compactionInterval | int | `0` | ETCD Compaction interval (e.g. "5m0s"). (default: "0" (disabled)) |
 | etcd.deploy | bool | `true` | Install an etcd with enabled multi-tenancy along with Kamaji |
 | etcd.image | object | `{"pullPolicy":"IfNotPresent","repository":"quay.io/coreos/etcd","tag":"v3.5.4"}` | Install specific etcd image |
@@ -133,9 +112,3 @@ Here the values you can override:
 | serviceAccount.name | string | `"kamaji-controller-manager"` |  |
 | temporaryDirectoryPath | string | `"/tmp/kamaji"` | Directory which will be used to work with temporary files. (default "/tmp/kamaji") |
 | tolerations | list | `[]` | Kubernetes node taints that the Kamaji controller pods would tolerate |
-
-## Installing and managing etcd as DataStore
-
-Kamaji supports multiple data store, although `etcd` is the default one: thus, an initial cluster will be created upon the Chart installation.
-
-The `DataStore` resource can be configured with the proper values in case of overrides when using a different driver, otherwise all the required data will be inherited by the Chart values.

charts/kamaji/README.md.gotmpl

@@ -54,9 +54,3 @@ If you only need to make minor customizations, you can specify them on the comma
 Here the values you can override:
 
 {{ template "chart.valuesSection" . }}
-
-## Installing and managing etcd as DataStore
-
-Kamaji supports multiple data store, although `etcd` is the default one: thus, an initial cluster will be created upon the Chart installation.
-
-The `DataStore` resource can be configured with the proper values in case of overrides when using a different driver, otherwise all the required data will be inherited by the Chart values.

charts/kamaji/crds/datastore.yaml

@@ -1,268 +0,0 @@
----
-apiVersion: apiextensions.k8s.io/v1
-kind: CustomResourceDefinition
-metadata:
-  annotations:
-    controller-gen.kubebuilder.io/version: v0.9.2
-  creationTimestamp: null
-  name: datastores.kamaji.clastix.io
-spec:
-  group: kamaji.clastix.io
-  names:
-    kind: DataStore
-    listKind: DataStoreList
-    plural: datastores
-    singular: datastore
-  scope: Cluster
-  versions:
-  - additionalPrinterColumns:
-    - description: Kamaji data store driver
-      jsonPath: .spec.driver
-      name: Driver
-      type: string
-    - description: Age
-      jsonPath: .metadata.creationTimestamp
-      name: Age
-      type: date
-    name: v1alpha1
-    schema:
-      openAPIV3Schema:
-        description: DataStore is the Schema for the datastores API.
-        properties:
-          apiVersion:
-            description: 'APIVersion defines the versioned schema of this representation
-              of an object. Servers should convert recognized schemas to the latest
-              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
-            type: string
-          kind:
-            description: 'Kind is a string value representing the REST resource this
-              object represents. Servers may infer this from the endpoint the client
-              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
-            type: string
-          metadata:
-            type: object
-          spec:
-            description: DataStoreSpec defines the desired state of DataStore.
-            properties:
-              basicAuth:
-                description: In case of authentication enabled for the given data
-                  store, specifies the username and password pair. This value is optional.
-                properties:
-                  password:
-                    properties:
-                      content:
-                        description: Bare content of the file, base64 encoded. It
-                          has precedence over the SecretReference value.
-                        format: byte
-                        type: string
-                      secretReference:
-                        properties:
-                          keyPath:
-                            description: Name of the key for the given Secret reference
-                              where the content is stored. This value is mandatory.
-                            type: string
-                          name:
-                            description: name is unique within a namespace to reference
-                              a secret resource.
-                            type: string
-                          namespace:
-                            description: namespace defines the space within which
-                              the secret name must be unique.
-                            type: string
-                        required:
-                        - keyPath
-                        type: object
-                        x-kubernetes-map-type: atomic
-                    type: object
-                  username:
-                    properties:
-                      content:
-                        description: Bare content of the file, base64 encoded. It
-                          has precedence over the SecretReference value.
-                        format: byte
-                        type: string
-                      secretReference:
-                        properties:
-                          keyPath:
-                            description: Name of the key for the given Secret reference
-                              where the content is stored. This value is mandatory.
-                            type: string
-                          name:
-                            description: name is unique within a namespace to reference
-                              a secret resource.
-                            type: string
-                          namespace:
-                            description: namespace defines the space within which
-                              the secret name must be unique.
-                            type: string
-                        required:
-                        - keyPath
-                        type: object
-                        x-kubernetes-map-type: atomic
-                    type: object
-                required:
-                - password
-                - username
-                type: object
-              driver:
-                description: The driver to use to connect to the shared datastore.
-                type: string
-              endpoints:
-                description: List of the endpoints to connect to the shared datastore.
-                  No need for protocol, just bare IP/FQDN and port.
-                items:
-                  type: string
-                type: array
-              tlsConfig:
-                description: Defines the TLS/SSL configuration required to connect
-                  to the data store in a secure way.
-                properties:
-                  certificateAuthority:
-                    description: Retrieve the Certificate Authority certificate and
-                      private key, such as bare content of the file, or a SecretReference.
-                      The key reference is required since etcd authentication is based
-                      on certificates, and Kamaji is responsible in creating this.
-                    properties:
-                      certificate:
-                        properties:
-                          content:
-                            description: Bare content of the file, base64 encoded.
-                              It has precedence over the SecretReference value.
-                            format: byte
-                            type: string
-                          secretReference:
-                            properties:
-                              keyPath:
-                                description: Name of the key for the given Secret
-                                  reference where the content is stored. This value
-                                  is mandatory.
-                                type: string
-                              name:
-                                description: name is unique within a namespace to
-                                  reference a secret resource.
-                                type: string
-                              namespace:
-                                description: namespace defines the space within which
-                                  the secret name must be unique.
-                                type: string
-                            required:
-                            - keyPath
-                            type: object
-                            x-kubernetes-map-type: atomic
-                        type: object
-                      privateKey:
-                        properties:
-                          content:
-                            description: Bare content of the file, base64 encoded.
-                              It has precedence over the SecretReference value.
-                            format: byte
-                            type: string
-                          secretReference:
-                            properties:
-                              keyPath:
-                                description: Name of the key for the given Secret
-                                  reference where the content is stored. This value
-                                  is mandatory.
-                                type: string
-                              name:
-                                description: name is unique within a namespace to
-                                  reference a secret resource.
-                                type: string
-                              namespace:
-                                description: namespace defines the space within which
-                                  the secret name must be unique.
-                                type: string
-                            required:
-                            - keyPath
-                            type: object
-                            x-kubernetes-map-type: atomic
-                        type: object
-                    required:
-                    - certificate
-                    type: object
-                  clientCertificate:
-                    description: Specifies the SSL/TLS key and private key pair used
-                      to connect to the data store.
-                    properties:
-                      certificate:
-                        properties:
-                          content:
-                            description: Bare content of the file, base64 encoded.
-                              It has precedence over the SecretReference value.
-                            format: byte
-                            type: string
-                          secretReference:
-                            properties:
-                              keyPath:
-                                description: Name of the key for the given Secret
-                                  reference where the content is stored. This value
-                                  is mandatory.
-                                type: string
-                              name:
-                                description: name is unique within a namespace to
-                                  reference a secret resource.
-                                type: string
-                              namespace:
-                                description: namespace defines the space within which
-                                  the secret name must be unique.
-                                type: string
-                            required:
-                            - keyPath
-                            type: object
-                            x-kubernetes-map-type: atomic
-                        type: object
-                      privateKey:
-                        properties:
-                          content:
-                            description: Bare content of the file, base64 encoded.
-                              It has precedence over the SecretReference value.
-                            format: byte
-                            type: string
-                          secretReference:
-                            properties:
-                              keyPath:
-                                description: Name of the key for the given Secret
-                                  reference where the content is stored. This value
-                                  is mandatory.
-                                type: string
-                              name:
-                                description: name is unique within a namespace to
-                                  reference a secret resource.
-                                type: string
-                              namespace:
-                                description: namespace defines the space within which
-                                  the secret name must be unique.
-                                type: string
-                            required:
-                            - keyPath
-                            type: object
-                            x-kubernetes-map-type: atomic
-                        type: object
-                    required:
-                    - certificate
-                    - privateKey
-                    type: object
-                required:
-                - certificateAuthority
-                - clientCertificate
-                type: object
-            required:
-            - driver
-            - endpoints
-            - tlsConfig
-            type: object
-          status:
-            description: DataStoreStatus defines the observed state of DataStore.
-            properties:
-              usedBy:
-                description: List of the Tenant Control Planes, namespaced named,
-                  using this data store.
-                items:
-                  type: string
-                type: array
-            type: object
-        type: object
-    served: true
-    storage: true
-    subresources:
-      status: {}

charts/kamaji/crds/tenantcontrolplane.yaml

@@ -1,9 +1,10 @@
+
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.9.2
+    controller-gen.kubebuilder.io/version: v0.6.1
   creationTimestamp: null
   name: tenantcontrolplanes.kamaji.clastix.io
 spec:
@@ -63,27 +64,13 @@ spec:
                 description: Addons contain which addons are enabled
                 properties:
                   coreDNS:
-                    description: Enables the DNS addon in the Tenant Cluster. The
-                      registry and the tag are configurable, the image is hard-coded
-                      to `coredns`.
-                    properties:
-                      imageRepository:
-                        description: ImageRepository sets the container registry to
-                          pull images from. if not set, the default ImageRepository
-                          will be used instead.
-                        type: string
-                      imageTag:
-                        description: ImageTag allows to specify a tag for the image.
-                          In case this value is set, kubeadm does not change automatically
-                          the version of the above components during upgrades.
-                        type: string
+                    description: AddonSpec defines the spec for every addon.
                     type: object
                   konnectivity:
-                    description: Enables the Konnectivity addon in the Tenant Cluster,
-                      required if the worker nodes are in a different network.
+                    description: KonnectivitySpec defines the spec for Konnectivity.
                     properties:
                       agentImage:
-                        default: registry.k8s.io/kas-network-proxy/proxy-agent
+                        default: us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-agent
                         description: AgentImage defines the container image for Konnectivity's
                           agent.
                         type: string
@@ -120,32 +107,19 @@ spec:
                             type: object
                         type: object
                       serverImage:
-                        default: registry.k8s.io/kas-network-proxy/proxy-server
+                        default: us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-server
                         description: ServerImage defines the container image for Konnectivity's
                           server.
                         type: string
                       version:
-                        default: v0.0.32
+                        default: v0.0.31
                         description: Version for Konnectivity server and agent.
                         type: string
                     required:
                     - proxyPort
                     type: object
                   kubeProxy:
-                    description: Enables the kube-proxy addon in the Tenant Cluster.
-                      The registry and the tag are configurable, the image is hard-coded
-                      to `kube-proxy`.
-                    properties:
-                      imageRepository:
-                        description: ImageRepository sets the container registry to
-                          pull images from. if not set, the default ImageRepository
-                          will be used instead.
-                        type: string
-                      imageTag:
-                        description: ImageTag allows to specify a tag for the image.
-                          In case this value is set, kubeadm does not change automatically
-                          the version of the above components during upgrades.
-                        type: string
+                    description: AddonSpec defines the spec for every addon.
                     type: object
                 type: object
               controlPlane:
@@ -289,194 +263,6 @@ spec:
                                 type: object
                             type: object
                         type: object
-                      topologySpreadConstraints:
-                        description: TopologySpreadConstraints describes how the Tenant
-                          Control Plane pods ought to spread across topology domains.
-                          Scheduler will schedule pods in a way which abides by the
-                          constraints. In case of nil underlying LabelSelector, the
-                          Kamaji one for the given Tenant Control Plane will be used.
-                          All topologySpreadConstraints are ANDed.
-                        items:
-                          description: TopologySpreadConstraint specifies how to spread
-                            matching pods among the given topology.
-                          properties:
-                            labelSelector:
-                              description: LabelSelector is used to find matching
-                                pods. Pods that match this label selector are counted
-                                to determine the number of pods in their corresponding
-                                topology domain.
-                              properties:
-                                matchExpressions:
-                                  description: matchExpressions is a list of label
-                                    selector requirements. The requirements are ANDed.
-                                  items:
-                                    description: A label selector requirement is a
-                                      selector that contains values, a key, and an
-                                      operator that relates the key and values.
-                                    properties:
-                                      key:
-                                        description: key is the label key that the
-                                          selector applies to.
-                                        type: string
-                                      operator:
-                                        description: operator represents a key's relationship
-                                          to a set of values. Valid operators are
-                                          In, NotIn, Exists and DoesNotExist.
-                                        type: string
-                                      values:
-                                        description: values is an array of string
-                                          values. If the operator is In or NotIn,
-                                          the values array must be non-empty. If the
-                                          operator is Exists or DoesNotExist, the
-                                          values array must be empty. This array is
-                                          replaced during a strategic merge patch.
-                                        items:
-                                          type: string
-                                        type: array
-                                    required:
-                                    - key
-                                    - operator
-                                    type: object
-                                  type: array
-                                matchLabels:
-                                  additionalProperties:
-                                    type: string
-                                  description: matchLabels is a map of {key,value}
-                                    pairs. A single {key,value} in the matchLabels
-                                    map is equivalent to an element of matchExpressions,
-                                    whose key field is "key", the operator is "In",
-                                    and the values array contains only "value". The
-                                    requirements are ANDed.
-                                  type: object
-                              type: object
-                              x-kubernetes-map-type: atomic
-                            matchLabelKeys:
-                              description: MatchLabelKeys is a set of pod label keys
-                                to select the pods over which spreading will be calculated.
-                                The keys are used to lookup values from the incoming
-                                pod labels, those key-value labels are ANDed with
-                                labelSelector to select the group of existing pods
-                                over which spreading will be calculated for the incoming
-                                pod. Keys that don't exist in the incoming pod labels
-                                will be ignored. A null or empty list means only match
-                                against labelSelector.
-                              items:
-                                type: string
-                              type: array
-                              x-kubernetes-list-type: atomic
-                            maxSkew:
-                              description: 'MaxSkew describes the degree to which
-                                pods may be unevenly distributed. When `whenUnsatisfiable=DoNotSchedule`,
-                                it is the maximum permitted difference between the
-                                number of matching pods in the target topology and
-                                the global minimum. The global minimum is the minimum
-                                number of matching pods in an eligible domain or zero
-                                if the number of eligible domains is less than MinDomains.
-                                For example, in a 3-zone cluster, MaxSkew is set to
-                                1, and pods with the same labelSelector spread as
-                                2/2/1: In this case, the global minimum is 1. | zone1
-                                | zone2 | zone3 | |  P P  |  P P  |   P   | - if MaxSkew
-                                is 1, incoming pod can only be scheduled to zone3
-                                to become 2/2/2; scheduling it onto zone1(zone2) would
-                                make the ActualSkew(3-1) on zone1(zone2) violate MaxSkew(1).
-                                - if MaxSkew is 2, incoming pod can be scheduled onto
-                                any zone. When `whenUnsatisfiable=ScheduleAnyway`,
-                                it is used to give higher precedence to topologies
-                                that satisfy it. It''s a required field. Default value
-                                is 1 and 0 is not allowed.'
-                              format: int32
-                              type: integer
-                            minDomains:
-                              description: "MinDomains indicates a minimum number
-                                of eligible domains. When the number of eligible domains
-                                with matching topology keys is less than minDomains,
-                                Pod Topology Spread treats \"global minimum\" as 0,
-                                and then the calculation of Skew is performed. And
-                                when the number of eligible domains with matching
-                                topology keys equals or greater than minDomains, this
-                                value has no effect on scheduling. As a result, when
-                                the number of eligible domains is less than minDomains,
-                                scheduler won't schedule more than maxSkew Pods to
-                                those domains. If value is nil, the constraint behaves
-                                as if MinDomains is equal to 1. Valid values are integers
-                                greater than 0. When value is not nil, WhenUnsatisfiable
-                                must be DoNotSchedule. \n For example, in a 3-zone
-                                cluster, MaxSkew is set to 2, MinDomains is set to
-                                5 and pods with the same labelSelector spread as 2/2/2:
-                                | zone1 | zone2 | zone3 | |  P P  |  P P  |  P P  |
-                                The number of domains is less than 5(MinDomains),
-                                so \"global minimum\" is treated as 0. In this situation,
-                                new pod with the same labelSelector cannot be scheduled,
-                                because computed skew will be 3(3 - 0) if new Pod
-                                is scheduled to any of the three zones, it will violate
-                                MaxSkew. \n This is a beta field and requires the
-                                MinDomainsInPodTopologySpread feature gate to be enabled
-                                (enabled by default)."
-                              format: int32
-                              type: integer
-                            nodeAffinityPolicy:
-                              description: "NodeAffinityPolicy indicates how we will
-                                treat Pod's nodeAffinity/nodeSelector when calculating
-                                pod topology spread skew. Options are: - Honor: only
-                                nodes matching nodeAffinity/nodeSelector are included
-                                in the calculations. - Ignore: nodeAffinity/nodeSelector
-                                are ignored. All nodes are included in the calculations.
-                                \n If this value is nil, the behavior is equivalent
-                                to the Honor policy. This is a alpha-level feature
-                                enabled by the NodeInclusionPolicyInPodTopologySpread
-                                feature flag."
-                              type: string
-                            nodeTaintsPolicy:
-                              description: "NodeTaintsPolicy indicates how we will
-                                treat node taints when calculating pod topology spread
-                                skew. Options are: - Honor: nodes without taints,
-                                along with tainted nodes for which the incoming pod
-                                has a toleration, are included. - Ignore: node taints
-                                are ignored. All nodes are included. \n If this value
-                                is nil, the behavior is equivalent to the Ignore policy.
-                                This is a alpha-level feature enabled by the NodeInclusionPolicyInPodTopologySpread
-                                feature flag."
-                              type: string
-                            topologyKey:
-                              description: TopologyKey is the key of node labels.
-                                Nodes that have a label with this key and identical
-                                values are considered to be in the same topology.
-                                We consider each <key, value> as a "bucket", and try
-                                to put balanced number of pods into each bucket. We
-                                define a domain as a particular instance of a topology.
-                                Also, we define an eligible domain as a domain whose
-                                nodes meet the requirements of nodeAffinityPolicy
-                                and nodeTaintsPolicy. e.g. If TopologyKey is "kubernetes.io/hostname",
-                                each Node is a domain of that topology. And, if TopologyKey
-                                is "topology.kubernetes.io/zone", each zone is a domain
-                                of that topology. It's a required field.
-                              type: string
-                            whenUnsatisfiable:
-                              description: 'WhenUnsatisfiable indicates how to deal
-                                with a pod if it doesn''t satisfy the spread constraint.
-                                - DoNotSchedule (default) tells the scheduler not
-                                to schedule it. - ScheduleAnyway tells the scheduler
-                                to schedule the pod in any location, but giving higher
-                                precedence to topologies that would help reduce the
-                                skew. A constraint is considered "Unsatisfiable" for
-                                an incoming pod if and only if every possible node
-                                assignment for that pod would violate "MaxSkew" on
-                                some topology. For example, in a 3-zone cluster, MaxSkew
-                                is set to 1, and pods with the same labelSelector
-                                spread as 3/1/1: | zone1 | zone2 | zone3 | | P P P
-                                |   P   |   P   | If WhenUnsatisfiable is set to DoNotSchedule,
-                                incoming pod can only be scheduled to zone2(zone3)
-                                to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3)
-                                satisfies MaxSkew(1). In other words, the cluster
-                                can still be imbalanced, but scheduler won''t make
-                                it *more* imbalanced. It''s a required field.'
-                              type: string
-                          required:
-                          - maxSkew
-                          - topologyKey
-                          - whenUnsatisfiable
-                          type: object
-                        type: array
                     type: object
                   ingress:
                     description: Defining the options for an Optional Ingress which
@@ -537,14 +323,6 @@ spec:
                 required:
                 - service
                 type: object
-              dataStore:
-                description: DataStore allows to specify a DataStore that should be
-                  used to store the Kubernetes data for the given Tenant Control Plane.
-                  This parameter is optional and acts as an override over the default
-                  one which is used by the Kamaji Operator. Migration from a different
-                  DataStore to another one is not yet supported and the reconciliation
-                  will be blocked.
-                type: string
               kubernetes:
                 description: Kubernetes specification for tenant control plane
                 properties:
@@ -776,15 +554,15 @@ spec:
                               description: "Condition contains details for one aspect
                                 of the current state of this API Resource. --- This
                                 struct is intended for direct use as an array at the
-                                field path .status.conditions.  For example, \n type
-                                FooStatus struct{ // Represents the observations of
-                                a foo's current state. // Known .status.conditions.type
+                                field path .status.conditions.  For example, type
+                                FooStatus struct{     // Represents the observations
+                                of a foo's current state.     // Known .status.conditions.type
                                 are: \"Available\", \"Progressing\", and \"Degraded\"
-                                // +patchMergeKey=type // +patchStrategy=merge //
-                                +listType=map // +listMapKey=type Conditions []metav1.Condition
-                                `json:\"conditions,omitempty\" patchStrategy:\"merge\"
-                                patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"`
-                                \n // other fields }"
+                                \    // +patchMergeKey=type     // +patchStrategy=merge
+                                \    // +listType=map     // +listMapKey=type     Conditions
+                                []metav1.Condition `json:\"conditions,omitempty\"
+                                patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"`
+                                \n     // other fields }"
                               properties:
                                 lastTransitionTime:
                                   description: lastTransitionTime is the last time
@@ -888,10 +666,11 @@ spec:
                                               the error shall comply with the following
                                               rules: - built-in error values shall
                                               be specified in this file and those
-                                              shall use CamelCase names - cloud provider
-                                              specific error values must have names
-                                              that comply with the format foo.example.com/CamelCase.
-                                              --- The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)'
+                                              shall use   CamelCase names - cloud
+                                              provider specific error values must
+                                              have names that comply with the   format
+                                              foo.example.com/CamelCase. --- The regex
+                                              it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)'
                                             maxLength: 316
                                             pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
                                             type: string
@@ -1281,9 +1060,9 @@ spec:
                                           with the service port The format of the
                                           error shall comply with the following rules:
                                           - built-in error values shall be specified
-                                          in this file and those shall use CamelCase
+                                          in this file and those shall use   CamelCase
                                           names - cloud provider specific error values
-                                          must have names that comply with the format
+                                          must have names that comply with the   format
                                           foo.example.com/CamelCase. --- The regex
                                           it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)'
                                         maxLength: 316
@@ -1332,14 +1111,14 @@ spec:
                           description: "Condition contains details for one aspect
                             of the current state of this API Resource. --- This struct
                             is intended for direct use as an array at the field path
-                            .status.conditions.  For example, \n type FooStatus struct{
-                            // Represents the observations of a foo's current state.
-                            // Known .status.conditions.type are: \"Available\", \"Progressing\",
-                            and \"Degraded\" // +patchMergeKey=type // +patchStrategy=merge
-                            // +listType=map // +listMapKey=type Conditions []metav1.Condition
-                            `json:\"conditions,omitempty\" patchStrategy:\"merge\"
-                            patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"`
-                            \n // other fields }"
+                            .status.conditions.  For example, type FooStatus struct{
+                            \    // Represents the observations of a foo's current
+                            state.     // Known .status.conditions.type are: \"Available\",
+                            \"Progressing\", and \"Degraded\"     // +patchMergeKey=type
+                            \    // +patchStrategy=merge     // +listType=map     //
+                            +listMapKey=type     Conditions []metav1.Condition `json:\"conditions,omitempty\"
+                            patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"`
+                            \n     // other fields }"
                           properties:
                             lastTransitionTime:
                               description: lastTransitionTime is the last time the
@@ -1438,9 +1217,9 @@ spec:
                                           with the service port The format of the
                                           error shall comply with the following rules:
                                           - built-in error values shall be specified
-                                          in this file and those shall use CamelCase
+                                          in this file and those shall use   CamelCase
                                           names - cloud provider specific error values
-                                          must have names that comply with the format
+                                          must have names that comply with the   format
                                           foo.example.com/CamelCase. --- The regex
                                           it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)'
                                         maxLength: 316
@@ -1508,38 +1287,78 @@ spec:
                 description: Storage Status contains information about Kubernetes
                   storage system
                 properties:
-                  certificate:
+                  etcd:
+                    description: ETCDStatus defines the observed state of ETCDStatus.
                     properties:
-                      checksum:
-                        type: string
-                      lastUpdate:
-                        format: date-time
-                        type: string
-                      secretName:
-                        type: string
-                    type: object
-                  config:
-                    properties:
-                      checksum:
-                        type: string
-                      secretName:
-                        type: string
-                    type: object
-                  dataStoreName:
-                    type: string
-                  driver:
-                    type: string
-                  setup:
-                    properties:
-                      checksum:
-                        type: string
-                      lastUpdate:
-                        format: date-time
-                        type: string
-                      schema:
-                        type: string
+                      role:
+                        properties:
+                          exists:
+                            type: boolean
+                          name:
+                            type: string
+                          permissions:
+                            items:
+                              properties:
+                                key:
+                                  type: string
+                                rangeEnd:
+                                  type: string
+                                type:
+                                  type: integer
+                              type: object
+                            type: array
+                        required:
+                        - exists
+                        - name
+                        type: object
                       user:
+                        properties:
+                          exists:
+                            type: boolean
+                          name:
+                            type: string
+                          roles:
+                            items:
+                              type: string
+                            type: array
+                        required:
+                        - exists
+                        - name
+                        type: object
+                    type: object
+                  kine:
+                    properties:
+                      certificate:
+                        properties:
+                          checksum:
+                            type: string
+                          lastUpdate:
+                            format: date-time
+                            type: string
+                          secretName:
+                            type: string
+                        type: object
+                      config:
+                        properties:
+                          checksum:
+                            type: string
+                          secretName:
+                            type: string
+                        type: object
+                      driver:
                         type: string
+                      setup:
+                        properties:
+                          checksum:
+                            type: string
+                          lastUpdate:
+                            format: date-time
+                            type: string
+                          schema:
+                            type: string
+                          user:
+                            type: string
+                        type: object
                     type: object
                 type: object
             type: object
@@ -1548,3 +1367,9 @@ spec:
     storage: true
     subresources:
       status: {}
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []

charts/kamaji/templates/_helpers_datastore.tpl

@@ -1,90 +0,0 @@
-{{/*
-Create a default fully qualified datastore name.
-*/}}
-{{- define "datastore.fullname" -}}
-{{- default "default" .Values.datastore.nameOverride | trunc 63 | trimSuffix "-" }}
-{{- end }}
-
-{{/*
-Common labels
-*/}}
-{{- define "datastore.labels" -}}
-kamaji.clastix.io/datastore: {{ .Values.datastore.driver }}
-helm.sh/chart: {{ include "kamaji.chart" . }}
-{{ include "kamaji.selectorLabels" . }}
-{{- if .Chart.AppVersion }}
-app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
-{{- end }}
-app.kubernetes.io/managed-by: {{ .Release.Service }}
-{{- end }}
-
-{{/*
-Datastore endpoints, in case of ETCD, retrieving the one provided by the chart.
-*/}}
-{{- define "datastore.endpoints" -}}
-{{- if eq .Values.datastore.driver "etcd" }}
-{{ include "etcd.endpoints" . }}
-{{- else }}
-{{ .Values.datastore.endpoints }}
-{{- end }}
-{{- end }}
-
-{{/*
-The Certificate Authority section for the DataSource object.
-*/}}
-{{- define "datastore.certificateAuthority" -}}
-{{- if eq .Values.datastore.driver "etcd" }}
-certificate:
-  secretReference:
-    name: {{ include "etcd.caSecretName" . }}
-    namespace: {{ include "etcd.caSecretNamespace" . }}
-    keyPath: ca.crt
-privateKey:
-  secretReference:
-    name: {{ include "etcd.caSecretName" . }}
-    namespace: {{ include "etcd.caSecretNamespace" . }}
-    keyPath: ca.key
-{{- else }}
-certificate:
-  secretReference:
-    name: {{ .Values.datastore.tlsConfig.certificateAuthority.certificate.name }}
-    namespace: {{ .Values.datastore.tlsConfig.certificateAuthority.certificate.namespace }}
-    keyPath: {{ .Values.datastore.tlsConfig.certificateAuthority.certificate.keyPath }}
-{{- if .Values.datastore.tlsConfig.certificateAuthority.privateKey.name }}
-privateKey:
-  secretReference:
-    name: {{ .Values.datastore.tlsConfig.certificateAuthority.privateKey.name }}
-    namespace: {{ .Values.datastore.tlsConfig.certificateAuthority.privateKey.namespace }}
-    keyPath: {{ .Values.datastore.tlsConfig.certificateAuthority.privateKey.keyPath }}
-{{- end }}
-{{- end }}
-{{- end }}
-
-{{/*
-The Client Certificate section for the DataSource object.
-*/}}
-{{- define "datastore.clientCertificate" -}}
-{{- if eq .Values.datastore.driver "etcd" }}
-certificate:
-    secretReference:
-      name: {{ include "etcd.clientSecretName" . }}
-      namespace: {{ include "etcd.clientSecretNamespace" . }}
-      keyPath: tls.crt
-privateKey:
-    secretReference:
-      name: {{ include "etcd.clientSecretName" . }}
-      namespace: {{ include "etcd.clientSecretNamespace" . }}
-      keyPath: tls.key
-{{- else }}
-certificate:
-  secretReference:
-    name: {{ .Values.datastore.tlsConfig.clientCertificate.certificate.name }}
-    namespace: {{ .Values.datastore.tlsConfig.clientCertificate.certificate.namespace }}
-    keyPath: {{ .Values.datastore.tlsConfig.clientCertificate.certificate.keyPath }}
-privateKey:
-  secretReference:
-    name: {{ .Values.datastore.tlsConfig.clientCertificate.privateKey.name }}
-    namespace: {{ .Values.datastore.tlsConfig.clientCertificate.privateKey.namespace }}
-    keyPath: {{ .Values.datastore.tlsConfig.clientCertificate.privateKey.keyPath }}
-{{- end }}
-{{- end }}

charts/kamaji/templates/_helpers_etcd.tpl

@@ -99,7 +99,7 @@ Comma separated list of etcd endpoints, using the overrides in case of unmanaged
 {{- $list := list -}}
 {{- if .Values.etcd.deploy }}
     {{- range $count := until 3 -}}
-        {{- $list = append $list (printf "%s-%d.%s.%s.svc.cluster.local:%d" "etcd" $count ( include "etcd.serviceName" . ) $.Release.Namespace (int $.Values.etcd.port) ) -}}
+        {{- $list = append $list (printf "https://%s-%d.%s.%s.svc.cluster.local:%d" "etcd" $count ( include "etcd.serviceName" . ) $.Release.Namespace (int $.Values.etcd.port) ) -}}
     {{- end }}
 {{- else if .Values.etcd.overrides.endpoints }}
     {{- range $v := .Values.etcd.overrides.endpoints -}}
@@ -108,7 +108,7 @@ Comma separated list of etcd endpoints, using the overrides in case of unmanaged
 {{- else if not .Values.etcd.overrides.endpoints }}
     {{- fail "A valid .Values.etcd.overrides.endpoints required!" }}
 {{- end }}
-{{- $list | toYaml }}
+{{- join "," $list -}}
 {{- end }}
 
 {{/*

charts/kamaji/templates/controller.yaml

@@ -40,11 +40,16 @@ spec:
           protocol: TCP
       - args:
         - --config-file={{ .Values.configPath }}
+        - --etcd-ca-secret-name={{ include "etcd.caSecretName" . }}
+        - --etcd-ca-secret-namespace={{ include "etcd.caSecretNamespace" . }}
+        - --etcd-client-secret-name={{ include "etcd.clientSecretName" . }}
+        - --etcd-client-secret-namespace={{ include "etcd.clientSecretNamespace" . }}
+        - --etcd-compaction-interval={{ .Values.etcd.compactionInterval }}
+        - --etcd-endpoints={{ include "etcd.endpoints" . }}
         - --health-probe-bind-address={{ .Values.healthProbeBindAddress }}
         - --leader-elect
         - --metrics-bind-address={{ .Values.metricsBindAddress }}
         - --tmp-directory={{ .Values.temporaryDirectoryPath }}
-        - --datastore={{ include "datastore.fullname" . }}
         {{- if .Values.loggingDevel.enable }}
         - --zap-devel
         {{- end }}

charts/kamaji/templates/datastore.yaml

@@ -1,19 +0,0 @@
-apiVersion: kamaji.clastix.io/v1alpha1
-kind: DataStore
-metadata:
-  name: {{ include "datastore.fullname" . }}
-  labels:
-    {{- include "datastore.labels" . | nindent 4 }}
-spec:
-  driver: {{ .Values.datastore.driver }}
-  endpoints:
-    {{- include "datastore.endpoints" . | indent 4 }}
-{{- if (and .Values.datastore.basicAuth.usernameSecret.name .Values.datastore.basicAuth.passwordSecret.name) }}
-  basicAuth:
-    {{- .Values.datastore.basicAuth | toYaml | nindent 4 }}
-{{- end }}
-  tlsConfig:
-    certificateAuthority:
-      {{- include "datastore.certificateAuthority" . | indent 6 }}
-    clientCertificate:
-      {{- include "datastore.clientCertificate" . | indent 6 }}

charts/kamaji/templates/rbac.yaml

@@ -102,32 +102,6 @@ rules:
   - patch
   - update
   - watch
-- apiGroups:
-    - kamaji.clastix.io
-  resources:
-    - datastores
-  verbs:
-    - create
-    - delete
-    - get
-    - list
-    - patch
-    - update
-    - watch
-- apiGroups:
-    - kamaji.clastix.io
-  resources:
-    - datastores/finalizers
-  verbs:
-    - update
-- apiGroups:
-    - kamaji.clastix.io
-  resources:
-    - datastores/status
-  verbs:
-    - get
-    - patch
-    - update
 - apiGroups:
   - kamaji.clastix.io
   resources:

charts/kamaji/values.yaml

@@ -154,57 +154,3 @@ temporaryDirectoryPath: "/tmp/kamaji"
 loggingDevel:
   # -- (string) Development Mode defaults(encoder=consoleEncoder,logLevel=Debug,stackTraceLevel=Warn). Production Mode defaults(encoder=jsonEncoder,logLevel=Info,stackTraceLevel=Error) (default false)
   enable: false
-
-datastore:
-  # -- (string) The Datastore name override, if empty defaults to `default`
-  nameOverride:
-  # -- (string) The Kamaji Datastore driver, supported: etcd, MySQL, PostgreSQL (defaults=etcd).
-  driver: etcd
-  # -- (array) List of endpoints of the selected Datastore. When letting the Chart install the etcd datastore, this field is populated automatically.
-  endpoints: []
-  basicAuth:
-    usernameSecret:
-      # -- The name of the Secret containing the username used to connect to the relational database.
-      name:
-      # -- The namespace of the Secret containing the username used to connect to the relational database.
-      namespace:
-      # -- The Secret key where the data is stored.
-      keyPath:
-    passwordSecret:
-      # -- The name of the Secret containing the password used to connect to the relational database.
-      name:
-      # -- The namespace of the Secret containing the password used to connect to the relational database.
-      namespace:
-      # -- The Secret key where the data is stored.
-      keyPath:
-  tlsConfig:
-    certificateAuthority:
-      certificate:
-        # -- Name of the Secret containing the CA required to establish the mandatory SSL/TLS connection to the datastore.
-        name:
-        # -- Namespace of the Secret containing the CA required to establish the mandatory SSL/TLS connection to the datastore.
-        namespace:
-        # -- Key of the Secret which contains the content of the certificate.
-        keyPath:
-      privateKey:
-        # -- Name of the Secret containing the CA private key required to establish the mandatory SSL/TLS connection to the datastore.
-        name:
-        # -- Namespace of the Secret containing the CA private key required to establish the mandatory SSL/TLS connection to the datastore.
-        namespace:
-        # -- Key of the Secret which contains the content of the private key.
-        keyPath:
-    clientCertificate:
-      certificate:
-        # -- Name of the Secret containing the client certificate required to establish the mandatory SSL/TLS connection to the datastore.
-        name:
-        # -- Namespace of the Secret containing the client certificate required to establish the mandatory SSL/TLS connection to the datastore.
-        namespace:
-        # -- Key of the Secret which contains the content of the certificate.
-        keyPath:
-      privateKey:
-        # -- Name of the Secret containing the client certificate private key required to establish the mandatory SSL/TLS connection to the datastore.
-        name:
-        # -- Namespace of the Secret containing the client certificate private key required to establish the mandatory SSL/TLS connection to the datastore.
-        namespace:
-        # -- Key of the Secret which contains the content of the private key.
-        keyPath:

config/crd/bases/kamaji.clastix.io_datastores.yaml

@@ -1,268 +0,0 @@
----
-apiVersion: apiextensions.k8s.io/v1
-kind: CustomResourceDefinition
-metadata:
-  annotations:
-    controller-gen.kubebuilder.io/version: v0.9.2
-  creationTimestamp: null
-  name: datastores.kamaji.clastix.io
-spec:
-  group: kamaji.clastix.io
-  names:
-    kind: DataStore
-    listKind: DataStoreList
-    plural: datastores
-    singular: datastore
-  scope: Cluster
-  versions:
-  - additionalPrinterColumns:
-    - description: Kamaji data store driver
-      jsonPath: .spec.driver
-      name: Driver
-      type: string
-    - description: Age
-      jsonPath: .metadata.creationTimestamp
-      name: Age
-      type: date
-    name: v1alpha1
-    schema:
-      openAPIV3Schema:
-        description: DataStore is the Schema for the datastores API.
-        properties:
-          apiVersion:
-            description: 'APIVersion defines the versioned schema of this representation
-              of an object. Servers should convert recognized schemas to the latest
-              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
-            type: string
-          kind:
-            description: 'Kind is a string value representing the REST resource this
-              object represents. Servers may infer this from the endpoint the client
-              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
-            type: string
-          metadata:
-            type: object
-          spec:
-            description: DataStoreSpec defines the desired state of DataStore.
-            properties:
-              basicAuth:
-                description: In case of authentication enabled for the given data
-                  store, specifies the username and password pair. This value is optional.
-                properties:
-                  password:
-                    properties:
-                      content:
-                        description: Bare content of the file, base64 encoded. It
-                          has precedence over the SecretReference value.
-                        format: byte
-                        type: string
-                      secretReference:
-                        properties:
-                          keyPath:
-                            description: Name of the key for the given Secret reference
-                              where the content is stored. This value is mandatory.
-                            type: string
-                          name:
-                            description: name is unique within a namespace to reference
-                              a secret resource.
-                            type: string
-                          namespace:
-                            description: namespace defines the space within which
-                              the secret name must be unique.
-                            type: string
-                        required:
-                        - keyPath
-                        type: object
-                        x-kubernetes-map-type: atomic
-                    type: object
-                  username:
-                    properties:
-                      content:
-                        description: Bare content of the file, base64 encoded. It
-                          has precedence over the SecretReference value.
-                        format: byte
-                        type: string
-                      secretReference:
-                        properties:
-                          keyPath:
-                            description: Name of the key for the given Secret reference
-                              where the content is stored. This value is mandatory.
-                            type: string
-                          name:
-                            description: name is unique within a namespace to reference
-                              a secret resource.
-                            type: string
-                          namespace:
-                            description: namespace defines the space within which
-                              the secret name must be unique.
-                            type: string
-                        required:
-                        - keyPath
-                        type: object
-                        x-kubernetes-map-type: atomic
-                    type: object
-                required:
-                - password
-                - username
-                type: object
-              driver:
-                description: The driver to use to connect to the shared datastore.
-                type: string
-              endpoints:
-                description: List of the endpoints to connect to the shared datastore.
-                  No need for protocol, just bare IP/FQDN and port.
-                items:
-                  type: string
-                type: array
-              tlsConfig:
-                description: Defines the TLS/SSL configuration required to connect
-                  to the data store in a secure way.
-                properties:
-                  certificateAuthority:
-                    description: Retrieve the Certificate Authority certificate and
-                      private key, such as bare content of the file, or a SecretReference.
-                      The key reference is required since etcd authentication is based
-                      on certificates, and Kamaji is responsible in creating this.
-                    properties:
-                      certificate:
-                        properties:
-                          content:
-                            description: Bare content of the file, base64 encoded.
-                              It has precedence over the SecretReference value.
-                            format: byte
-                            type: string
-                          secretReference:
-                            properties:
-                              keyPath:
-                                description: Name of the key for the given Secret
-                                  reference where the content is stored. This value
-                                  is mandatory.
-                                type: string
-                              name:
-                                description: name is unique within a namespace to
-                                  reference a secret resource.
-                                type: string
-                              namespace:
-                                description: namespace defines the space within which
-                                  the secret name must be unique.
-                                type: string
-                            required:
-                            - keyPath
-                            type: object
-                            x-kubernetes-map-type: atomic
-                        type: object
-                      privateKey:
-                        properties:
-                          content:
-                            description: Bare content of the file, base64 encoded.
-                              It has precedence over the SecretReference value.
-                            format: byte
-                            type: string
-                          secretReference:
-                            properties:
-                              keyPath:
-                                description: Name of the key for the given Secret
-                                  reference where the content is stored. This value
-                                  is mandatory.
-                                type: string
-                              name:
-                                description: name is unique within a namespace to
-                                  reference a secret resource.
-                                type: string
-                              namespace:
-                                description: namespace defines the space within which
-                                  the secret name must be unique.
-                                type: string
-                            required:
-                            - keyPath
-                            type: object
-                            x-kubernetes-map-type: atomic
-                        type: object
-                    required:
-                    - certificate
-                    type: object
-                  clientCertificate:
-                    description: Specifies the SSL/TLS key and private key pair used
-                      to connect to the data store.
-                    properties:
-                      certificate:
-                        properties:
-                          content:
-                            description: Bare content of the file, base64 encoded.
-                              It has precedence over the SecretReference value.
-                            format: byte
-                            type: string
-                          secretReference:
-                            properties:
-                              keyPath:
-                                description: Name of the key for the given Secret
-                                  reference where the content is stored. This value
-                                  is mandatory.
-                                type: string
-                              name:
-                                description: name is unique within a namespace to
-                                  reference a secret resource.
-                                type: string
-                              namespace:
-                                description: namespace defines the space within which
-                                  the secret name must be unique.
-                                type: string
-                            required:
-                            - keyPath
-                            type: object
-                            x-kubernetes-map-type: atomic
-                        type: object
-                      privateKey:
-                        properties:
-                          content:
-                            description: Bare content of the file, base64 encoded.
-                              It has precedence over the SecretReference value.
-                            format: byte
-                            type: string
-                          secretReference:
-                            properties:
-                              keyPath:
-                                description: Name of the key for the given Secret
-                                  reference where the content is stored. This value
-                                  is mandatory.
-                                type: string
-                              name:
-                                description: name is unique within a namespace to
-                                  reference a secret resource.
-                                type: string
-                              namespace:
-                                description: namespace defines the space within which
-                                  the secret name must be unique.
-                                type: string
-                            required:
-                            - keyPath
-                            type: object
-                            x-kubernetes-map-type: atomic
-                        type: object
-                    required:
-                    - certificate
-                    - privateKey
-                    type: object
-                required:
-                - certificateAuthority
-                - clientCertificate
-                type: object
-            required:
-            - driver
-            - endpoints
-            - tlsConfig
-            type: object
-          status:
-            description: DataStoreStatus defines the observed state of DataStore.
-            properties:
-              usedBy:
-                description: List of the Tenant Control Planes, namespaced named,
-                  using this data store.
-                items:
-                  type: string
-                type: array
-            type: object
-        type: object
-    served: true
-    storage: true
-    subresources:
-      status: {}

config/crd/bases/kamaji.clastix.io_tenantcontrolplanes.yaml

@@ -1,9 +1,10 @@
+
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.9.2
+    controller-gen.kubebuilder.io/version: v0.6.1
   creationTimestamp: null
   name: tenantcontrolplanes.kamaji.clastix.io
 spec:
@@ -63,27 +64,13 @@ spec:
                 description: Addons contain which addons are enabled
                 properties:
                   coreDNS:
-                    description: Enables the DNS addon in the Tenant Cluster. The
-                      registry and the tag are configurable, the image is hard-coded
-                      to `coredns`.
-                    properties:
-                      imageRepository:
-                        description: ImageRepository sets the container registry to
-                          pull images from. if not set, the default ImageRepository
-                          will be used instead.
-                        type: string
-                      imageTag:
-                        description: ImageTag allows to specify a tag for the image.
-                          In case this value is set, kubeadm does not change automatically
-                          the version of the above components during upgrades.
-                        type: string
+                    description: AddonSpec defines the spec for every addon.
                     type: object
                   konnectivity:
-                    description: Enables the Konnectivity addon in the Tenant Cluster,
-                      required if the worker nodes are in a different network.
+                    description: KonnectivitySpec defines the spec for Konnectivity.
                     properties:
                       agentImage:
-                        default: registry.k8s.io/kas-network-proxy/proxy-agent
+                        default: us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-agent
                         description: AgentImage defines the container image for Konnectivity's
                           agent.
                         type: string
@@ -120,32 +107,19 @@ spec:
                             type: object
                         type: object
                       serverImage:
-                        default: registry.k8s.io/kas-network-proxy/proxy-server
+                        default: us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-server
                         description: ServerImage defines the container image for Konnectivity's
                           server.
                         type: string
                       version:
-                        default: v0.0.32
+                        default: v0.0.31
                         description: Version for Konnectivity server and agent.
                         type: string
                     required:
                     - proxyPort
                     type: object
                   kubeProxy:
-                    description: Enables the kube-proxy addon in the Tenant Cluster.
-                      The registry and the tag are configurable, the image is hard-coded
-                      to `kube-proxy`.
-                    properties:
-                      imageRepository:
-                        description: ImageRepository sets the container registry to
-                          pull images from. if not set, the default ImageRepository
-                          will be used instead.
-                        type: string
-                      imageTag:
-                        description: ImageTag allows to specify a tag for the image.
-                          In case this value is set, kubeadm does not change automatically
-                          the version of the above components during upgrades.
-                        type: string
+                    description: AddonSpec defines the spec for every addon.
                     type: object
                 type: object
               controlPlane:
@@ -289,194 +263,6 @@ spec:
                                 type: object
                             type: object
                         type: object
-                      topologySpreadConstraints:
-                        description: TopologySpreadConstraints describes how the Tenant
-                          Control Plane pods ought to spread across topology domains.
-                          Scheduler will schedule pods in a way which abides by the
-                          constraints. In case of nil underlying LabelSelector, the
-                          Kamaji one for the given Tenant Control Plane will be used.
-                          All topologySpreadConstraints are ANDed.
-                        items:
-                          description: TopologySpreadConstraint specifies how to spread
-                            matching pods among the given topology.
-                          properties:
-                            labelSelector:
-                              description: LabelSelector is used to find matching
-                                pods. Pods that match this label selector are counted
-                                to determine the number of pods in their corresponding
-                                topology domain.
-                              properties:
-                                matchExpressions:
-                                  description: matchExpressions is a list of label
-                                    selector requirements. The requirements are ANDed.
-                                  items:
-                                    description: A label selector requirement is a
-                                      selector that contains values, a key, and an
-                                      operator that relates the key and values.
-                                    properties:
-                                      key:
-                                        description: key is the label key that the
-                                          selector applies to.
-                                        type: string
-                                      operator:
-                                        description: operator represents a key's relationship
-                                          to a set of values. Valid operators are
-                                          In, NotIn, Exists and DoesNotExist.
-                                        type: string
-                                      values:
-                                        description: values is an array of string
-                                          values. If the operator is In or NotIn,
-                                          the values array must be non-empty. If the
-                                          operator is Exists or DoesNotExist, the
-                                          values array must be empty. This array is
-                                          replaced during a strategic merge patch.
-                                        items:
-                                          type: string
-                                        type: array
-                                    required:
-                                    - key
-                                    - operator
-                                    type: object
-                                  type: array
-                                matchLabels:
-                                  additionalProperties:
-                                    type: string
-                                  description: matchLabels is a map of {key,value}
-                                    pairs. A single {key,value} in the matchLabels
-                                    map is equivalent to an element of matchExpressions,
-                                    whose key field is "key", the operator is "In",
-                                    and the values array contains only "value". The
-                                    requirements are ANDed.
-                                  type: object
-                              type: object
-                              x-kubernetes-map-type: atomic
-                            matchLabelKeys:
-                              description: MatchLabelKeys is a set of pod label keys
-                                to select the pods over which spreading will be calculated.
-                                The keys are used to lookup values from the incoming
-                                pod labels, those key-value labels are ANDed with
-                                labelSelector to select the group of existing pods
-                                over which spreading will be calculated for the incoming
-                                pod. Keys that don't exist in the incoming pod labels
-                                will be ignored. A null or empty list means only match
-                                against labelSelector.
-                              items:
-                                type: string
-                              type: array
-                              x-kubernetes-list-type: atomic
-                            maxSkew:
-                              description: 'MaxSkew describes the degree to which
-                                pods may be unevenly distributed. When `whenUnsatisfiable=DoNotSchedule`,
-                                it is the maximum permitted difference between the
-                                number of matching pods in the target topology and
-                                the global minimum. The global minimum is the minimum
-                                number of matching pods in an eligible domain or zero
-                                if the number of eligible domains is less than MinDomains.
-                                For example, in a 3-zone cluster, MaxSkew is set to
-                                1, and pods with the same labelSelector spread as
-                                2/2/1: In this case, the global minimum is 1. | zone1
-                                | zone2 | zone3 | |  P P  |  P P  |   P   | - if MaxSkew
-                                is 1, incoming pod can only be scheduled to zone3
-                                to become 2/2/2; scheduling it onto zone1(zone2) would
-                                make the ActualSkew(3-1) on zone1(zone2) violate MaxSkew(1).
-                                - if MaxSkew is 2, incoming pod can be scheduled onto
-                                any zone. When `whenUnsatisfiable=ScheduleAnyway`,
-                                it is used to give higher precedence to topologies
-                                that satisfy it. It''s a required field. Default value
-                                is 1 and 0 is not allowed.'
-                              format: int32
-                              type: integer
-                            minDomains:
-                              description: "MinDomains indicates a minimum number
-                                of eligible domains. When the number of eligible domains
-                                with matching topology keys is less than minDomains,
-                                Pod Topology Spread treats \"global minimum\" as 0,
-                                and then the calculation of Skew is performed. And
-                                when the number of eligible domains with matching
-                                topology keys equals or greater than minDomains, this
-                                value has no effect on scheduling. As a result, when
-                                the number of eligible domains is less than minDomains,
-                                scheduler won't schedule more than maxSkew Pods to
-                                those domains. If value is nil, the constraint behaves
-                                as if MinDomains is equal to 1. Valid values are integers
-                                greater than 0. When value is not nil, WhenUnsatisfiable
-                                must be DoNotSchedule. \n For example, in a 3-zone
-                                cluster, MaxSkew is set to 2, MinDomains is set to
-                                5 and pods with the same labelSelector spread as 2/2/2:
-                                | zone1 | zone2 | zone3 | |  P P  |  P P  |  P P  |
-                                The number of domains is less than 5(MinDomains),
-                                so \"global minimum\" is treated as 0. In this situation,
-                                new pod with the same labelSelector cannot be scheduled,
-                                because computed skew will be 3(3 - 0) if new Pod
-                                is scheduled to any of the three zones, it will violate
-                                MaxSkew. \n This is a beta field and requires the
-                                MinDomainsInPodTopologySpread feature gate to be enabled
-                                (enabled by default)."
-                              format: int32
-                              type: integer
-                            nodeAffinityPolicy:
-                              description: "NodeAffinityPolicy indicates how we will
-                                treat Pod's nodeAffinity/nodeSelector when calculating
-                                pod topology spread skew. Options are: - Honor: only
-                                nodes matching nodeAffinity/nodeSelector are included
-                                in the calculations. - Ignore: nodeAffinity/nodeSelector
-                                are ignored. All nodes are included in the calculations.
-                                \n If this value is nil, the behavior is equivalent
-                                to the Honor policy. This is a alpha-level feature
-                                enabled by the NodeInclusionPolicyInPodTopologySpread
-                                feature flag."
-                              type: string
-                            nodeTaintsPolicy:
-                              description: "NodeTaintsPolicy indicates how we will
-                                treat node taints when calculating pod topology spread
-                                skew. Options are: - Honor: nodes without taints,
-                                along with tainted nodes for which the incoming pod
-                                has a toleration, are included. - Ignore: node taints
-                                are ignored. All nodes are included. \n If this value
-                                is nil, the behavior is equivalent to the Ignore policy.
-                                This is a alpha-level feature enabled by the NodeInclusionPolicyInPodTopologySpread
-                                feature flag."
-                              type: string
-                            topologyKey:
-                              description: TopologyKey is the key of node labels.
-                                Nodes that have a label with this key and identical
-                                values are considered to be in the same topology.
-                                We consider each <key, value> as a "bucket", and try
-                                to put balanced number of pods into each bucket. We
-                                define a domain as a particular instance of a topology.
-                                Also, we define an eligible domain as a domain whose
-                                nodes meet the requirements of nodeAffinityPolicy
-                                and nodeTaintsPolicy. e.g. If TopologyKey is "kubernetes.io/hostname",
-                                each Node is a domain of that topology. And, if TopologyKey
-                                is "topology.kubernetes.io/zone", each zone is a domain
-                                of that topology. It's a required field.
-                              type: string
-                            whenUnsatisfiable:
-                              description: 'WhenUnsatisfiable indicates how to deal
-                                with a pod if it doesn''t satisfy the spread constraint.
-                                - DoNotSchedule (default) tells the scheduler not
-                                to schedule it. - ScheduleAnyway tells the scheduler
-                                to schedule the pod in any location, but giving higher
-                                precedence to topologies that would help reduce the
-                                skew. A constraint is considered "Unsatisfiable" for
-                                an incoming pod if and only if every possible node
-                                assignment for that pod would violate "MaxSkew" on
-                                some topology. For example, in a 3-zone cluster, MaxSkew
-                                is set to 1, and pods with the same labelSelector
-                                spread as 3/1/1: | zone1 | zone2 | zone3 | | P P P
-                                |   P   |   P   | If WhenUnsatisfiable is set to DoNotSchedule,
-                                incoming pod can only be scheduled to zone2(zone3)
-                                to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3)
-                                satisfies MaxSkew(1). In other words, the cluster
-                                can still be imbalanced, but scheduler won''t make
-                                it *more* imbalanced. It''s a required field.'
-                              type: string
-                          required:
-                          - maxSkew
-                          - topologyKey
-                          - whenUnsatisfiable
-                          type: object
-                        type: array
                     type: object
                   ingress:
                     description: Defining the options for an Optional Ingress which
@@ -537,14 +323,6 @@ spec:
                 required:
                 - service
                 type: object
-              dataStore:
-                description: DataStore allows to specify a DataStore that should be
-                  used to store the Kubernetes data for the given Tenant Control Plane.
-                  This parameter is optional and acts as an override over the default
-                  one which is used by the Kamaji Operator. Migration from a different
-                  DataStore to another one is not yet supported and the reconciliation
-                  will be blocked.
-                type: string
               kubernetes:
                 description: Kubernetes specification for tenant control plane
                 properties:
@@ -776,15 +554,15 @@ spec:
                               description: "Condition contains details for one aspect
                                 of the current state of this API Resource. --- This
                                 struct is intended for direct use as an array at the
-                                field path .status.conditions.  For example, \n type
-                                FooStatus struct{ // Represents the observations of
-                                a foo's current state. // Known .status.conditions.type
+                                field path .status.conditions.  For example, type
+                                FooStatus struct{     // Represents the observations
+                                of a foo's current state.     // Known .status.conditions.type
                                 are: \"Available\", \"Progressing\", and \"Degraded\"
-                                // +patchMergeKey=type // +patchStrategy=merge //
-                                +listType=map // +listMapKey=type Conditions []metav1.Condition
-                                `json:\"conditions,omitempty\" patchStrategy:\"merge\"
-                                patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"`
-                                \n // other fields }"
+                                \    // +patchMergeKey=type     // +patchStrategy=merge
+                                \    // +listType=map     // +listMapKey=type     Conditions
+                                []metav1.Condition `json:\"conditions,omitempty\"
+                                patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"`
+                                \n     // other fields }"
                               properties:
                                 lastTransitionTime:
                                   description: lastTransitionTime is the last time
@@ -888,10 +666,11 @@ spec:
                                               the error shall comply with the following
                                               rules: - built-in error values shall
                                               be specified in this file and those
-                                              shall use CamelCase names - cloud provider
-                                              specific error values must have names
-                                              that comply with the format foo.example.com/CamelCase.
-                                              --- The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)'
+                                              shall use   CamelCase names - cloud
+                                              provider specific error values must
+                                              have names that comply with the   format
+                                              foo.example.com/CamelCase. --- The regex
+                                              it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)'
                                             maxLength: 316
                                             pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
                                             type: string
@@ -1281,9 +1060,9 @@ spec:
                                           with the service port The format of the
                                           error shall comply with the following rules:
                                           - built-in error values shall be specified
-                                          in this file and those shall use CamelCase
+                                          in this file and those shall use   CamelCase
                                           names - cloud provider specific error values
-                                          must have names that comply with the format
+                                          must have names that comply with the   format
                                           foo.example.com/CamelCase. --- The regex
                                           it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)'
                                         maxLength: 316
@@ -1332,14 +1111,14 @@ spec:
                           description: "Condition contains details for one aspect
                             of the current state of this API Resource. --- This struct
                             is intended for direct use as an array at the field path
-                            .status.conditions.  For example, \n type FooStatus struct{
-                            // Represents the observations of a foo's current state.
-                            // Known .status.conditions.type are: \"Available\", \"Progressing\",
-                            and \"Degraded\" // +patchMergeKey=type // +patchStrategy=merge
-                            // +listType=map // +listMapKey=type Conditions []metav1.Condition
-                            `json:\"conditions,omitempty\" patchStrategy:\"merge\"
-                            patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"`
-                            \n // other fields }"
+                            .status.conditions.  For example, type FooStatus struct{
+                            \    // Represents the observations of a foo's current
+                            state.     // Known .status.conditions.type are: \"Available\",
+                            \"Progressing\", and \"Degraded\"     // +patchMergeKey=type
+                            \    // +patchStrategy=merge     // +listType=map     //
+                            +listMapKey=type     Conditions []metav1.Condition `json:\"conditions,omitempty\"
+                            patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"`
+                            \n     // other fields }"
                           properties:
                             lastTransitionTime:
                               description: lastTransitionTime is the last time the
@@ -1438,9 +1217,9 @@ spec:
                                           with the service port The format of the
                                           error shall comply with the following rules:
                                           - built-in error values shall be specified
-                                          in this file and those shall use CamelCase
+                                          in this file and those shall use   CamelCase
                                           names - cloud provider specific error values
-                                          must have names that comply with the format
+                                          must have names that comply with the   format
                                           foo.example.com/CamelCase. --- The regex
                                           it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)'
                                         maxLength: 316
@@ -1508,38 +1287,78 @@ spec:
                 description: Storage Status contains information about Kubernetes
                   storage system
                 properties:
-                  certificate:
+                  etcd:
+                    description: ETCDStatus defines the observed state of ETCDStatus.
                     properties:
-                      checksum:
-                        type: string
-                      lastUpdate:
-                        format: date-time
-                        type: string
-                      secretName:
-                        type: string
-                    type: object
-                  config:
-                    properties:
-                      checksum:
-                        type: string
-                      secretName:
-                        type: string
-                    type: object
-                  dataStoreName:
-                    type: string
-                  driver:
-                    type: string
-                  setup:
-                    properties:
-                      checksum:
-                        type: string
-                      lastUpdate:
-                        format: date-time
-                        type: string
-                      schema:
-                        type: string
+                      role:
+                        properties:
+                          exists:
+                            type: boolean
+                          name:
+                            type: string
+                          permissions:
+                            items:
+                              properties:
+                                key:
+                                  type: string
+                                rangeEnd:
+                                  type: string
+                                type:
+                                  type: integer
+                              type: object
+                            type: array
+                        required:
+                        - exists
+                        - name
+                        type: object
                       user:
+                        properties:
+                          exists:
+                            type: boolean
+                          name:
+                            type: string
+                          roles:
+                            items:
+                              type: string
+                            type: array
+                        required:
+                        - exists
+                        - name
+                        type: object
+                    type: object
+                  kine:
+                    properties:
+                      certificate:
+                        properties:
+                          checksum:
+                            type: string
+                          lastUpdate:
+                            format: date-time
+                            type: string
+                          secretName:
+                            type: string
+                        type: object
+                      config:
+                        properties:
+                          checksum:
+                            type: string
+                          secretName:
+                            type: string
+                        type: object
+                      driver:
                         type: string
+                      setup:
+                        properties:
+                          checksum:
+                            type: string
+                          lastUpdate:
+                            format: date-time
+                            type: string
+                          schema:
+                            type: string
+                          user:
+                            type: string
+                        type: object
                     type: object
                 type: object
             type: object
@@ -1548,3 +1367,9 @@ spec:
     storage: true
     subresources:
       status: {}
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []

config/crd/patches/cainjection_in_datastores.yaml

@@ -1,7 +0,0 @@
-# The following patch adds a directive for certmanager to inject CA into the CRD
-apiVersion: apiextensions.k8s.io/v1
-kind: CustomResourceDefinition
-metadata:
-  annotations:
-    cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME)
-  name: datastores.kamaji.clastix.io

config/crd/patches/webhook_in_datastores.yaml

@@ -1,16 +0,0 @@
-# The following patch enables a conversion webhook for the CRD
-apiVersion: apiextensions.k8s.io/v1
-kind: CustomResourceDefinition
-metadata:
-  name: datastores.kamaji.clastix.io
-spec:
-  conversion:
-    strategy: Webhook
-    webhook:
-      clientConfig:
-        service:
-          namespace: system
-          name: webhook-service
-          path: /convert
-      conversionReviewVersions:
-      - v1

config/default/kustomization.yaml

@@ -16,7 +16,6 @@ bases:
 - ../crd
 - ../rbac
 - ../manager
-- ../samples
 # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
 # crd/kustomization.yaml
 #- ../webhook

config/install.yaml

@@ -9,7 +9,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.9.2
+    controller-gen.kubebuilder.io/version: v0.6.1
   creationTimestamp: null
   name: tenantcontrolplanes.kamaji.clastix.io
 spec:
@@ -64,20 +64,13 @@ spec:
                 description: Addons contain which addons are enabled
                 properties:
                   coreDNS:
-                    description: Enables the DNS addon in the Tenant Cluster. The registry and the tag are configurable, the image is hard-coded to `coredns`.
-                    properties:
-                      imageRepository:
-                        description: ImageRepository sets the container registry to pull images from. if not set, the default ImageRepository will be used instead.
-                        type: string
-                      imageTag:
-                        description: ImageTag allows to specify a tag for the image. In case this value is set, kubeadm does not change automatically the version of the above components during upgrades.
-                        type: string
+                    description: AddonSpec defines the spec for every addon.
                     type: object
                   konnectivity:
-                    description: Enables the Konnectivity addon in the Tenant Cluster, required if the worker nodes are in a different network.
+                    description: KonnectivitySpec defines the spec for Konnectivity.
                     properties:
                       agentImage:
-                        default: registry.k8s.io/kas-network-proxy/proxy-agent
+                        default: us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-agent
                         description: AgentImage defines the container image for Konnectivity's agent.
                         type: string
                       proxyPort:
@@ -107,25 +100,18 @@ spec:
                             type: object
                         type: object
                       serverImage:
-                        default: registry.k8s.io/kas-network-proxy/proxy-server
+                        default: us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-server
                         description: ServerImage defines the container image for Konnectivity's server.
                         type: string
                       version:
-                        default: v0.0.32
+                        default: v0.0.31
                         description: Version for Konnectivity server and agent.
                         type: string
                     required:
                     - proxyPort
                     type: object
                   kubeProxy:
-                    description: Enables the kube-proxy addon in the Tenant Cluster. The registry and the tag are configurable, the image is hard-coded to `kube-proxy`.
-                    properties:
-                      imageRepository:
-                        description: ImageRepository sets the container registry to pull images from. if not set, the default ImageRepository will be used instead.
-                        type: string
-                      imageTag:
-                        description: ImageTag allows to specify a tag for the image. In case this value is set, kubeadm does not change automatically the version of the above components during upgrades.
-                        type: string
+                    description: AddonSpec defines the spec for every addon.
                     type: object
                 type: object
               controlPlane:
@@ -241,74 +227,6 @@ spec:
                                 type: object
                             type: object
                         type: object
-                      topologySpreadConstraints:
-                        description: TopologySpreadConstraints describes how the Tenant Control Plane pods ought to spread across topology domains. Scheduler will schedule pods in a way which abides by the constraints. In case of nil underlying LabelSelector, the Kamaji one for the given Tenant Control Plane will be used. All topologySpreadConstraints are ANDed.
-                        items:
-                          description: TopologySpreadConstraint specifies how to spread matching pods among the given topology.
-                          properties:
-                            labelSelector:
-                              description: LabelSelector is used to find matching pods. Pods that match this label selector are counted to determine the number of pods in their corresponding topology domain.
-                              properties:
-                                matchExpressions:
-                                  description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
-                                  items:
-                                    description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.
-                                    properties:
-                                      key:
-                                        description: key is the label key that the selector applies to.
-                                        type: string
-                                      operator:
-                                        description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.
-                                        type: string
-                                      values:
-                                        description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.
-                                        items:
-                                          type: string
-                                        type: array
-                                    required:
-                                    - key
-                                    - operator
-                                    type: object
-                                  type: array
-                                matchLabels:
-                                  additionalProperties:
-                                    type: string
-                                  description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed.
-                                  type: object
-                              type: object
-                              x-kubernetes-map-type: atomic
-                            matchLabelKeys:
-                              description: MatchLabelKeys is a set of pod label keys to select the pods over which spreading will be calculated. The keys are used to lookup values from the incoming pod labels, those key-value labels are ANDed with labelSelector to select the group of existing pods over which spreading will be calculated for the incoming pod. Keys that don't exist in the incoming pod labels will be ignored. A null or empty list means only match against labelSelector.
-                              items:
-                                type: string
-                              type: array
-                              x-kubernetes-list-type: atomic
-                            maxSkew:
-                              description: 'MaxSkew describes the degree to which pods may be unevenly distributed. When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference between the number of matching pods in the target topology and the global minimum. The global minimum is the minimum number of matching pods in an eligible domain or zero if the number of eligible domains is less than MinDomains. For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same labelSelector spread as 2/2/1: In this case, the global minimum is 1. | zone1 | zone2 | zone3 | |  P P  |  P P  |   P   | - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) violate MaxSkew(1). - if MaxSkew is 2, incoming pod can be scheduled onto any zone. When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence to topologies that satisfy it. It''s a required field. Default value is 1 and 0 is not allowed.'
-                              format: int32
-                              type: integer
-                            minDomains:
-                              description: "MinDomains indicates a minimum number of eligible domains. When the number of eligible domains with matching topology keys is less than minDomains, Pod Topology Spread treats \"global minimum\" as 0, and then the calculation of Skew is performed. And when the number of eligible domains with matching topology keys equals or greater than minDomains, this value has no effect on scheduling. As a result, when the number of eligible domains is less than minDomains, scheduler won't schedule more than maxSkew Pods to those domains. If value is nil, the constraint behaves as if MinDomains is equal to 1. Valid values are integers greater than 0. When value is not nil, WhenUnsatisfiable must be DoNotSchedule. \n For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same labelSelector spread as 2/2/2: | zone1 | zone2 | zone3 | |  P P  |  P P  |  P P  | The number of domains is less than 5(MinDomains), so \"global minimum\" is treated as 0. In this situation, new pod with the same labelSelector cannot be scheduled, because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, it will violate MaxSkew. \n This is a beta field and requires the MinDomainsInPodTopologySpread feature gate to be enabled (enabled by default)."
-                              format: int32
-                              type: integer
-                            nodeAffinityPolicy:
-                              description: "NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector when calculating pod topology spread skew. Options are: - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. \n If this value is nil, the behavior is equivalent to the Honor policy. This is a alpha-level feature enabled by the NodeInclusionPolicyInPodTopologySpread feature flag."
-                              type: string
-                            nodeTaintsPolicy:
-                              description: "NodeTaintsPolicy indicates how we will treat node taints when calculating pod topology spread skew. Options are: - Honor: nodes without taints, along with tainted nodes for which the incoming pod has a toleration, are included. - Ignore: node taints are ignored. All nodes are included. \n If this value is nil, the behavior is equivalent to the Ignore policy. This is a alpha-level feature enabled by the NodeInclusionPolicyInPodTopologySpread feature flag."
-                              type: string
-                            topologyKey:
-                              description: TopologyKey is the key of node labels. Nodes that have a label with this key and identical values are considered to be in the same topology. We consider each <key, value> as a "bucket", and try to put balanced number of pods into each bucket. We define a domain as a particular instance of a topology. Also, we define an eligible domain as a domain whose nodes meet the requirements of nodeAffinityPolicy and nodeTaintsPolicy. e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. It's a required field.
-                              type: string
-                            whenUnsatisfiable:
-                              description: 'WhenUnsatisfiable indicates how to deal with a pod if it doesn''t satisfy the spread constraint. - DoNotSchedule (default) tells the scheduler not to schedule it. - ScheduleAnyway tells the scheduler to schedule the pod in any location, but giving higher precedence to topologies that would help reduce the skew. A constraint is considered "Unsatisfiable" for an incoming pod if and only if every possible node assignment for that pod would violate "MaxSkew" on some topology. For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same labelSelector spread as 3/1/1: | zone1 | zone2 | zone3 | | P P P |   P   |   P   | If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler won''t make it *more* imbalanced. It''s a required field.'
-                              type: string
-                          required:
-                          - maxSkew
-                          - topologyKey
-                          - whenUnsatisfiable
-                          type: object
-                        type: array
                     type: object
                   ingress:
                     description: Defining the options for an Optional Ingress which will expose API Server of the Tenant Control Plane
@@ -359,9 +277,6 @@ spec:
                 required:
                 - service
                 type: object
-              dataStore:
-                description: DataStore allows to specify a DataStore that should be used to store the Kubernetes data for the given Tenant Control Plane. This parameter is optional and acts as an override over the default one which is used by the Kamaji Operator. Migration from a different DataStore to another one is not yet supported and the reconciliation will be blocked.
-                type: string
               kubernetes:
                 description: Kubernetes specification for tenant control plane
                 properties:
@@ -578,7 +493,7 @@ spec:
                           conditions:
                             description: Current service state
                             items:
-                              description: "Condition contains details for one aspect of the current state of this API Resource. --- This struct is intended for direct use as an array at the field path .status.conditions.  For example, \n type FooStatus struct{ // Represents the observations of a foo's current state. // Known .status.conditions.type are: \"Available\", \"Progressing\", and \"Degraded\" // +patchMergeKey=type // +patchStrategy=merge // +listType=map // +listMapKey=type Conditions []metav1.Condition `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"` \n // other fields }"
+                              description: "Condition contains details for one aspect of the current state of this API Resource. --- This struct is intended for direct use as an array at the field path .status.conditions.  For example, type FooStatus struct{     // Represents the observations of a foo's current state.     // Known .status.conditions.type are: \"Available\", \"Progressing\", and \"Degraded\"     // +patchMergeKey=type     // +patchStrategy=merge     // +listType=map     // +listMapKey=type     Conditions []metav1.Condition `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"` \n     // other fields }"
                               properties:
                                 lastTransitionTime:
                                   description: lastTransitionTime is the last time the condition transitioned from one status to another. This should be when the underlying condition changed.  If that is not known, then using the time when the API field changed is acceptable.
@@ -641,7 +556,7 @@ spec:
                                       items:
                                         properties:
                                           error:
-                                            description: 'Error is to record the problem with the service port The format of the error shall comply with the following rules: - built-in error values shall be specified in this file and those shall use CamelCase names - cloud provider specific error values must have names that comply with the format foo.example.com/CamelCase. --- The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)'
+                                            description: 'Error is to record the problem with the service port The format of the error shall comply with the following rules: - built-in error values shall be specified in this file and those shall use   CamelCase names - cloud provider specific error values must have names that comply with the   format foo.example.com/CamelCase. --- The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)'
                                             maxLength: 316
                                             pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
                                             type: string
@@ -977,7 +892,7 @@ spec:
                                   items:
                                     properties:
                                       error:
-                                        description: 'Error is to record the problem with the service port The format of the error shall comply with the following rules: - built-in error values shall be specified in this file and those shall use CamelCase names - cloud provider specific error values must have names that comply with the format foo.example.com/CamelCase. --- The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)'
+                                        description: 'Error is to record the problem with the service port The format of the error shall comply with the following rules: - built-in error values shall be specified in this file and those shall use   CamelCase names - cloud provider specific error values must have names that comply with the   format foo.example.com/CamelCase. --- The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)'
                                         maxLength: 316
                                         pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
                                         type: string
@@ -1014,7 +929,7 @@ spec:
                       conditions:
                         description: Current service state
                         items:
-                          description: "Condition contains details for one aspect of the current state of this API Resource. --- This struct is intended for direct use as an array at the field path .status.conditions.  For example, \n type FooStatus struct{ // Represents the observations of a foo's current state. // Known .status.conditions.type are: \"Available\", \"Progressing\", and \"Degraded\" // +patchMergeKey=type // +patchStrategy=merge // +listType=map // +listMapKey=type Conditions []metav1.Condition `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"` \n // other fields }"
+                          description: "Condition contains details for one aspect of the current state of this API Resource. --- This struct is intended for direct use as an array at the field path .status.conditions.  For example, type FooStatus struct{     // Represents the observations of a foo's current state.     // Known .status.conditions.type are: \"Available\", \"Progressing\", and \"Degraded\"     // +patchMergeKey=type     // +patchStrategy=merge     // +listType=map     // +listMapKey=type     Conditions []metav1.Condition `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"` \n     // other fields }"
                           properties:
                             lastTransitionTime:
                               description: lastTransitionTime is the last time the condition transitioned from one status to another. This should be when the underlying condition changed.  If that is not known, then using the time when the API field changed is acceptable.
@@ -1077,7 +992,7 @@ spec:
                                   items:
                                     properties:
                                       error:
-                                        description: 'Error is to record the problem with the service port The format of the error shall comply with the following rules: - built-in error values shall be specified in this file and those shall use CamelCase names - cloud provider specific error values must have names that comply with the format foo.example.com/CamelCase. --- The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)'
+                                        description: 'Error is to record the problem with the service port The format of the error shall comply with the following rules: - built-in error values shall be specified in this file and those shall use   CamelCase names - cloud provider specific error values must have names that comply with the   format foo.example.com/CamelCase. --- The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)'
                                         maxLength: 316
                                         pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
                                         type: string
@@ -1133,38 +1048,78 @@ spec:
               storage:
                 description: Storage Status contains information about Kubernetes storage system
                 properties:
-                  certificate:
+                  etcd:
+                    description: ETCDStatus defines the observed state of ETCDStatus.
                     properties:
-                      checksum:
-                        type: string
-                      lastUpdate:
-                        format: date-time
-                        type: string
-                      secretName:
-                        type: string
-                    type: object
-                  config:
-                    properties:
-                      checksum:
-                        type: string
-                      secretName:
-                        type: string
-                    type: object
-                  dataStoreName:
-                    type: string
-                  driver:
-                    type: string
-                  setup:
-                    properties:
-                      checksum:
-                        type: string
-                      lastUpdate:
-                        format: date-time
-                        type: string
-                      schema:
-                        type: string
+                      role:
+                        properties:
+                          exists:
+                            type: boolean
+                          name:
+                            type: string
+                          permissions:
+                            items:
+                              properties:
+                                key:
+                                  type: string
+                                rangeEnd:
+                                  type: string
+                                type:
+                                  type: integer
+                              type: object
+                            type: array
+                        required:
+                        - exists
+                        - name
+                        type: object
                       user:
+                        properties:
+                          exists:
+                            type: boolean
+                          name:
+                            type: string
+                          roles:
+                            items:
+                              type: string
+                            type: array
+                        required:
+                        - exists
+                        - name
+                        type: object
+                    type: object
+                  kine:
+                    properties:
+                      certificate:
+                        properties:
+                          checksum:
+                            type: string
+                          lastUpdate:
+                            format: date-time
+                            type: string
+                          secretName:
+                            type: string
+                        type: object
+                      config:
+                        properties:
+                          checksum:
+                            type: string
+                          secretName:
+                            type: string
+                        type: object
+                      driver:
                         type: string
+                      setup:
+                        properties:
+                          checksum:
+                            type: string
+                          lastUpdate:
+                            format: date-time
+                            type: string
+                          schema:
+                            type: string
+                          user:
+                            type: string
+                        type: object
                     type: object
                 type: object
             type: object
@@ -1173,6 +1128,12 @@ spec:
     storage: true
     subresources:
       status: {}
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []
 ---
 apiVersion: v1
 kind: ServiceAccount
@@ -1272,26 +1233,6 @@ rules:
   - patch
   - update
   - watch
-- apiGroups:
-  - kamaji.clastix.io
-  resources:
-  - datastores
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch
-- apiGroups:
-  - kamaji.clastix.io
-  resources:
-  - datastores/status
-  verbs:
-  - get
-  - patch
-  - update
 - apiGroups:
   - kamaji.clastix.io
   resources:
@@ -1497,39 +1438,3 @@ spec:
         runAsNonRoot: true
       serviceAccountName: kamaji-controller-manager
       terminationGracePeriodSeconds: 10
----
-apiVersion: kamaji.clastix.io/v1alpha1
-kind: DataStore
-metadata:
-  name: kamaji-etcd
-  namespace: kamaji-system
-spec:
-  basicAuth: null
-  driver: etcd
-  endpoints:
-  - etcd-0.etcd.kamaji-system.svc:2379
-  - etcd-1.etcd.kamaji-system.svc:2379
-  - etcd-2.etcd.kamaji-system.svc:2379
-  tlsConfig:
-    certificateAuthority:
-      certificate:
-        secretReference:
-          keyPath: ca.crt
-          name: etcd-certs
-          namespace: kamaji-system
-      privateKey:
-        secretReference:
-          keyPath: ca.key
-          name: etcd-certs
-          namespace: kamaji-system
-    clientCertificate:
-      certificate:
-        secretReference:
-          keyPath: tls.crt
-          name: root-client-certs
-          namespace: kamaji-system
-      privateKey:
-        secretReference:
-          keyPath: tls.key
-          name: root-client-certs
-          namespace: kamaji-system

config/rbac/datastore_editor_role.yaml

@@ -1,24 +0,0 @@
-# permissions for end users to edit datastores.
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: datastore-editor-role
-rules:
-- apiGroups:
-  - kamaji.clastix.io
-  resources:
-  - datastores
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch
-- apiGroups:
-  - kamaji.clastix.io
-  resources:
-  - datastores/status
-  verbs:
-  - get

config/rbac/datastore_viewer_role.yaml

@@ -1,20 +0,0 @@
-# permissions for end users to view datastores.
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: datastore-viewer-role
-rules:
-- apiGroups:
-  - kamaji.clastix.io
-  resources:
-  - datastores
-  verbs:
-  - get
-  - list
-  - watch
-- apiGroups:
-  - kamaji.clastix.io
-  resources:
-  - datastores/status
-  verbs:
-  - get

config/rbac/role.yaml

@@ -1,3 +1,4 @@
+
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
@@ -53,26 +54,6 @@ rules:
   - patch
   - update
   - watch
-- apiGroups:
-  - kamaji.clastix.io
-  resources:
-  - datastores
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch
-- apiGroups:
-  - kamaji.clastix.io
-  resources:
-  - datastores/status
-  verbs:
-  - get
-  - patch
-  - update
 - apiGroups:
   - kamaji.clastix.io
   resources:

config/samples/kamaji_v1alpha1_datastore_etcd.yaml

@@ -1,34 +0,0 @@
-apiVersion: kamaji.clastix.io/v1alpha1
-kind: DataStore
-metadata:
-  name: etcd
-spec:
-  driver: etcd
-  endpoints:
-    - etcd-0.etcd.kamaji-system.svc:2379
-    - etcd-1.etcd.kamaji-system.svc:2379
-    - etcd-2.etcd.kamaji-system.svc:2379
-  basicAuth: null
-  tlsConfig:
-    certificateAuthority:
-      certificate:
-        secretReference:
-          name: etcd-certs
-          namespace: kamaji-system
-          keyPath: "ca.crt"
-      privateKey:
-        secretReference:
-          name: etcd-certs
-          namespace: kamaji-system
-          keyPath: "ca.key"
-    clientCertificate:
-      certificate:
-        secretReference:
-          name: root-client-certs
-          namespace: kamaji-system
-          keyPath: "tls.crt"
-      privateKey:
-        secretReference:
-          name: root-client-certs
-          namespace: kamaji-system
-          keyPath: "tls.key"

config/samples/kamaji_v1alpha1_datastore_mysql.yaml

@@ -1,34 +0,0 @@
-apiVersion: kamaji.clastix.io/v1alpha1
-kind: DataStore
-metadata:
-  name: mysql
-spec:
-  driver: MySQL
-  endpoints:
-    - mariadb.kamaji-system.svc:3306
-  basicAuth:
-    username:
-      content: cm9vdA==
-    password:
-      secretReference:
-        name: mysql-config
-        namespace: kamaji-system
-        keyPath: MYSQL_ROOT_PASSWORD
-  tlsConfig:
-    certificateAuthority:
-      certificate:
-        secretReference:
-          name: mysql-config
-          namespace: kamaji-system
-          keyPath: "ca.crt"
-    clientCertificate:
-      certificate:
-        secretReference:
-          name: mysql-config
-          namespace: kamaji-system
-          keyPath: "server.crt"
-      privateKey:
-        secretReference:
-          name: mysql-config
-          namespace: kamaji-system
-          keyPath: "server.key"

config/samples/kamaji_v1alpha1_datastore_postgresql.yaml

@@ -1,37 +0,0 @@
-apiVersion: kamaji.clastix.io/v1alpha1
-kind: DataStore
-metadata:
-  name: postgresql
-spec:
-  driver: PostgreSQL
-  endpoints:
-    - postgresql-rw.kamaji-system.svc:5432
-  basicAuth:
-    username:
-      secretReference:
-        name: postgresql-superuser
-        namespace: kamaji-system
-        keyPath: username
-    password:
-      secretReference:
-        name: postgresql-superuser
-        namespace: kamaji-system
-        keyPath: password
-  tlsConfig:
-    certificateAuthority:
-      certificate:
-        secretReference:
-          name: postgresql-ca
-          namespace: kamaji-system
-          keyPath: ca.crt
-    clientCertificate:
-      certificate:
-        secretReference:
-          name: postgres-root-cert
-          namespace: kamaji-system
-          keyPath: tls.crt
-      privateKey:
-        secretReference:
-          name: postgres-root-cert
-          namespace: kamaji-system
-          keyPath: tls.key

config/samples/kustomization.yaml

@@ -1,4 +1,4 @@
 ## Append samples you want in your CSV to this file as resources ##
 resources:
-- kamaji_v1alpha1_datastore_etcd.yaml
+- kamaji_v1alpha1_tenantcontrolplane.yaml
 #+kubebuilder:scaffold:manifestskustomizesamples

controllers/datastore_controller.go

@@ -1,180 +0,0 @@
-// Copyright 2022 Clastix Labs
-// SPDX-License-Identifier: Apache-2.0
-
-package controllers
-
-import (
-	"context"
-	"fmt"
-
-	k8serrors "k8s.io/apimachinery/pkg/api/errors"
-	"k8s.io/apimachinery/pkg/fields"
-	k8stypes "k8s.io/apimachinery/pkg/types"
-	"k8s.io/apimachinery/pkg/util/sets"
-	"k8s.io/client-go/util/workqueue"
-	controllerruntime "sigs.k8s.io/controller-runtime"
-	"sigs.k8s.io/controller-runtime/pkg/builder"
-	"sigs.k8s.io/controller-runtime/pkg/client"
-	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
-	"sigs.k8s.io/controller-runtime/pkg/event"
-	"sigs.k8s.io/controller-runtime/pkg/handler"
-	"sigs.k8s.io/controller-runtime/pkg/log"
-	"sigs.k8s.io/controller-runtime/pkg/predicate"
-	"sigs.k8s.io/controller-runtime/pkg/reconcile"
-	"sigs.k8s.io/controller-runtime/pkg/source"
-
-	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
-	"github.com/clastix/kamaji/indexers"
-)
-
-const (
-	dataStoreFinalizer = "finalizer.kamaji.clastix.io/datastore"
-)
-
-type DataStore struct {
-	client client.Client
-	// TenantControlPlaneTrigger is the channel used to communicate across the controllers:
-	// if a Data Source is updated we have to be sure that the reconciliation of the certificates content
-	// for each Tenant Control Plane is put in place properly.
-	TenantControlPlaneTrigger TenantControlPlaneChannel
-}
-
-//+kubebuilder:rbac:groups=kamaji.clastix.io,resources=datastores,verbs=get;list;watch;create;update;patch;delete
-//+kubebuilder:rbac:groups=kamaji.clastix.io,resources=datastores/status,verbs=get;update;patch
-
-func (r *DataStore) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) {
-	log := log.FromContext(ctx)
-
-	ds := &kamajiv1alpha1.DataStore{}
-	if err := r.client.Get(ctx, request.NamespacedName, ds); err != nil {
-		if k8serrors.IsNotFound(err) {
-			return reconcile.Result{}, nil
-		}
-
-		log.Error(err, "unable to retrieve the request")
-
-		return reconcile.Result{}, err
-	}
-	// Managing the finalizer, required to don't drop a DataSource if this is still used by a Tenant Control Plane.
-	switch {
-	case ds.DeletionTimestamp != nil && controllerutil.ContainsFinalizer(ds, dataStoreFinalizer):
-		log.Info("marked for deletion, checking conditions")
-
-		if len(ds.Status.UsedBy) == 0 {
-			log.Info("resource is no more used by any Tenant Control Plane")
-
-			controllerutil.RemoveFinalizer(ds, dataStoreFinalizer)
-
-			return reconcile.Result{}, r.client.Update(ctx, ds)
-		}
-
-		log.Info("DataStore is still used by some Tenant Control Planes, cannot be removed")
-	case ds.DeletionTimestamp == nil && !controllerutil.ContainsFinalizer(ds, dataStoreFinalizer):
-		log.Info("the resource is missing the required finalizer, adding it")
-
-		controllerutil.AddFinalizer(ds, dataStoreFinalizer)
-
-		return reconcile.Result{}, r.client.Update(ctx, ds)
-	}
-	// A Data Source can trigger several Tenant Control Planes and requires a minimum validation:
-	// we have to ensure the data provided by the Data Source is valid and referencing an existing Secret object.
-	if _, err := ds.Spec.TLSConfig.CertificateAuthority.Certificate.GetContent(ctx, r.client); err != nil {
-		log.Error(err, "invalid Certificate Authority data")
-
-		return reconcile.Result{}, err
-	}
-
-	if ds.Spec.Driver == kamajiv1alpha1.EtcdDriver {
-		if ds.Spec.TLSConfig.CertificateAuthority.PrivateKey == nil {
-			err := fmt.Errorf("a valid private key is required for the etcd driver")
-
-			log.Error(err, "missing Certificate Authority private key data")
-
-			return reconcile.Result{}, err
-		}
-		if _, err := ds.Spec.TLSConfig.CertificateAuthority.PrivateKey.GetContent(ctx, r.client); err != nil {
-			log.Error(err, "invalid Certificate Authority private key data")
-
-			return reconcile.Result{}, err
-		}
-	}
-
-	if _, err := ds.Spec.TLSConfig.ClientCertificate.Certificate.GetContent(ctx, r.client); err != nil {
-		log.Error(err, "invalid Client Certificate data")
-
-		return reconcile.Result{}, err
-	}
-
-	if _, err := ds.Spec.TLSConfig.ClientCertificate.PrivateKey.GetContent(ctx, r.client); err != nil {
-		log.Error(err, "invalid Client Certificate private key data")
-
-		return reconcile.Result{}, err
-	}
-
-	tcpList := kamajiv1alpha1.TenantControlPlaneList{}
-
-	if err := r.client.List(ctx, &tcpList, client.MatchingFieldsSelector{
-		Selector: fields.OneTermEqualSelector(indexers.TenantControlPlaneUsedDataStoreKey, ds.GetName()),
-	}); err != nil {
-		log.Error(err, "cannot retrieve list of the Tenant Control Plane using the following instance")
-
-		return reconcile.Result{}, err
-	}
-	// Updating the status with the list of Tenant Control Plane using the following Data Source
-	tcpSets := sets.NewString()
-	for _, tcp := range tcpList.Items {
-		tcpSets.Insert(getNamespacedName(tcp.GetNamespace(), tcp.GetName()).String())
-	}
-
-	ds.Status.UsedBy = tcpSets.List()
-
-	if err := r.client.Status().Update(ctx, ds); err != nil {
-		log.Error(err, "cannot update the status for the given instance")
-
-		return reconcile.Result{}, err
-	}
-	// Triggering the reconciliation of the Tenant Control Plane upon a Secret change
-	for _, i := range tcpList.Items {
-		tcp := i
-
-		r.TenantControlPlaneTrigger <- event.GenericEvent{Object: &tcp}
-	}
-
-	return reconcile.Result{}, nil
-}
-
-func (r *DataStore) InjectClient(client client.Client) error {
-	r.client = client
-
-	return nil
-}
-
-func (r *DataStore) SetupWithManager(mgr controllerruntime.Manager) error {
-	enqueueFn := func(tcp *kamajiv1alpha1.TenantControlPlane, limitingInterface workqueue.RateLimitingInterface) {
-		if dataStoreName := tcp.Status.Storage.DataStoreName; len(dataStoreName) > 0 {
-			limitingInterface.AddRateLimited(reconcile.Request{
-				NamespacedName: k8stypes.NamespacedName{
-					Name: dataStoreName,
-				},
-			})
-		}
-	}
-	//nolint:forcetypeassert
-	return controllerruntime.NewControllerManagedBy(mgr).
-		For(&kamajiv1alpha1.DataStore{}, builder.WithPredicates(
-			predicate.ResourceVersionChangedPredicate{},
-		)).
-		Watches(&source.Kind{Type: &kamajiv1alpha1.TenantControlPlane{}}, handler.Funcs{
-			CreateFunc: func(createEvent event.CreateEvent, limitingInterface workqueue.RateLimitingInterface) {
-				enqueueFn(createEvent.Object.(*kamajiv1alpha1.TenantControlPlane), limitingInterface)
-			},
-			UpdateFunc: func(updateEvent event.UpdateEvent, limitingInterface workqueue.RateLimitingInterface) {
-				enqueueFn(updateEvent.ObjectOld.(*kamajiv1alpha1.TenantControlPlane), limitingInterface)
-				enqueueFn(updateEvent.ObjectNew.(*kamajiv1alpha1.TenantControlPlane), limitingInterface)
-			},
-			DeleteFunc: func(deleteEvent event.DeleteEvent, limitingInterface workqueue.RateLimitingInterface) {
-				enqueueFn(deleteEvent.Object.(*kamajiv1alpha1.TenantControlPlane), limitingInterface)
-			},
-		}).
-		Complete(r)
-}

controllers/resources.go

@@ -1,10 +1,10 @@
 // Copyright 2022 Clastix Labs
 // SPDX-License-Identifier: Apache-2.0
-
 package controllers
 
 import (
 	"fmt"
+	"strings"
 
 	"github.com/go-logr/logr"
 	"github.com/google/uuid"
@@ -12,10 +12,14 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
-	"github.com/clastix/kamaji/internal/datastore"
 	"github.com/clastix/kamaji/internal/resources"
-	ds "github.com/clastix/kamaji/internal/resources/datastore"
 	"github.com/clastix/kamaji/internal/resources/konnectivity"
+	"github.com/clastix/kamaji/internal/sql"
+	"github.com/clastix/kamaji/internal/types"
+)
+
+const (
+	separator = ","
 )
 
 type GroupResourceBuilderConfiguration struct {
@@ -23,8 +27,7 @@ type GroupResourceBuilderConfiguration struct {
 	log                 logr.Logger
 	tcpReconcilerConfig TenantControlPlaneReconcilerConfig
 	tenantControlPlane  kamajiv1alpha1.TenantControlPlane
-	Connection          datastore.Connection
-	DataStore           kamajiv1alpha1.DataStore
+	DBConnection        sql.DBConnection
 }
 
 type GroupDeleteableResourceBuilderConfiguration struct {
@@ -32,7 +35,7 @@ type GroupDeleteableResourceBuilderConfiguration struct {
 	log                 logr.Logger
 	tcpReconcilerConfig TenantControlPlaneReconcilerConfig
 	tenantControlPlane  kamajiv1alpha1.TenantControlPlane
-	connection          datastore.Connection
+	DBConnection        sql.DBConnection
 }
 
 // GetResources returns a list of resources that will be used to provide tenant control planes
@@ -50,39 +53,57 @@ func GetDeletableResources(config GroupDeleteableResourceBuilderConfiguration) [
 }
 
 func getDefaultResources(config GroupResourceBuilderConfiguration) []resources.Resource {
-	resources := append(getUpgradeResources(config.client), getKubernetesServiceResources(config.client)...)
-	resources = append(resources, getKubeadmConfigResources(config.client, getTmpDirectory(config.tcpReconcilerConfig.TmpBaseDirectory, config.tenantControlPlane), config.DataStore)...)
+	resources := append(getUpgradeResources(config.client, config.tenantControlPlane), getKubernetesServiceResources(config.client, config.tenantControlPlane)...)
+	resources = append(resources, getKubeadmConfigResources(config.client, config.tcpReconcilerConfig, config.tenantControlPlane)...)
 	resources = append(resources, getKubernetesCertificatesResources(config.client, config.log, config.tcpReconcilerConfig, config.tenantControlPlane)...)
 	resources = append(resources, getKubeconfigResources(config.client, config.log, config.tcpReconcilerConfig, config.tenantControlPlane)...)
-	resources = append(resources, getKubernetesStorageResources(config.client, config.Connection, config.DataStore)...)
-	resources = append(resources, getInternalKonnectivityResources(config.client, config.log)...)
-	resources = append(resources, getKubernetesDeploymentResources(config.client, config.tcpReconcilerConfig, config.DataStore)...)
-	resources = append(resources, getKubernetesIngressResources(config.client)...)
-	resources = append(resources, getKubeadmPhaseResources(config.client, config.log)...)
-	resources = append(resources, getKubeadmAddonResources(config.client, config.log)...)
-	resources = append(resources, getExternalKonnectivityResources(config.client)...)
+	resources = append(resources, getKubernetesStorageResources(config.client, config.log, config.tcpReconcilerConfig, config.DBConnection, config.tenantControlPlane)...)
+	resources = append(resources, getInternalKonnectivityResources(config.client, config.log, config.tcpReconcilerConfig, config.tenantControlPlane)...)
+	resources = append(resources, getKubernetesDeploymentResources(config.client, config.tcpReconcilerConfig, config.tenantControlPlane)...)
+	resources = append(resources, getKubernetesIngressResources(config.client, config.tenantControlPlane)...)
+	resources = append(resources, getKubeadmPhaseResources(config.client, config.log, config.tenantControlPlane)...)
+	resources = append(resources, getKubeadmAddonResources(config.client, config.log, config.tenantControlPlane)...)
+	resources = append(resources, getExternalKonnectivityResources(config.client, config.log, config.tcpReconcilerConfig, config.tenantControlPlane)...)
 
 	return resources
 }
 
 func getDefaultDeleteableResources(config GroupDeleteableResourceBuilderConfiguration) []resources.DeleteableResource {
-	return []resources.DeleteableResource{
-		&ds.Setup{
-			Client:     config.client,
-			Connection: config.connection,
-		},
+	switch config.tcpReconcilerConfig.ETCDStorageType {
+	case types.ETCD:
+		return []resources.DeleteableResource{
+			&resources.ETCDSetupResource{
+				Name:                  "etcd-setup",
+				Client:                config.client,
+				Log:                   config.log,
+				ETCDClientCertsSecret: getNamespacedName(config.tcpReconcilerConfig.ETCDClientSecretNamespace, config.tcpReconcilerConfig.ETCDClientSecretName),
+				ETCDCACertsSecret:     getNamespacedName(config.tcpReconcilerConfig.ETCDCASecretNamespace, config.tcpReconcilerConfig.ETCDCASecretName),
+				Endpoints:             getArrayFromString(config.tcpReconcilerConfig.ETCDEndpoints),
+			},
+		}
+	case types.KineMySQL, types.KinePostgreSQL:
+		return []resources.DeleteableResource{
+			&resources.SQLSetup{
+				Client:       config.client,
+				Name:         "sql-setup",
+				DBConnection: config.DBConnection,
+			},
+		}
+	default:
+		return []resources.DeleteableResource{}
 	}
 }
 
-func getUpgradeResources(c client.Client) []resources.Resource {
+func getUpgradeResources(c client.Client, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource {
 	return []resources.Resource{
 		&resources.KubernetesUpgrade{
+			Name:   "upgrade",
 			Client: c,
 		},
 	}
 }
 
-func getKubernetesServiceResources(c client.Client) []resources.Resource {
+func getKubernetesServiceResources(c client.Client, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource {
 	return []resources.Resource{
 		&resources.KubernetesServiceResource{
 			Client: c,
@@ -90,21 +111,14 @@ func getKubernetesServiceResources(c client.Client) []resources.Resource {
 	}
 }
 
-func getKubeadmConfigResources(c client.Client, tmpDirectory string, dataStore kamajiv1alpha1.DataStore) []resources.Resource {
-	var endpoints []string
-
-	switch dataStore.Spec.Driver {
-	case kamajiv1alpha1.EtcdDriver:
-		endpoints = dataStore.Spec.Endpoints
-	default:
-		endpoints = []string{"127.0.0.1:2379"}
-	}
-
+func getKubeadmConfigResources(c client.Client, tcpReconcilerConfig TenantControlPlaneReconcilerConfig, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource {
 	return []resources.Resource{
 		&resources.KubeadmConfigResource{
-			ETCDs:        endpoints,
-			Client:       c,
-			TmpDirectory: tmpDirectory,
+			Name:                   "kubeadmconfig",
+			ETCDs:                  getArrayFromString(tcpReconcilerConfig.ETCDEndpoints),
+			ETCDCompactionInterval: tcpReconcilerConfig.ETCDCompactionInterval,
+			Client:                 c,
+			TmpDirectory:           getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
 		},
 	}
 }
@@ -112,31 +126,37 @@ func getKubeadmConfigResources(c client.Client, tmpDirectory string, dataStore k
 func getKubernetesCertificatesResources(c client.Client, log logr.Logger, tcpReconcilerConfig TenantControlPlaneReconcilerConfig, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource {
 	return []resources.Resource{
 		&resources.CACertificate{
+			Name:         "ca",
 			Client:       c,
 			Log:          log,
 			TmpDirectory: getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
 		},
 		&resources.FrontProxyCACertificate{
+			Name:         "front-proxy-ca-certificate",
 			Client:       c,
 			Log:          log,
 			TmpDirectory: getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
 		},
 		&resources.SACertificate{
+			Name:         "sa-certificate",
 			Client:       c,
 			Log:          log,
 			TmpDirectory: getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
 		},
 		&resources.APIServerCertificate{
+			Name:         "api-server-certificate",
 			Client:       c,
 			Log:          log,
 			TmpDirectory: getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
 		},
 		&resources.APIServerKubeletClientCertificate{
+			Name:         "api-server-kubelet-client-certificate",
 			Client:       c,
 			Log:          log,
 			TmpDirectory: getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
 		},
 		&resources.FrontProxyClientCertificate{
+			Name:         "front-proxy-client-certificate",
 			Client:       c,
 			Log:          log,
 			TmpDirectory: getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
@@ -170,36 +190,72 @@ func getKubeconfigResources(c client.Client, log logr.Logger, tcpReconcilerConfi
 	}
 }
 
-func getKubernetesStorageResources(c client.Client, dbConnection datastore.Connection, datastore kamajiv1alpha1.DataStore) []resources.Resource {
-	return []resources.Resource{
-		&ds.Config{
-			Client:     c,
-			ConnString: dbConnection.GetConnectionString(),
-			DataStore:  datastore,
-		},
-		&ds.Setup{
-			Client:     c,
-			Connection: dbConnection,
-			DataStore:  datastore,
-		},
-		&ds.Certificate{
-			Client:    c,
-			DataStore: datastore,
-		},
+func getKubernetesStorageResources(c client.Client, log logr.Logger, tcpReconcilerConfig TenantControlPlaneReconcilerConfig, dbConnection sql.DBConnection, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource {
+	switch tcpReconcilerConfig.ETCDStorageType {
+	case types.ETCD:
+		return []resources.Resource{
+			&resources.ETCDCACertificatesResource{
+				Name:                  "etcd-ca-certificates",
+				Client:                c,
+				Log:                   log,
+				ETCDCASecretName:      tcpReconcilerConfig.ETCDCASecretName,
+				ETCDCASecretNamespace: tcpReconcilerConfig.ETCDCASecretNamespace,
+			},
+			&resources.ETCDCertificatesResource{
+				Name:   "etcd-certificates",
+				Client: c,
+				Log:    log,
+			},
+			&resources.ETCDSetupResource{
+				Name:                  "etcd-setup",
+				Client:                c,
+				Log:                   log,
+				ETCDClientCertsSecret: getNamespacedName(tcpReconcilerConfig.ETCDClientSecretNamespace, tcpReconcilerConfig.ETCDClientSecretName),
+				ETCDCACertsSecret:     getNamespacedName(tcpReconcilerConfig.ETCDCASecretNamespace, tcpReconcilerConfig.ETCDCASecretName),
+				Endpoints:             getArrayFromString(tcpReconcilerConfig.ETCDEndpoints),
+			},
+		}
+	case types.KineMySQL, types.KinePostgreSQL:
+		return []resources.Resource{
+			&resources.SQLStorageConfig{
+				Client: c,
+				Name:   "sql-config",
+				Host:   dbConnection.GetHost(),
+				Port:   dbConnection.GetPort(),
+				Driver: dbConnection.Driver(),
+			},
+			&resources.SQLSetup{
+				Client:       c,
+				Name:         "sql-setup",
+				DBConnection: dbConnection,
+				Driver:       dbConnection.Driver(),
+			},
+			&resources.SQLCertificate{
+				Client:                   c,
+				Name:                     "sql-certificate",
+				StorageType:              tcpReconcilerConfig.ETCDStorageType,
+				SQLConfigSecretName:      tcpReconcilerConfig.KineSecretName,
+				SQLConfigSecretNamespace: tcpReconcilerConfig.KineSecretNamespace,
+			},
+		}
+	default:
+		return []resources.Resource{}
 	}
 }
 
-func getKubernetesDeploymentResources(c client.Client, tcpReconcilerConfig TenantControlPlaneReconcilerConfig, dataStore kamajiv1alpha1.DataStore) []resources.Resource {
+func getKubernetesDeploymentResources(c client.Client, tcpReconcilerConfig TenantControlPlaneReconcilerConfig, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource {
 	return []resources.Resource{
 		&resources.KubernetesDeploymentResource{
-			Client:             c,
-			DataStore:          dataStore,
-			KineContainerImage: tcpReconcilerConfig.KineContainerImage,
+			Client:                 c,
+			ETCDEndpoints:          getArrayFromString(tcpReconcilerConfig.ETCDEndpoints),
+			ETCDCompactionInterval: tcpReconcilerConfig.ETCDCompactionInterval,
+			ETCDStorageType:        tcpReconcilerConfig.ETCDStorageType,
+			KineContainerImage:     tcpReconcilerConfig.KineContainerImage,
 		},
 	}
 }
 
-func getKubernetesIngressResources(c client.Client) []resources.Resource {
+func getKubernetesIngressResources(c client.Client, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource {
 	return []resources.Resource{
 		&resources.KubernetesIngressResource{
 			Client: c,
@@ -207,7 +263,7 @@ func getKubernetesIngressResources(c client.Client) []resources.Resource {
 	}
 }
 
-func getKubeadmPhaseResources(c client.Client, log logr.Logger) []resources.Resource {
+func getKubeadmPhaseResources(c client.Client, log logr.Logger, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource {
 	return []resources.Resource{
 		&resources.KubeadmPhase{
 			Name:   "upload-config-kubeadm",
@@ -230,7 +286,7 @@ func getKubeadmPhaseResources(c client.Client, log logr.Logger) []resources.Reso
 	}
 }
 
-func getKubeadmAddonResources(c client.Client, log logr.Logger) []resources.Resource {
+func getKubeadmAddonResources(c client.Client, log logr.Logger, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource {
 	return []resources.Resource{
 		&resources.KubeadmAddonResource{
 			Name:         "coredns",
@@ -247,7 +303,7 @@ func getKubeadmAddonResources(c client.Client, log logr.Logger) []resources.Reso
 	}
 }
 
-func getExternalKonnectivityResources(c client.Client) []resources.Resource {
+func getExternalKonnectivityResources(c client.Client, log logr.Logger, tcpReconcilerConfig TenantControlPlaneReconcilerConfig, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource {
 	return []resources.Resource{
 		&konnectivity.ServiceAccountResource{
 			Client: c,
@@ -272,7 +328,7 @@ func getExternalKonnectivityResources(c client.Client) []resources.Resource {
 	}
 }
 
-func getInternalKonnectivityResources(c client.Client, log logr.Logger) []resources.Resource {
+func getInternalKonnectivityResources(c client.Client, log logr.Logger, tcpReconcilerConfig TenantControlPlaneReconcilerConfig, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource {
 	return []resources.Resource{
 		&konnectivity.EgressSelectorConfigurationResource{
 			Client: c,
@@ -290,6 +346,13 @@ func getInternalKonnectivityResources(c client.Client, log logr.Logger) []resour
 	}
 }
 
+func getArrayFromString(s string) []string {
+	var a []string
+	a = append(a, strings.Split(s, separator)...)
+
+	return a
+}
+
 func getNamespacedName(namespace string, name string) k8stypes.NamespacedName {
 	return k8stypes.NamespacedName{Namespace: namespace, Name: name}
 }

controllers/storage.go

@@ -8,97 +8,78 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"fmt"
-	"net"
-	"strconv"
+	"strings"
 
 	"github.com/pkg/errors"
+	corev1 "k8s.io/api/core/v1"
+	k8stypes "k8s.io/apimachinery/pkg/types"
 
-	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
-	"github.com/clastix/kamaji/internal/datastore"
+	"github.com/clastix/kamaji/internal/sql"
+	"github.com/clastix/kamaji/internal/types"
 )
 
-func (r *TenantControlPlaneReconciler) getStorageConnection(ctx context.Context, ds kamajiv1alpha1.DataStore) (datastore.Connection, error) {
-	ca, err := ds.Spec.TLSConfig.CertificateAuthority.Certificate.GetContent(ctx, r.Client)
-	if err != nil {
+func (r *TenantControlPlaneReconciler) getStorageConnection(ctx context.Context) (sql.DBConnection, error) {
+	var driver sql.Driver
+	var dbName string
+
+	// TODO: https://github.com/clastix/kamaji/issues/67
+	switch r.Config.ETCDStorageType {
+	case types.ETCD:
+		return nil, nil
+	case types.KineMySQL:
+		driver = sql.MySQL
+		dbName = "mysql"
+	case types.KinePostgreSQL:
+		driver = sql.PostgreSQL
+	default:
+		return nil, nil
+	}
+
+	secret := &corev1.Secret{}
+	namespacedName := k8stypes.NamespacedName{Namespace: r.Config.KineSecretNamespace, Name: r.Config.KineSecretName}
+	if err := r.Client.Get(ctx, namespacedName, secret); err != nil {
 		return nil, err
 	}
 
-	crt, err := ds.Spec.TLSConfig.ClientCertificate.Certificate.GetContent(ctx, r.Client)
-	if err != nil {
-		return nil, err
+	if t := "kamaji.clastix.io/kine"; string(secret.Type) != t {
+		return nil, fmt.Errorf("expecting a secret of type %s", t)
 	}
 
-	key, err := ds.Spec.TLSConfig.ClientCertificate.PrivateKey.GetContent(ctx, r.Client)
-	if err != nil {
-		return nil, err
+	keys := []string{"ca.crt", "server.crt", "server.key", "username", "password"}
+
+	if secret.Data == nil {
+		return nil, fmt.Errorf("the Kine secret %s/%s is missing all the required keys (%s)", secret.GetNamespace(), secret.GetName(), strings.Join(keys, ","))
+	}
+
+	for _, key := range keys {
+		if _, ok := secret.Data[key]; !ok {
+			return nil, fmt.Errorf("missing required key %s for the Kine secret %s/%s", key, secret.GetNamespace(), secret.GetName())
+		}
 	}
 
 	rootCAs := x509.NewCertPool()
-	if ok := rootCAs.AppendCertsFromPEM(ca); !ok {
+	if ok := rootCAs.AppendCertsFromPEM(secret.Data["ca.crt"]); !ok {
 		return nil, fmt.Errorf("error create root CA for the DB connector")
 	}
 
-	certificate, err := tls.X509KeyPair(crt, key)
+	certificate, err := tls.X509KeyPair(secret.Data["server.crt"], secret.Data["server.key"])
 	if err != nil {
 		return nil, errors.Wrap(err, "cannot retrieve x.509 key pair from the Kine Secret")
 	}
 
-	var user, password string
-	if auth := ds.Spec.BasicAuth; auth != nil {
-		u, err := auth.Username.GetContent(ctx, r.Client)
-		if err != nil {
-			return nil, err
-		}
-		user = string(u)
-
-		p, err := auth.Password.GetContent(ctx, r.Client)
-		if err != nil {
-			return nil, err
-		}
-		password = string(p)
-	}
-
-	eps := make([]datastore.ConnectionEndpoint, 0, len(ds.Spec.Endpoints))
-
-	for _, ep := range ds.Spec.Endpoints {
-		host, stringPort, err := net.SplitHostPort(ep)
-		if err != nil {
-			return nil, errors.Wrap(err, "cannot retrieve host-port pair from DataStore endpoints")
-		}
-
-		port, err := strconv.Atoi(stringPort)
-		if err != nil {
-			return nil, errors.Wrap(err, "cannot convert port from string for the given DataStore")
-		}
-
-		eps = append(eps, datastore.ConnectionEndpoint{
-			Host: host,
-			Port: port,
-		})
-	}
-
-	cc := datastore.ConnectionConfig{
-		User:      user,
-		Password:  password,
-		Endpoints: eps,
-		TLSConfig: &tls.Config{
-			RootCAs:      rootCAs,
-			Certificates: []tls.Certificate{certificate},
+	return sql.GetDBConnection(
+		sql.ConnectionConfig{
+			SQLDriver: driver,
+			User:      string(secret.Data["username"]),
+			Password:  string(secret.Data["password"]),
+			Host:      r.Config.KineHost,
+			Port:      r.Config.KinePort,
+			DBName:    dbName,
+			TLSConfig: &tls.Config{
+				ServerName:   r.Config.KineHost,
+				RootCAs:      rootCAs,
+				Certificates: []tls.Certificate{certificate},
+			},
 		},
-	}
-
-	switch ds.Spec.Driver {
-	case kamajiv1alpha1.KineMySQLDriver:
-		cc.TLSConfig.ServerName = cc.Endpoints[0].Host
-
-		return datastore.NewMySQLConnection(cc)
-	case kamajiv1alpha1.KinePostgreSQLDriver:
-		cc.TLSConfig.ServerName = cc.Endpoints[0].Host
-		//nolint:contextcheck
-		return datastore.NewPostgreSQLConnection(cc)
-	case kamajiv1alpha1.EtcdDriver:
-		return datastore.NewETCDConnection(cc)
-	default:
-		return nil, fmt.Errorf("%s is not a valid driver", ds.Spec.Driver)
-	}
+	)
 }

controllers/tcp_channel.go

@@ -1,8 +0,0 @@
-// Copyright 2022 Clastix Labs
-// SPDX-License-Identifier: Apache-2.0
-
-package controllers
-
-import "sigs.k8s.io/controller-runtime/pkg/event"
-
-type TenantControlPlaneChannel chan event.GenericEvent

controllers/tenantcontrolplane_controller.go

@@ -7,54 +7,61 @@ import (
 	"context"
 	"fmt"
 
-	"github.com/pkg/errors"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	networkingv1 "k8s.io/api/networking/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime"
 	k8stypes "k8s.io/apimachinery/pkg/types"
-	"k8s.io/client-go/util/workqueue"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
-	"sigs.k8s.io/controller-runtime/pkg/event"
-	"sigs.k8s.io/controller-runtime/pkg/handler"
 	"sigs.k8s.io/controller-runtime/pkg/log"
-	"sigs.k8s.io/controller-runtime/pkg/source"
 
 	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
 	kamajierrors "github.com/clastix/kamaji/internal/errors"
 	"github.com/clastix/kamaji/internal/resources"
+	"github.com/clastix/kamaji/internal/sql"
+	"github.com/clastix/kamaji/internal/types"
 )
 
 const (
-	tenantControlPlaneFinalizer = "finalizer.kamaji.clastix.io"
+	finalizer = "finalizer.kamaji.clastix.io"
 )
 
 // TenantControlPlaneReconciler reconciles a TenantControlPlane object.
 type TenantControlPlaneReconciler struct {
 	client.Client
-	Scheme      *runtime.Scheme
-	Config      TenantControlPlaneReconcilerConfig
-	TriggerChan TenantControlPlaneChannel
+	Scheme *runtime.Scheme
+	Config TenantControlPlaneReconcilerConfig
 }
 
 // TenantControlPlaneReconcilerConfig gives the necessary configuration for TenantControlPlaneReconciler.
 type TenantControlPlaneReconcilerConfig struct {
-	DefaultDataStoreName string
-	KineContainerImage   string
-	TmpBaseDirectory     string
+	ETCDStorageType           types.ETCDStorageType
+	ETCDCASecretName          string
+	ETCDCASecretNamespace     string
+	ETCDClientSecretName      string
+	ETCDClientSecretNamespace string
+	ETCDEndpoints             string
+	ETCDCompactionInterval    string
+	TmpBaseDirectory          string
+	DBConnection              sql.DBConnection
+	KineSecretName            string
+	KineSecretNamespace       string
+	KineHost                  string
+	KinePort                  int
+	KineContainerImage        string
 }
 
 //+kubebuilder:rbac:groups=kamaji.clastix.io,resources=tenantcontrolplanes,verbs=get;list;watch;create;update;patch;delete
 //+kubebuilder:rbac:groups=kamaji.clastix.io,resources=tenantcontrolplanes/status,verbs=get;update;patch
 //+kubebuilder:rbac:groups=kamaji.clastix.io,resources=tenantcontrolplanes/finalizers,verbs=update
-//+kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch;create;update;patch;delete
-//+kubebuilder:rbac:groups=core,resources=configmaps,verbs=get;list;watch;create;update;patch;delete
-//+kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;update;patch;delete
-//+kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete
-//+kubebuilder:rbac:groups=networking.k8s.io,resources=ingresses,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=core,resources=configmaps,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=networking.k8s.io,resources=ingresses,verbs=get;list;watch;create;update;patch;delete
 
 func (r *TenantControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
 	log := log.FromContext(ctx)
@@ -62,8 +69,6 @@ func (r *TenantControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R
 	tenantControlPlane := &kamajiv1alpha1.TenantControlPlane{}
 	isTenantControlPlane, err := r.getTenantControlPlane(ctx, req.NamespacedName, tenantControlPlane)
 	if err != nil {
-		log.Error(err, "cannot retrieve the required instance")
-
 		return ctrl.Result{}, err
 	}
 	if !isTenantControlPlane {
@@ -71,26 +76,23 @@ func (r *TenantControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R
 	}
 
 	markedToBeDeleted := tenantControlPlane.GetDeletionTimestamp() != nil
-	hasFinalizer := controllerutil.ContainsFinalizer(tenantControlPlane, tenantControlPlaneFinalizer)
+	hasFinalizer := hasFinalizer(*tenantControlPlane)
 
 	if markedToBeDeleted && !hasFinalizer {
 		return ctrl.Result{}, nil
 	}
-	// Retrieving the DataStore to use for the current reconciliation
-	ds, err := r.dataStore(ctx, tenantControlPlane)
-	if err != nil {
-		log.Error(err, "cannot retrieve the DataStore for the given instance")
 
+	dbConnection, err := r.getStorageConnection(ctx)
+	if err != nil {
 		return ctrl.Result{}, err
 	}
-
-	dsConnection, err := r.getStorageConnection(ctx, *ds)
-	if err != nil {
-		log.Error(err, "cannot generate the DataStore connection for the given instance")
-
-		return ctrl.Result{}, err
-	}
-	defer dsConnection.Close()
+	defer func() {
+		// TODO: Currently, etcd is not accessed using this dbConnection. For that reason we need this check
+		// Check: https://github.com/clastix/kamaji/issues/67
+		if dbConnection != nil {
+			dbConnection.Close()
+		}
+	}()
 
 	if markedToBeDeleted {
 		log.Info("marked for deletion, performing clean-up")
@@ -100,14 +102,12 @@ func (r *TenantControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R
 			log:                 log,
 			tcpReconcilerConfig: r.Config,
 			tenantControlPlane:  *tenantControlPlane,
-			connection:          dsConnection,
+			DBConnection:        dbConnection,
 		}
 		registeredDeletableResources := GetDeletableResources(groupDeleteableResourceBuilderConfiguration)
 
 		for _, resource := range registeredDeletableResources {
-			if err = resources.HandleDeletion(ctx, resource, tenantControlPlane); err != nil {
-				log.Error(err, "resource deletion failed", "resource", resource.GetName())
-
+			if err := resources.HandleDeletion(ctx, resource, tenantControlPlane); err != nil {
 				return ctrl.Result{}, err
 			}
 		}
@@ -115,9 +115,7 @@ func (r *TenantControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R
 		if hasFinalizer {
 			log.Info("removing finalizer")
 
-			if err = r.RemoveFinalizer(ctx, tenantControlPlane); err != nil {
-				log.Error(err, "cannot remove the finalizer for the given resource")
-
+			if err := r.RemoveFinalizer(ctx, tenantControlPlane); err != nil {
 				return ctrl.Result{}, err
 			}
 		}
@@ -136,8 +134,7 @@ func (r *TenantControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R
 		log:                 log,
 		tcpReconcilerConfig: r.Config,
 		tenantControlPlane:  *tenantControlPlane,
-		DataStore:           *ds,
-		Connection:          dsConnection,
+		DBConnection:        dbConnection,
 	}
 	registeredResources := GetResources(groupResourceBuilderConfiguration)
 
@@ -150,8 +147,6 @@ func (r *TenantControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R
 				return ctrl.Result{Requeue: true}, nil
 			}
 
-			log.Error(err, "handling of resource failed", "resource", resource.GetName())
-
 			return ctrl.Result{}, err
 		}
 
@@ -160,8 +155,6 @@ func (r *TenantControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R
 		}
 
 		if err := r.updateStatus(ctx, req.NamespacedName, resource); err != nil {
-			log.Error(err, "update of the resource failed", "resource", resource.GetName())
-
 			return ctrl.Result{}, err
 		}
 
@@ -178,14 +171,6 @@ func (r *TenantControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R
 // SetupWithManager sets up the controller with the Manager.
 func (r *TenantControlPlaneReconciler) SetupWithManager(mgr ctrl.Manager) error {
 	return ctrl.NewControllerManagedBy(mgr).
-		Watches(&source.Channel{Source: r.TriggerChan}, handler.Funcs{GenericFunc: func(genericEvent event.GenericEvent, limitingInterface workqueue.RateLimitingInterface) {
-			limitingInterface.AddRateLimited(ctrl.Request{
-				NamespacedName: k8stypes.NamespacedName{
-					Namespace: genericEvent.Object.GetNamespace(),
-					Name:      genericEvent.Object.GetName(),
-				},
-			})
-		}}).
 		For(&kamajiv1alpha1.TenantControlPlane{}).
 		Owns(&corev1.Secret{}).
 		Owns(&corev1.ConfigMap{}).
@@ -229,34 +214,24 @@ func (r *TenantControlPlaneReconciler) updateStatus(ctx context.Context, namespa
 	return nil
 }
 
+func hasFinalizer(tenantControlPlane kamajiv1alpha1.TenantControlPlane) bool {
+	for _, f := range tenantControlPlane.GetFinalizers() {
+		if f == finalizer {
+			return true
+		}
+	}
+
+	return false
+}
+
 func (r *TenantControlPlaneReconciler) AddFinalizer(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
-	controllerutil.AddFinalizer(tenantControlPlane, tenantControlPlaneFinalizer)
+	controllerutil.AddFinalizer(tenantControlPlane, finalizer)
 
 	return r.Update(ctx, tenantControlPlane)
 }
 
 func (r *TenantControlPlaneReconciler) RemoveFinalizer(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
-	controllerutil.RemoveFinalizer(tenantControlPlane, tenantControlPlaneFinalizer)
+	controllerutil.RemoveFinalizer(tenantControlPlane, finalizer)
 
 	return r.Update(ctx, tenantControlPlane)
 }
-
-// dataStore retrieves the override DataStore for the given Tenant Control Plane if specified,
-// otherwise fallback to the default one specified in the Kamaji setup.
-func (r *TenantControlPlaneReconciler) dataStore(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (*kamajiv1alpha1.DataStore, error) {
-	dataStoreName := tenantControlPlane.Spec.DataStore
-	if len(dataStoreName) == 0 {
-		dataStoreName = r.Config.DefaultDataStoreName
-	}
-
-	if statusDataStore := tenantControlPlane.Status.Storage.DataStoreName; len(statusDataStore) > 0 && dataStoreName != statusDataStore {
-		return nil, fmt.Errorf("migration from a DataStore (current: %s) to another one (desired: %s) is not yet supported", statusDataStore, dataStoreName)
-	}
-
-	ds := &kamajiv1alpha1.DataStore{}
-	if err := r.Client.Get(ctx, k8stypes.NamespacedName{Name: dataStoreName}, ds); err != nil {
-		return nil, errors.Wrap(err, "cannot retrieve *kamajiv1alpha.DataStore object")
-	}
-
-	return ds, nil
-}

deploy/kine/README.md

@@ -7,20 +7,41 @@ This can help in overcoming the `etcd` limitation regarding scalability and clus
 
 ## Kamaji additional CLI flags
 
-Kamaji read the data store configuration from a cluster-scoped resource named `DataStore`, containing all tha required details to secure a connection using a specific driver. 
-
-- [Example of a `etcd` DataStore](./../../config/samples/kamaji_v1alpha1_datastore_etcd.yaml)
-- [Example of a `MySQL` DataStore](./../../config/samples/kamaji_v1alpha1_datastore_mysql.yaml)
-- [Example of a `PostgreSQL` DataStore](./../../config/samples/kamaji_v1alpha1_datastore_postgresql.yaml)
-
-Once the datastore is running, and the `DataStore` has been created with the required details, we need to provide information about it to Kamaji by using the following flag and pointing to the resource name:
+Once a compatible database is running, we need to provide information about it to Kamaji by using the following flags:
 
 ```
---datastore={.metadata.name}
+--etcd-storage-type={kine-mysql,kine-postgresql}
+--kine-host=<database host>
+--kine-port=<database port>
+--kine-secret-name=<secret name>
+--kine-secret-namespace=<secret namespace>
 ```
 
+## Kine Secret
+
+The Kine Secret must be configured as follows:
+
+```yaml
+apiVersion: v1
+data:
+  ca.crt: "content of the Certificate Authority for SSL connection"
+  password: "password of the super user"
+  server.crt: "content of the certificate for SSL connection"
+  server.key: "content of the private key for SSL connection"
+  username: "username of the super user"
+kind: Secret
+metadata:
+  name: kine-secret
+  namespace: kamaji-system
+type: kamaji.clastix.io/kine
+```
+
+> Please, pay attention to the type `kamaji.clastix.io/kine`: this check is enforced at the code level to ensure the required data is provided.
+
+> Actually, the `kine` integration expects a secured connection to the database since the sensitivity data of the Tenant.
+
 ## Drivers
 
 Further details on the setup for each driver are available here:
 - [MySQL/MariaDB](../deploy/kine/mysql/README.md)
-- [PostgreSQL](../deploy/kine/postgresql/README.md)
+- [PostgreSQL](../deploy/kine/postgresql/README.md)

deploy/kine/mysql/Makefile

@@ -1,6 +1,6 @@
 ROOT_DIR:=$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))
 
-mariadb: mariadb-certificates mariadb-secret mariadb-deployment
+mariadb: mariadb-certificates mariadb-secret mariadb-kine-secret mariadb-deployment
 
 mariadb-certificates:
 	rm -rf $(ROOT_DIR)/certs && mkdir $(ROOT_DIR)/certs
@@ -22,6 +22,15 @@ mariadb-secret:
 		--from-literal=MYSQL_ROOT_PASSWORD=root \
 		--dry-run=client -o yaml | kubectl apply -f -
 
+mariadb-kine-secret:
+	@\
+		CA=$$(cat $(ROOT_DIR)/certs/ca.crt | base64 | tr -d '\n') \
+		CRT=$$(cat $(ROOT_DIR)/certs/server.crt | base64 | tr -d '\n') \
+		KEY=$$(cat $(ROOT_DIR)/certs/server.key | base64 | tr -d '\n') \
+		ROOT_USERNAME=$$(echo -n root | base64) \
+		ROOT_PASSWORD=$$(kubectl -n kamaji-system get secret mysql-config -o jsonpath='{.data.MYSQL_ROOT_PASSWORD}') \
+		envsubst < $(ROOT_DIR)/../secret.yaml | kubectl -n kamaji-system apply -f -
+
 mariadb-deployment:
 	@kubectl -n kamaji-system apply -f $(ROOT_DIR)/mariadb.yaml
 

deploy/kine/mysql/README.md

@@ -59,6 +59,14 @@ $ make mariadb-secrets
 Previous certificates and MySQL configuration have to be available in order to be used.
 They will be under the secret `kamaji-system:mysql-config`, used by the MySQL/MariaDB instance.
 
+## Kine Secret
+
+```bash
+$ make mariadb-kine-secret
+```
+
+Organize the required Kine data such as username, password, CA, certificate, and private key to be stored in the Kamaji desired format.
+
 ## Deployment
 
 ```bash

deploy/kine/postgresql/Makefile

@@ -1,6 +1,6 @@
 ROOT_DIR:=$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))
 
-postgresql: cnpg-setup cnpg-deploy postgresql-secret
+postgresql: cnpg-setup cnpg-deploy postgresql-secret postgresql-kine-secret
 
 cnpg-setup:
 	@kubectl apply -f https://raw.githubusercontent.com/cloudnative-pg/cloudnative-pg/main/releases/cnpg-1.16.0.yaml
@@ -21,6 +21,15 @@ postgresql-secret: cnpg
     		--cnpg-cluster postgresql \
     		--cnpg-user $$(kubectl -n kamaji-system get secret postgresql-superuser -o jsonpath='{.data.username}' | base64 -d)
 
+postgresql-kine-secret:
+	@\
+		CA=$$(kubectl -n kamaji-system get secret postgresql-ca -o jsonpath='{.data.ca\.crt}') \
+		CRT=$$(kubectl -n kamaji-system get secret postgres-root-cert -o jsonpath='{.data.tls\.crt}') \
+		KEY=$$(kubectl -n kamaji-system get secret postgres-root-cert -o jsonpath='{.data.tls\.key}') \
+		ROOT_USERNAME=$$(kubectl -n kamaji-system get secret postgresql-superuser -o jsonpath='{.data.username}' ) \
+		ROOT_PASSWORD=$$(kubectl -n kamaji-system get secret postgresql-superuser -o jsonpath='{.data.password}' ) \
+		envsubst < $(ROOT_DIR)/../secret.yaml | kubectl -n kamaji-system apply -f -
+
 postgresql-destroy:
 	@kubectl delete -f https://raw.githubusercontent.com/cloudnative-pg/cloudnative-pg/main/releases/cnpg-1.16.0.yaml --ignore-not-found && \
 	kubectl delete secret postgres-root-cert --ignore-not-found && \

deploy/kine/postgresql/README.md

@@ -1,7 +1,6 @@
 # PostgreSQL as Kubernetes Storage
 
-Kamaji offers the possibility of having a different storage system than `etcd` thanks to [kine](https://github.com/k3s-io/kine).
-One of the implementations is [PostgreSQL](https://www.postgresql.org/).
+Kamaji offers the possibility of having a different storage system than `etcd` thanks to [kine](https://github.com/k3s-io/kine). One of the implementations is [PostgreSQL](https://www.postgresql.org/).
 
 Kamaji project is developed using [kind](https://kind.sigs.k8s.io), therefore, a PostgreSQL instance must be deployed in advance into the local kubernetes cluster in order to be used as storage for the tenants.
 For the sake of simplicity, the [cloudnative-pg](https://cloudnative-pg.io/) Operator will be used to simplify the setup of it.
@@ -35,6 +34,7 @@ validatingwebhookconfiguration.admissionregistration.k8s.io/cnpg-validating-webh
 deployment "cnpg-controller-manager" successfully rolled out
 cluster.postgresql.cnpg.io/postgresql unchanged
 secret/postgres-root-cert created
+secret/kine-secret created
 ```
 
 ## Operator setup
@@ -55,13 +55,15 @@ $ make postgresql-secret
 
 This target will download locally the `kubectl-cnpg` utility to generate an SSL certificate required to secure the connection to the PostgreSQL instance.
 
-## Certificate generation
+## Kine Secret generation
 
 ```bash
-$ make postgresql-secret
+$ make postgresql-kine-secret
 ```
 
-Generate the Certificate required to connect to the DataStore.
+Generate the Kine secret required for Kamaji.
+
+> Requires the generation of the `postgresql-secret`
 
 ## Teardown
 

deploy/kine/secret.yaml

@@ -0,0 +1,14 @@
+# secret.yaml is the Secret object that Kamaji is expecting to user to connect to the Kine SQL datastore:
+# certificates keys are required, username and password are optional.
+apiVersion: v1
+kind: Secret
+data:
+  ca.crt: ${CA}
+  server.crt: ${CRT}
+  server.key: ${KEY}
+  username: ${ROOT_USERNAME}
+  password: ${ROOT_PASSWORD}
+metadata:
+  creationTimestamp: null
+  name: kine-secret
+type: kamaji.clastix.io/kine

docs/getting-started-with-kamaji.md

@@ -57,7 +57,9 @@ Kamaji offers the possibility of using a different storage system than `ETCD` fo
 
 Read it more in the provided [guide](../deploy/kine/README.md).
 
-Assuming you adjusted the [Kamaji manifest](../config/install.yaml) to connect to Kine and compatible database using the proper driver, you can now install it.
+
+
+Assuming you adjusted the [Kamaji manifest](./config/install.yaml) to connect to Kine and compatible database using the proper driver, you can now install it.
 
 ### Install Kamaji
 

docs/reference.md

@@ -15,30 +15,49 @@ This is easily explained in this way:
 Available flags are the following:
 
 ```
---config-file string                 Configuration file alternative. (default "kamaji.yaml")
---datastore string                   The default DataStore that should be used by Kamaji to setup the required storage (default "etcd")
+--config-file string                 Configuration file alternative. (default "./kamaji.yaml")
+--etcd-ca-secret-name                Name of the secret which contains CA's certificate and private key. (default: "etcd-certs")
+--etcd-ca-secret-namespace           Namespace of the secret which contains CA's certificate and private key. (default: "kamaji")
+--etcd-client-secret-name            Name of the secret which contains ETCD client certificates. (default: "root-client-certs")
+--etcd-client-secret-namespace       Name of the namespace where the secret which contains ETCD client certificates is. (default: "kamaji")
+--etcd-compaction-interval           ETCD Compaction interval (i.e. "5m0s"). (default: "0" (disabled))
+--etcd-endpoints                     Comma-separated list with ETCD endpoints (i.e. https://etcd-0.etcd.kamaji.svc.cluster.local,https://etcd-1.etcd.kamaji.svc.cluster.local,https://etcd-2.etcd.kamaji.svc.cluster.local)
+--etcd-storage-type                  ETCD Storage type (i.e. "etcd", "kine-mysql", "kine-postgresql"). (default: "etcd")
 --health-probe-bind-address string   The address the probe endpoint binds to. (default ":8081")
---kine-image string                  Container image along with tag to use for the Kine sidecar container (used only if etcd-storage-type is set to one of kine strategies) (default "rancher/kine:v0.9.2-amd64")
+--kine-port int                      Port where the DB used by Kine is listening to.
+--kine-host string                   Host where the DB used by Kine is working.
+--kine-secret-name                   Name of the secret which contains the Kine configuration. (default: "kine-secret")
+--kine-secret-name                   Name of the namespace where the secret which contains the Kine configuration. (default: "kamaji-system")
 --kubeconfig string                  Paths to a kubeconfig. Only required if out-of-cluster.
 --leader-elect                       Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.
 --metrics-bind-address string        The address the metric endpoint binds to. (default ":8080")
---tmp-directory string               Directory which will be used to work with temporary files. (default "/tmp/kamaji")
+--tmp-directory                      Directory which will be used to work with temporary files. (default "/tmp/kamaji")
 --zap-devel                          Development Mode defaults(encoder=consoleEncoder,logLevel=Debug,stackTraceLevel=Warn). Production Mode defaults(encoder=jsonEncoder,logLevel=Info,stackTraceLevel=Error) (default true)
 --zap-encoder encoder                Zap log encoding (one of 'json' or 'console')
 --zap-log-level level                Zap Level to configure the verbosity of logging. Can be one of 'debug', 'info', 'error', or any integer value > 0 which corresponds to custom debug levels of increasing verbosity
 --zap-stacktrace-level level         Zap Level at and above which stacktraces are captured (one of 'info', 'error', 'panic').
---zap-time-encoding time-encoding    Zap time encoding (one of 'epoch', 'millis', 'nano', 'iso8601', 'rfc3339' or 'rfc3339nano'). Defaults to 'epoch'.
 ```
 
 Available environment variables are:
 
-| Environment variable                 | Description                                                                                                            |
-|--------------------------------------|------------------------------------------------------------------------------------------------------------------------|
-| `KAMAJI_DATASTORE`                   | Name of the DataStore resource with driver definition and settings. (default "etcd")                                   |
-| `KAMAJI_METRICS_BIND_ADDRESS`        | The address the metric endpoint binds to. (default ":8080")                                                            |
-| `KAMAJI_HEALTH_PROBE_BIND_ADDRESS`   | The address the probe endpoint binds to. (default ":8081")                                                             |
-| `KAMAJI_LEADER_ELECTION`             | Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.  |
-| `KAMAJI_TMP_DIRECTORY`               | Directory which will be used to work with temporary files. (default "/tmp/kamaji")                                     |
+| Environment variable               | Description                                                  |
+| ---------------------------------- | ------------------------------------------------------------ |
+| `KAMAJI_ETCD_CA_SECRET_NAME`      | Name of the secret which contains CA's certificate and private key. (default: "etcd-certs")  |
+| `KAMAJI_ETCD_CA_SECRET_NAMESPACE`      | Namespace of the secret which contains CA's certificate and private key. (default: "kamaji")  |
+| `KAMAJI_ETCD_CLIENT_SECRET_NAME`      | Name of the secret which contains ETCD client certificates. (default: "root-client-certs")  |
+| `KAMAJI_ETCD_CLIENT_SECRET_NAMESPACE`      | Name of the namespace where the secret which contains ETCD client certificates is. (default: "kamaji")  |
+| `KAMAJI_ETCD_COMPACTION_INTERVAL`      | ETCD Compaction interval (i.e. "5m0s"). (default: "0" (disabled))  |
+| `KAMAJI_ETCD_ENDPOINTS`      | Comma-separated list with ETCD endpoints (i.e. etcd-server-1:2379,etcd-server-2:2379). (default: "etcd-server:2379")  |
+| `KAMAJI_ETCD_STORAGE_TYPE`      | ETCD Storage type (i.e. "etcd", "kine-mysql"). (default: "etcd")  |
+| `KAMAJI_ETCD_SERVERS`      | Comma-separated list with ETCD servers (i.e. etcd-0.etcd.kamaji.svc.cluster.local,etcd-1.etcd.kamaji.svc.cluster.local,etcd-2.etcd.kamaji.svc.cluster.local)  |
+| `KAMAJI_METRICS_BIND_ADDRESS`      | The address the metric endpoint binds to. (default ":8080")  |
+| `KAMAJI_HEALTH_PROBE_BIND_ADDRESS` | The address the probe endpoint binds to. (default ":8081")   |
+| `KAMAJI_KINE_MYSQL_HOST` | Host where MySQL is running(default "localhost")   |
+| `KAMAJI_KINE_MYSQL_PORT` | Port where MySQL is running (default: 3306)   |
+| `KAMAJI_KINE_MYSQL_SECRET_NAME` | Name of the secret where the necessary configuration and certificates are. (default: "mysql-config")   |
+| `KAMAJI_KINE_MYSQL_SECRET_NAMESPACE` | Name of the namespace of the secret where the necessary configuration and certificates are. (default: "kamaji-system")   |
+| `KAMAJI_LEADER_ELECTION`           | Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager. |
+| `KAMAJI_TMP_DIRECTORY`           | Directory which will be used to work with temporary files. (default "/tmp/kamaji") |
 
 
 ## Build and deploy
@@ -101,7 +120,7 @@ addons:
 addons:
   konnectivity:
     proxyPort: 31132 # mandatory
-    version: v0.0.32
+    version: v0.0.31
     resources:
       requests:
         cpu: 100m
@@ -109,6 +128,5 @@ addons:
       limits:
         cpu: 100m
         memory: 128Mi
-    serverImage: registry.k8s.io/kas-network-proxy/proxy-server
-    agentImage: registry.k8s.io/kas-network-proxy/proxy-agent
-```
+    serverImage: us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-server
+    agentImage: us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-agent

e2e/suite_test.go

@@ -55,6 +55,7 @@ var _ = BeforeSuite(func() {
 	Expect(err).NotTo(HaveOccurred())
 
 	//+kubebuilder:scaffold:scheme
+
 	k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme})
 	Expect(err).NotTo(HaveOccurred())
 	Expect(k8sClient).NotTo(BeNil())

e2e/tenant_control_plane_ready_test.go

@@ -18,6 +18,7 @@ import (
 var _ = Describe("Deploy a TenantControlPlane resource", func() {
 	// Fill TenantControlPlane object
 	tcp := kamajiv1alpha1.TenantControlPlane{
+		TypeMeta: metav1.TypeMeta{},
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      "tcp-clusterip",
 			Namespace: "default",

e2e/utils_test.go

@@ -7,7 +7,7 @@ import (
 	"bytes"
 	"context"
 	"fmt"
-	"io"
+	"io/ioutil"
 	"os/exec"
 
 	. "github.com/onsi/ginkgo"
@@ -96,7 +96,7 @@ func PrintKamajiLogs() {
 
 		defer podLogs.Close()
 
-		podBytes, err := io.ReadAll(podLogs)
+		podBytes, err := ioutil.ReadAll(podLogs)
 		Expect(err).ToNot(HaveOccurred())
 
 		_, _ = fmt.Fprintln(GinkgoWriter, "DEBUG: retrieving Kamaji Pod logs")

e2e/worker_kubeadm_join_test.go

@@ -7,6 +7,7 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"io/ioutil"
 	"os"
 	"strings"
 	"time"
@@ -36,6 +37,7 @@ var _ = Describe("starting a kind worker with kubeadm", func() {
 
 	JustBeforeEach(func() {
 		tcp = kamajiv1alpha1.TenantControlPlane{
+			TypeMeta: metav1.TypeMeta{},
 			ObjectMeta: metav1.ObjectMeta{
 				Name:      "worker-nodes-join",
 				Namespace: "default",
@@ -82,7 +84,7 @@ var _ = Describe("starting a kind worker with kubeadm", func() {
 		})
 		Expect(err).ToNot(HaveOccurred())
 
-		kubeconfigFile, err = os.CreateTemp("", "kamaji")
+		kubeconfigFile, err = ioutil.TempFile("", "kamaji")
 		Expect(err).ToNot(HaveOccurred())
 	})
 

e2e/worker_tcp_change_port_test.go

@@ -6,6 +6,7 @@ package e2e
 import (
 	"context"
 	"fmt"
+	"io/ioutil"
 	"os"
 	"time"
 
@@ -65,7 +66,7 @@ var _ = Describe("validating kubeconfig", func() {
 
 		var err error
 
-		kubeconfigFile, err = os.CreateTemp("", "kamaji")
+		kubeconfigFile, err = ioutil.TempFile("", "kamaji")
 		Expect(err).ToNot(HaveOccurred())
 	})
 

go.mod

@@ -3,29 +3,30 @@ module github.com/clastix/kamaji
 go 1.18
 
 require (
-	github.com/go-logr/logr v1.2.3
+	github.com/go-logr/logr v1.2.0
 	github.com/go-pg/pg/v10 v10.10.6
 	github.com/go-sql-driver/mysql v1.6.0
 	github.com/google/uuid v1.3.0
 	github.com/json-iterator/go v1.1.12
 	github.com/onsi/ginkgo v1.16.5
-	github.com/onsi/gomega v1.19.0
+	github.com/onsi/gomega v1.17.0
 	github.com/pkg/errors v0.9.1
 	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.10.1
 	github.com/testcontainers/testcontainers-go v0.13.0
-	go.etcd.io/etcd/api/v3 v3.5.4
-	go.etcd.io/etcd/client/v3 v3.5.4
-	k8s.io/api v0.25.0
-	k8s.io/apimachinery v0.25.0
-	k8s.io/apiserver v0.25.0
-	k8s.io/client-go v0.25.0
+	go.etcd.io/etcd/api/v3 v3.5.1
+	go.etcd.io/etcd/client/v3 v3.5.1
+	google.golang.org/grpc v1.43.0
+	k8s.io/api v0.23.5
+	k8s.io/apimachinery v0.23.5
+	k8s.io/apiserver v0.23.5
+	k8s.io/client-go v0.23.5
 	k8s.io/cluster-bootstrap v0.0.0
-	k8s.io/component-base v0.25.0
+	k8s.io/component-base v0.23.5
 	k8s.io/kube-proxy v0.0.0
 	k8s.io/kubelet v0.0.0
 	k8s.io/kubernetes v1.23.5
-	k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed
+	k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9
 	sigs.k8s.io/controller-runtime v0.11.0
 )
 
@@ -34,8 +35,8 @@ require (
 	cloud.google.com/go/storage v1.18.2 // indirect
 	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
 	github.com/Azure/go-autorest v14.2.0+incompatible // indirect
-	github.com/Azure/go-autorest/autorest v0.11.27 // indirect
-	github.com/Azure/go-autorest/autorest/adal v0.9.20 // indirect
+	github.com/Azure/go-autorest/autorest v0.11.18 // indirect
+	github.com/Azure/go-autorest/autorest/adal v0.9.13 // indirect
 	github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
 	github.com/Azure/go-autorest/logger v0.2.1 // indirect
 	github.com/Azure/go-autorest/tracing v0.6.0 // indirect
@@ -57,31 +58,30 @@ require (
 	github.com/coreos/go-semver v0.3.0 // indirect
 	github.com/coreos/go-systemd/v22 v22.3.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/docker/distribution v2.8.1+incompatible // indirect
+	github.com/docker/distribution v2.7.1+incompatible // indirect
 	github.com/docker/docker v20.10.11+incompatible // indirect
 	github.com/docker/go-connections v0.4.0 // indirect
 	github.com/docker/go-units v0.4.0 // indirect
-	github.com/emicklei/go-restful/v3 v3.8.0 // indirect
-	github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1 // indirect
+	github.com/envoyproxy/go-control-plane v0.10.1 // indirect
 	github.com/envoyproxy/protoc-gen-validate v0.6.2 // indirect
 	github.com/evanphx/json-patch v4.12.0+incompatible // indirect
+	github.com/form3tech-oss/jwt-go v3.2.3+incompatible // indirect
 	github.com/fsnotify/fsnotify v1.5.1 // indirect
 	github.com/go-errors/errors v1.0.1 // indirect
-	github.com/go-logr/zapr v1.2.3 // indirect
+	github.com/go-logr/zapr v1.2.0 // indirect
 	github.com/go-openapi/jsonpointer v0.19.5 // indirect
 	github.com/go-openapi/jsonreference v0.19.5 // indirect
 	github.com/go-openapi/swag v0.19.14 // indirect
 	github.com/go-pg/zerochecker v0.2.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang-jwt/jwt/v4 v4.2.0 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/google/btree v1.0.1 // indirect
-	github.com/google/gnostic v0.5.7-v3refs // indirect
 	github.com/google/go-cmp v0.5.6 // indirect
 	github.com/google/gofuzz v1.1.0 // indirect
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
 	github.com/googleapis/gax-go/v2 v2.1.1 // indirect
+	github.com/googleapis/gnostic v0.5.5 // indirect
 	github.com/googleapis/google-cloud-go-testing v0.0.0-20210719221736-1c9a4c676720 // indirect
 	github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
@@ -96,13 +96,12 @@ require (
 	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
 	github.com/mitchellh/mapstructure v1.4.3 // indirect
 	github.com/moby/sys/mount v0.2.0 // indirect
-	github.com/moby/sys/mountinfo v0.6.0 // indirect
+	github.com/moby/sys/mountinfo v0.5.0 // indirect
 	github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
 	github.com/morikuni/aec v1.0.0 // indirect
-	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/nxadm/tail v1.4.8 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.0.2 // indirect
@@ -119,76 +118,76 @@ require (
 	github.com/spf13/cast v1.4.1 // indirect
 	github.com/spf13/cobra v1.4.0 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
+	github.com/stretchr/testify v1.7.0 // indirect
 	github.com/subosito/gotenv v1.2.0 // indirect
 	github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc // indirect
 	github.com/vmihailenco/bufpool v0.1.11 // indirect
 	github.com/vmihailenco/msgpack/v5 v5.3.4 // indirect
 	github.com/vmihailenco/tagparser v0.1.2 // indirect
 	github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
-	github.com/xlab/treeprint v1.1.0 // indirect
-	go.etcd.io/etcd/client/pkg/v3 v3.5.4 // indirect
+	github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca // indirect
+	go.etcd.io/etcd/client/pkg/v3 v3.5.1 // indirect
 	go.opencensus.io v0.23.0 // indirect
 	go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect
 	go.uber.org/atomic v1.7.0 // indirect
 	go.uber.org/multierr v1.6.0 // indirect
 	go.uber.org/zap v1.19.1 // indirect
-	golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd // indirect
-	golang.org/x/net v0.0.0-20220722155237-a158d28d115b // indirect
+	golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa // indirect
+	golang.org/x/net v0.0.0-20211216030914-fe4d6282115f // indirect
 	golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect
-	golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f // indirect
-	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
+	golang.org/x/sys v0.0.0-20220209214540-3681064d5158 // indirect
+	golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b // indirect
 	golang.org/x/text v0.3.7 // indirect
-	golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect
+	golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect
 	google.golang.org/api v0.63.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect
-	google.golang.org/grpc v1.47.0 // indirect
-	google.golang.org/protobuf v1.28.0 // indirect
+	google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368 // indirect
+	google.golang.org/protobuf v1.27.1 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/ini.v1 v1.66.2 // indirect
 	gopkg.in/square/go-jose.v2 v2.5.1 // indirect
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	gopkg.in/yaml.v3 v3.0.1 // indirect
-	k8s.io/apiextensions-apiserver v0.25.0 // indirect
-	k8s.io/cli-runtime v0.25.0 // indirect
-	k8s.io/klog/v2 v2.70.1 // indirect
-	k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+	k8s.io/apiextensions-apiserver v0.23.5 // indirect
+	k8s.io/cli-runtime v0.23.5 // indirect
+	k8s.io/klog/v2 v2.60.1 // indirect
+	k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 // indirect
 	k8s.io/system-validators v1.6.0 // indirect
 	mellium.im/sasl v0.3.0 // indirect
-	sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect
-	sigs.k8s.io/kustomize/api v0.12.1 // indirect
-	sigs.k8s.io/kustomize/kyaml v0.13.9 // indirect
-	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
+	sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 // indirect
+	sigs.k8s.io/kustomize/api v0.10.1 // indirect
+	sigs.k8s.io/kustomize/kyaml v0.13.0 // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect
 	sigs.k8s.io/yaml v1.3.0 // indirect
 )
 
 replace (
-	k8s.io/api => k8s.io/api v0.25.0
-	k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.25.0
-	k8s.io/apimachinery => k8s.io/apimachinery v0.25.0
-	k8s.io/apiserver => k8s.io/apiserver v0.25.0
-	k8s.io/cli-runtime => k8s.io/cli-runtime v0.25.0
-	k8s.io/client-go => k8s.io/client-go v0.25.0
-	k8s.io/cloud-provider => k8s.io/cloud-provider v0.25.0
-	k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.25.0
-	k8s.io/code-generator => k8s.io/code-generator v0.25.0
-	k8s.io/component-base => k8s.io/component-base v0.25.0
-	k8s.io/component-helpers => k8s.io/component-helpers v0.25.0
-	k8s.io/controller-manager => k8s.io/controller-manager v0.25.0
-	k8s.io/cri-api => k8s.io/cri-api v0.25.0
-	k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.25.0
-	k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.25.0
-	k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.25.0
-	k8s.io/kube-proxy => k8s.io/kube-proxy v0.25.0
-	k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.25.0
-	k8s.io/kubectl => k8s.io/kubectl v0.25.0
-	k8s.io/kubelet => k8s.io/kubelet v0.25.0
-	k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.25.0
-	k8s.io/metrics => k8s.io/metrics v0.25.0
-	k8s.io/mount-utils => k8s.io/mount-utils v0.25.0
-	k8s.io/pod-security-admission => k8s.io/pod-security-admission v0.25.0
-	k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.25.0
+	k8s.io/api => k8s.io/api v0.23.5
+	k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.23.5
+	k8s.io/apimachinery => k8s.io/apimachinery v0.23.5
+	k8s.io/apiserver => k8s.io/apiserver v0.23.5
+	k8s.io/cli-runtime => k8s.io/cli-runtime v0.23.5
+	k8s.io/client-go => k8s.io/client-go v0.23.5
+	k8s.io/cloud-provider => k8s.io/cloud-provider v0.23.5
+	k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.23.5
+	k8s.io/code-generator => k8s.io/code-generator v0.23.5
+	k8s.io/component-base => k8s.io/component-base v0.23.5
+	k8s.io/component-helpers => k8s.io/component-helpers v0.23.5
+	k8s.io/controller-manager => k8s.io/controller-manager v0.23.5
+	k8s.io/cri-api => k8s.io/cri-api v0.23.5
+	k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.23.5
+	k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.23.5
+	k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.23.5
+	k8s.io/kube-proxy => k8s.io/kube-proxy v0.23.5
+	k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.23.5
+	k8s.io/kubectl => k8s.io/kubectl v0.23.5
+	k8s.io/kubelet => k8s.io/kubelet v0.23.5
+	k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.23.5
+	k8s.io/metrics => k8s.io/metrics v0.23.5
+	k8s.io/mount-utils => k8s.io/mount-utils v0.23.5
+	k8s.io/pod-security-admission => k8s.io/pod-security-admission v0.23.5
+	k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.23.5
 )

go.sum

@@ -57,23 +57,20 @@ dmitri.shuralyov.com/gpu/mtl v0.0.0-20201218220906-28db891af037/go.mod h1:H6x//7
 github.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
 github.com/Azure/azure-sdk-for-go v55.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
 github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
+github.com/Azure/go-ansiterm v0.0.0-20210608223527-2377c96fe795/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
 github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8=
 github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
 github.com/Azure/go-autorest v10.8.1+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
 github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs=
 github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
+github.com/Azure/go-autorest/autorest v0.11.18 h1:90Y4srNYrwOtAgVo3ndrQkTYn6kf1Eg/AjTFJ8Is2aM=
 github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA=
-github.com/Azure/go-autorest/autorest v0.11.27 h1:F3R3q42aWytozkV8ihzcgMO4OA4cuqr3bNlsEuF6//A=
-github.com/Azure/go-autorest/autorest v0.11.27/go.mod h1:7l8ybrIdUmGqZMTD0sRtAr8NvbHjfofbf8RSP2q7w7U=
+github.com/Azure/go-autorest/autorest/adal v0.9.13 h1:Mp5hbtOePIzM8pJVRa3YLrWWmZtoxRXqUEzCfJt3+/Q=
 github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M=
-github.com/Azure/go-autorest/autorest/adal v0.9.18/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ=
-github.com/Azure/go-autorest/autorest/adal v0.9.20 h1:gJ3E98kMpFB1MFqQCvA1yFab8vthOeD4VlFRQULxahg=
-github.com/Azure/go-autorest/autorest/adal v0.9.20/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ=
 github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw=
 github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74=
+github.com/Azure/go-autorest/autorest/mocks v0.4.1 h1:K0laFcLE6VLTOwNgSxaGbUcLPuGXlNkbVvq4cW4nIHk=
 github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k=
-github.com/Azure/go-autorest/autorest/mocks v0.4.2 h1:PGN4EDXnuQbojHbU0UWoNvmu9AGVwYHG9/fkDYhtAfw=
-github.com/Azure/go-autorest/autorest/mocks v0.4.2/go.mod h1:Vy7OitM9Kei0i1Oj+LvyAWMXJHeKH1MVlzFugfVrmyU=
 github.com/Azure/go-autorest/autorest/to v0.4.0/go.mod h1:fE8iZBn7LQR7zH/9XU2NcPR4o9jEImooCeWJcYV/zLE=
 github.com/Azure/go-autorest/autorest/validation v0.1.0/go.mod h1:Ha3z/SqBeaalWQvokg3NZAlQTalVMtOIAs1aGK7G6u8=
 github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+ZtXWSmf4Tg=
@@ -83,9 +80,8 @@ github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBp
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/GoogleCloudPlatform/k8s-cloud-provider v1.16.1-0.20210702024009-ea6160c1d0e3/go.mod h1:8XasY4ymP2V/tn2OOV9ZadmiTE1FIB/h3W+yNlPttKw=
-github.com/GoogleCloudPlatform/k8s-cloud-provider v1.18.1-0.20220218231025-f11817397a1b/go.mod h1:FNj4KYEAAHfYu68kRYolGoxkaJn+6mdEsaM12VTwuI0=
 github.com/JeffAshton/win_pdh v0.0.0-20161109143554-76bb4ee9f0ab/go.mod h1:3VYc5hodBMJ5+l/7J4xAyMeuM2PNuepvHlGs8yilUCA=
-github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE=
+github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd/go.mod h1:64YHyfSL2R96J44Nlwm39UHepQbyR5q10x7iYa1ks2E=
 github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA=
 github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA=
 github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw=
@@ -124,12 +120,11 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF
 github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
 github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0=
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
-github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20220418222510-f25a4f6275ed/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY=
+github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210826220005-b48c857c3a0e/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY=
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
-github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/auth0/go-jwt-middleware v1.0.1/go.mod h1:YSeUX3z6+TF2H+7padiEqNJ73Zy9vXW72U//IgN0BIM=
 github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0=
@@ -152,7 +147,6 @@ github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnweb
 github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
 github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=
 github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
-github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
 github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
 github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
 github.com/bshuster-repo/logrus-logstash-hook v0.4.1/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk=
@@ -172,7 +166,7 @@ github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghf
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
 github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/chai2010/gettext-go v1.0.2/go.mod h1:y+wnP2cHYaVj19NZhYKAwEMH2CI1gNHeQQ+5AjwawxA=
+github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5/go.mod h1:/iP1qXHoty45bqomnu2LM+VVyAEdWN+vtSHGlQgyxbw=
 github.com/checkpoint-restore/go-criu/v4 v4.1.0/go.mod h1:xUQBLp4RLc5zJtWY++yjOoMoB5lihDt7fai+75m+rGw=
 github.com/checkpoint-restore/go-criu/v5 v5.0.0/go.mod h1:cfwC0EG7HMUenopBsUf9d89JlCLQIfgVcNsNN0t6T2M=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
@@ -191,6 +185,7 @@ github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnht
 github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4 h1:hzAQntlaYRkVSFEfj9OTWlVV1H155FMD8BTKktLv0QI=
 github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI=
 github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
+github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
@@ -322,7 +317,7 @@ github.com/d2g/hardwareaddr v0.0.0-20190221164911-e7d9fbe030e4/go.mod h1:bMl4RjI
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/daviddengcn/go-colortext v1.0.0/go.mod h1:zDqEI5NVUop5QPpVJUxE9UO10hRnmkD5G4Pmri9+m4c=
+github.com/daviddengcn/go-colortext v0.0.0-20160507010035-511bcaf42ccd/go.mod h1:dv4zxwHi5C/8AeI+4gX4dCWOIvNi7I6JCSX0HvlKPgE=
 github.com/denverdino/aliyungo v0.0.0-20190125010748-a747050bb1ba/go.mod h1:dV8lFg6daOBZbT6/BDGIz6Y3WFGn8juu6G+CQ6LHtl0=
 github.com/dgrijalva/jwt-go v0.0.0-20170104182250-a601269ab70c/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
@@ -331,9 +326,8 @@ github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyG
 github.com/dnephin/pflag v1.0.7/go.mod h1:uxE91IoWURlOiTUIA8Mq5ZZkAv3dPUfZNaT80Zm7OQE=
 github.com/docker/distribution v0.0.0-20190905152932-14b96e55d84c/go.mod h1:0+TTO4EOBfRPhZXAeF1Vu+W3hHZ8eLp8PgKVZlcvtFY=
 github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
+github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug=
 github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
-github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6bk8XXApsHjKukMl68=
-github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/docker v20.10.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/docker v20.10.11+incompatible h1:OqzI/g/W54LczvhnccGqniFoQghHx3pklbLuhfXpqGo=
 github.com/docker/docker v20.10.11+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
@@ -352,8 +346,6 @@ github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25Kn
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
 github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
-github.com/emicklei/go-restful/v3 v3.8.0 h1:eCZ8ulSerjdAiaNpF7GxXIE7ZCMo1moN1qX+S609eVw=
-github.com/emicklei/go-restful/v3 v3.8.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
@@ -361,8 +353,9 @@ github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5y
 github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
 github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
 github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ=
-github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1 h1:xvqufLtNVwAhN8NMyWklVgxnWohi+wtMGQMhtxexlm0=
-github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE=
+github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0=
+github.com/envoyproxy/go-control-plane v0.10.1 h1:cgDRLG7bs59Zd+apAWuzLQL95obVYAymNJek76W3mgw=
+github.com/envoyproxy/go-control-plane v0.10.1/go.mod h1:AY7fTTXNdv/aJ2O5jwpxAPOWUZ7hQAEvzN5Pf27BkQQ=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/envoyproxy/protoc-gen-validate v0.6.2 h1:JiO+kJTpmYGjEodY7O1Zk8oZcNz1+f30UtwtXoFUPzE=
 github.com/envoyproxy/protoc-gen-validate v0.6.2/go.mod h1:2t7qjJNvHPx8IjnBOzl9E9/baC+qXE/TeeyBRzgJDws=
@@ -379,6 +372,7 @@ github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSw
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
 github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
 github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
+github.com/form3tech-oss/jwt-go v3.2.3+incompatible h1:7ZaBxOI7TMoYBfyA3cQHErNNyAWIKUMIwqxEtgHOs5c=
 github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
 github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
@@ -405,12 +399,10 @@ github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V
 github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
 github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
 github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
+github.com/go-logr/logr v1.2.0 h1:QK40JKJyMdUDz+h+xvCsru/bJhvG0UxvePV0ufL/AcE=
 github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
-github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
-github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0=
-github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
-github.com/go-logr/zapr v1.2.3 h1:a9vnzlIBPQBBkeaR9IuMUfmVOrQlkoC4YfPoFkX3T7A=
-github.com/go-logr/zapr v1.2.3/go.mod h1:eIauM6P8qSvTw5o2ez6UEAfGjQKrxQTl5EoK+Qa2oG4=
+github.com/go-logr/zapr v1.2.0 h1:n4JnPI1T3Qq1SFEi/F8rwLrZERp2bso19PJZDB9dayk=
+github.com/go-logr/zapr v1.2.0/go.mod h1:Qa4Bsj2Vb+FAVeAKsLD8RLQ+YRJB8YDmOAKxaBQf7Ro=
 github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
 github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY=
 github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
@@ -445,9 +437,6 @@ github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXP
 github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
-github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
-github.com/golang-jwt/jwt/v4 v4.2.0 h1:besgBTC8w8HjP6NzQdxwKH9Z5oQMZ24ThTrHp3cZ8eU=
-github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
 github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4=
@@ -485,18 +474,14 @@ github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx
 github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
 github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
-github.com/golangplus/bytes v0.0.0-20160111154220-45c989fe5450/go.mod h1:Bk6SMAONeMXrxql8uvOKuAZSu8aM5RUGv+1C6IJaEho=
-github.com/golangplus/bytes v1.0.0/go.mod h1:AdRaCFwmc/00ZzELMWb01soso6W1R/++O1XL80yAn+A=
-github.com/golangplus/fmt v1.0.0/go.mod h1:zpM0OfbMCjPtd2qkTD/jX2MgiFCqklhSUFyDW44gVQE=
-github.com/golangplus/testing v1.0.0/go.mod h1:ZDreixUV3YzhoVraIDyOzHrr76p6NUh6k/pPg/Q3gYA=
+github.com/golangplus/testing v0.0.0-20180327235837-af21d9c3145e/go.mod h1:0AA//k/eakGydO4jKRoRL2j92ZKSzTgj9tclaCrvXHk=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4=
 github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA=
 github.com/google/cadvisor v0.43.0/go.mod h1:+RdMSbc3FVr5NYCD2dOEJy/LI0jYJ/0xJXkzWXEyiFQ=
-github.com/google/cel-go v0.12.4/go.mod h1:Av7CU6r6X3YmcHR9GXqVDaEJYfEtSxl6wvIjUQTriCw=
-github.com/google/gnostic v0.5.7-v3refs h1:FhTMOKj2VhjpouxvWJAV1TL304uMlb9zcDqkl6cEI54=
-github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ=
+github.com/google/cel-go v0.9.0/go.mod h1:U7ayypeSkw23szu4GaQTPJGx66c20mx8JklMSxrmI1w=
+github.com/google/cel-spec v0.6.0/go.mod h1:Nwjgxy5CbjlPrtCWjeDjUyKMl8w41YBYGjsyDdqk0xA=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
@@ -531,7 +516,6 @@ github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLe
 github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
-github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
@@ -549,6 +533,7 @@ github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pf
 github.com/googleapis/gax-go/v2 v2.1.1 h1:dp3bWCh+PPO1zjRRiCSczJav13sBvG4UhNyVTa1KqdU=
 github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM=
 github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU=
+github.com/googleapis/gnostic v0.5.5 h1:9fHAtK0uDfpveeqqo1hkEZJcFvYXAiCN3UutL8F9xHw=
 github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA=
 github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g=
 github.com/googleapis/google-cloud-go-testing v0.0.0-20210719221736-1c9a4c676720 h1:zC34cGQu69FG7qzJ3WiKW244WfhDC3xxYMeNOX2gtUQ=
@@ -599,7 +584,7 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO
 github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
-github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
+github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
 github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
@@ -665,6 +650,7 @@ github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaW
 github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60=
 github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
 github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA=
 github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
 github.com/marstr/guid v1.1.0/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho=
@@ -701,11 +687,11 @@ github.com/moby/sys/mount v0.2.0 h1:WhCW5B355jtxndN5ovugJlMFJawbUODuW8fSnEH6SSM=
 github.com/moby/sys/mount v0.2.0/go.mod h1:aAivFE2LB3W4bACsUXChRHQ0qKWsetY4Y9V7sxOougM=
 github.com/moby/sys/mountinfo v0.4.0/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A=
 github.com/moby/sys/mountinfo v0.4.1/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A=
+github.com/moby/sys/mountinfo v0.5.0 h1:2Ks8/r6lopsxWi9m58nlwjaeSzUX9iiL1vj5qB/9ObI=
 github.com/moby/sys/mountinfo v0.5.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU=
-github.com/moby/sys/mountinfo v0.6.0 h1:gUDhXQx58YNrpHlK4nSL+7y2pxFZkUcXqzFDKWdC0Oo=
-github.com/moby/sys/mountinfo v0.6.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU=
 github.com/moby/sys/symlink v0.1.0/go.mod h1:GGDODQmbFOjFsXvfLVn3+ZRxkch54RkSiGqsZeMYowQ=
 github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc=
+github.com/moby/term v0.0.0-20210610120745-9d4ed1856297/go.mod h1:vgPCkQMyxTZ7IDy8SXRufE172gr8+K/JE/7hHFxHW3A=
 github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 h1:dcztxKSvZ4Id8iPpHERQBbIJfabdt4wUm5qy3wOL2Zc=
 github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -723,7 +709,6 @@ github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
 github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
 github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ=
 github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
-github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mvdan/xurls v1.1.0/go.mod h1:tQlNn3BED8bE/15hnSL2HLkDeLWpNPAwtw7wkEq44oU=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
@@ -748,18 +733,14 @@ github.com/onsi/ginkgo v1.14.2/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9k
 github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
 github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
 github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
-github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c=
-github.com/onsi/ginkgo/v2 v2.1.4 h1:GNapqRSid3zijZ9H77KrgVG4/8KqiyRsxcSxe+7ApXY=
-github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU=
 github.com/onsi/gomega v0.0.0-20151007035656-2152b45fa28a/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
 github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc=
+github.com/onsi/gomega v1.17.0 h1:9Luw4uT5HTjHTN8+aNcSThgH1vdXnmdJ8xIfZ4wyTRE=
 github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
-github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw=
-github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro=
 github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
 github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
 github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
@@ -806,7 +787,7 @@ github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qR
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
-github.com/pquerna/cachecontrol v0.1.0/go.mod h1:NrUG3Z7Rdu85UNR3vm7SOsl1nFIeSiQnrHV5K9mBcUI=
+github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA=
 github.com/prometheus/client_golang v0.0.0-20180209125602-c332b6f63c06/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
@@ -814,7 +795,6 @@ github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5Fsn
 github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g=
 github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
 github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
-github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
 github.com/prometheus/client_golang v1.12.1 h1:ZiaPsmm9uiBeaSMRznKsCDNtPCS0T3JVDGF+06gjBzk=
 github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
 github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
@@ -900,6 +880,7 @@ github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
 github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk=
 github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
+github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.1-0.20171106142849-4c012f6dcd95/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
@@ -963,8 +944,8 @@ github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2
 github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
 github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
-github.com/xlab/treeprint v1.1.0 h1:G/1DjNkPpfZCFt9CSh6b5/nY4VimlbHF3Rh4obvtzDk=
-github.com/xlab/treeprint v1.1.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0=
+github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca h1:1CFlNzQhALwjS9mBAUkycX616GzgsuYUOCHA5+HSlXI=
+github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -972,8 +953,6 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
-github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
-github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs=
 github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50/go.mod h1:NUSPSUX/bi6SeDMUh6brw0nXpxHnc96TguQh0+r/ssA=
 github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f/go.mod h1:GlGEuHIJweS1mbCqG+7vt2nvWLzLLnRHbXz5JKd/Qbg=
@@ -982,19 +961,18 @@ go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
 go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4=
 go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs=
-go.etcd.io/etcd/api/v3 v3.5.4 h1:OHVyt3TopwtUQ2GKdd5wu3PmmipR4FTwCqoEjSyRdIc=
-go.etcd.io/etcd/api/v3 v3.5.4/go.mod h1:5GB2vv4A4AOn3yk7MftYGHkUfGtDHnEraIjym4dYz5A=
+go.etcd.io/etcd/api/v3 v3.5.1 h1:v28cktvBq+7vGyJXF8G+rWJmj+1XUmMtqcLnH8hDocM=
+go.etcd.io/etcd/api/v3 v3.5.1/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs=
 go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g=
-go.etcd.io/etcd/client/pkg/v3 v3.5.4 h1:lrneYvz923dvC14R54XcA7FXoZ3mlGZAgmwhfm7HqOg=
-go.etcd.io/etcd/client/pkg/v3 v3.5.4/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g=
+go.etcd.io/etcd/client/pkg/v3 v3.5.1 h1:XIQcHCFSG53bJETYeRJtIxdLv2EWRGxcfzR8lSnTH4E=
+go.etcd.io/etcd/client/pkg/v3 v3.5.1/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g=
 go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ=
-go.etcd.io/etcd/client/v2 v2.305.4/go.mod h1:Ud+VUwIi9/uQHOMA+4ekToJ12lTxlv0zB/+DHwTGEbU=
 go.etcd.io/etcd/client/v3 v3.5.0/go.mod h1:AIKXXVX/DQXtfTEqBryiLTUXwON+GuvO6Z7lLS/oTh0=
-go.etcd.io/etcd/client/v3 v3.5.4 h1:p83BUL3tAYS0OT/r0qglgc3M1JjhM0diV8DSWAhVXv4=
-go.etcd.io/etcd/client/v3 v3.5.4/go.mod h1:ZaRkVgBZC+L+dLCjTcF1hRXpgZXQPOvnA/Ak/gq3kiY=
-go.etcd.io/etcd/pkg/v3 v3.5.4/go.mod h1:OI+TtO+Aa3nhQSppMbwE4ld3uF1/fqqwbpfndbbrEe0=
-go.etcd.io/etcd/raft/v3 v3.5.4/go.mod h1:SCuunjYvZFC0fBX0vxMSPjuZmpcSk+XaAcMrD6Do03w=
-go.etcd.io/etcd/server/v3 v3.5.4/go.mod h1:S5/YTU15KxymM5l3T6b09sNOHPXqGYIZStpuuGbb65c=
+go.etcd.io/etcd/client/v3 v3.5.1 h1:oImGuV5LGKjCqXdjkMHCyWa5OO1gYKCnC/1sgdfj1Uk=
+go.etcd.io/etcd/client/v3 v3.5.1/go.mod h1:OnjH4M8OnAotwaB2l9bVgZzRFKru7/ZMoS46OtKyd3Q=
+go.etcd.io/etcd/pkg/v3 v3.5.0/go.mod h1:UzJGatBQ1lXChBkQF0AuAtkRQMYnHubxAEYIrC3MSsE=
+go.etcd.io/etcd/raft/v3 v3.5.0/go.mod h1:UFOHSIvO/nKwd4lhkwabrTD3cqW5yVyYYf/KlD00Szc=
+go.etcd.io/etcd/server/v3 v3.5.0/go.mod h1:3Ah5ruV+M+7RZr0+Y/5mNLwC+eQlni+mQmOVdCRJoS4=
 go.mozilla.org/pkcs7 v0.0.0-20200128120323-432b2356ecb1/go.mod h1:SNgMg+EgDFwmvSmLRTNKC5fegJjB7v23qTQ0XLGUNHk=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
@@ -1051,11 +1029,8 @@ golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm
 golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
 golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa h1:idItI2DDfCokpg0N51B2VtiLdJ4vAuXC9fnCb2gACo4=
 golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.0.0-20220131195533-30dcbda58838/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd h1:XcWmESyNjXJMLahc3mqVQJcgSTDxFxhETVlfk9uGc38=
-golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -1102,8 +1077,6 @@ golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
-golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY=
-golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -1158,14 +1131,11 @@ golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qx
 golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211108170745-6635138e15ea/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
-golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
-golang.org/x/net v0.0.0-20220722155237-a158d28d115b h1:PxfKdU9lEEDYjdIzOtC4qFWgkU2rGHdKlKowJSMN9h0=
-golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.0.0-20211216030914-fe4d6282115f h1:hEYJvxw1lSnWIl8X9ofsYMklzaDs90JI2az5YMd4fPM=
+golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -1197,7 +1167,6 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -1300,20 +1269,15 @@ golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210917161153-d61c044b1678/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210923061019-b8560ed6a9b7/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f h1:v4INt8xihDGvnrfjMDVXGxw9wrfxYyCjk0KbXjhR55s=
-golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220209214540-3681064d5158 h1:rm+CHSpPEEW2IsXUib1ThaHIjuBVZjxNgSKmBLFfD4c=
+golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b h1:9zKuko04nR4gjZ4+DNjHqRlAJqbJETHwiNKDqTfOjfE=
 golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY=
-golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -1328,9 +1292,8 @@ golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxb
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs=
 golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 h1:vVKdlvoWBphwdxWKrFZEuM0kGgGLxUOYcY4U/2Vjg44=
-golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -1397,8 +1360,6 @@ golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.6-0.20210820212750-d4cc65f0b2ff/go.mod h1:YD9qOF0M9xpSpdWTBbzEl5e/RnCefISl8E5Noe10jFM=
-golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E=
-golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -1445,7 +1406,6 @@ google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqiv
 google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE=
 google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI=
 google.golang.org/api v0.58.0/go.mod h1:cAbP2FsxoGVNwtgNAmmn3y5G1TWAiVYRmg4yku3lv+E=
-google.golang.org/api v0.60.0/go.mod h1:d7rl65NZAkEQ90JFzqBjcRq1TVeG5ZoGV3sSpEnnVb4=
 google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I=
 google.golang.org/api v0.63.0 h1:n2bqqK895ygnBpdPDYetfy23K7fJ22wsrZKCyfuRkkA=
 google.golang.org/api v0.63.0/go.mod h1:gs4ij2ffTRXwuzzgJl/56BdwJaA194ijkfn++9tDuPo=
@@ -1493,6 +1453,7 @@ google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6D
 google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20201102152239-715cce707fb0/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
@@ -1523,12 +1484,11 @@ google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEc
 google.golang.org/genproto v0.0.0-20210917145530-b395a37504d4/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
 google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
 google.golang.org/genproto v0.0.0-20211016002631-37fc39342514/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
-google.golang.org/genproto v0.0.0-20211021150943-2b146023228c/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
 google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
 google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
 google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
-google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 h1:hrbNEivu7Zn1pxvHk6MBrq9iE22woVILTHqexqBxe6I=
-google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4=
+google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368 h1:Et6SkiuvnBn+SgrSYXs/BrUpGB4mbdwt4R3vaPIlicA=
+google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
 google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
@@ -1559,9 +1519,8 @@ google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnD
 google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE=
 google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
 google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
-google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk=
-google.golang.org/grpc v1.47.0 h1:9n77onPX5F3qfFCqjy9dhn8PbNQsIKeVU04J9G7umt8=
-google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk=
+google.golang.org/grpc v1.43.0 h1:Eeu7bZtDZ2DpRCsLhUlcrLnvYaMK1Gz86a+hMVvELmM=
+google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
@@ -1575,9 +1534,8 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
 google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw=
-google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -1617,9 +1575,8 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
-gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
 gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
 gotest.tools/gotestsum v1.7.0/go.mod h1:V1m4Jw3eBerhI/A6qCxUE07RnCg7ACkKj9BYcAm09V8=
@@ -1633,64 +1590,62 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
 honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
-k8s.io/api v0.25.0 h1:H+Q4ma2U/ww0iGB78ijZx6DRByPz6/733jIuFpX70e0=
-k8s.io/api v0.25.0/go.mod h1:ttceV1GyV1i1rnmvzT3BST08N6nGt+dudGrquzVQWPk=
-k8s.io/apiextensions-apiserver v0.25.0 h1:CJ9zlyXAbq0FIW8CD7HHyozCMBpDSiH7EdrSTCZcZFY=
-k8s.io/apiextensions-apiserver v0.25.0/go.mod h1:3pAjZiN4zw7R8aZC5gR0y3/vCkGlAjCazcg1me8iB/E=
-k8s.io/apimachinery v0.25.0 h1:MlP0r6+3XbkUG2itd6vp3oxbtdQLQI94fD5gCS+gnoU=
-k8s.io/apimachinery v0.25.0/go.mod h1:qMx9eAk0sZQGsXGu86fab8tZdffHbwUfsvzqKn4mfB0=
-k8s.io/apiserver v0.25.0 h1:8kl2ifbNffD440MyvHtPaIz1mw4mGKVgWqM0nL+oyu4=
-k8s.io/apiserver v0.25.0/go.mod h1:BKwsE+PTC+aZK+6OJQDPr0v6uS91/HWxX7evElAH6xo=
-k8s.io/cli-runtime v0.25.0 h1:XBnTc2Fi+w818jcJGzhiJKQuXl8479sZ4FhtV5hVJ1Q=
-k8s.io/cli-runtime v0.25.0/go.mod h1:bHOI5ZZInRHhbq12OdUiYZQN8ml8aKZLwQgt9QlLINw=
-k8s.io/client-go v0.25.0 h1:CVWIaCETLMBNiTUta3d5nzRbXvY5Hy9Dpl+VvREpu5E=
-k8s.io/client-go v0.25.0/go.mod h1:lxykvypVfKilxhTklov0wz1FoaUZ8X4EwbhS6rpRfN8=
-k8s.io/cloud-provider v0.25.0/go.mod h1:afVfVCIYOUER914WmSp0QpAtJn12gv4qu9NMT4XBxZo=
-k8s.io/cluster-bootstrap v0.25.0 h1:KJ2/r0dV+bLfTK5EBobAVKvjGel3N4Qqh3bvnzh9qPk=
-k8s.io/cluster-bootstrap v0.25.0/go.mod h1:x/TCtY3EiuR/rODkA3SvVQT3uSssQLf9cXcmSjdDTe0=
-k8s.io/code-generator v0.25.0/go.mod h1:B6jZgI3DvDFAualltPitbYMQ74NjaCFxum3YeKZZ+3w=
-k8s.io/component-base v0.25.0 h1:haVKlLkPCFZhkcqB6WCvpVxftrg6+FK5x1ZuaIDaQ5Y=
-k8s.io/component-base v0.25.0/go.mod h1:F2Sumv9CnbBlqrpdf7rKZTmmd2meJq0HizeyY/yAFxk=
-k8s.io/component-helpers v0.25.0/go.mod h1:auaFj2bvb5Zmy0mLk4WJNmwP0w4e7Zk+/Tu9FFBGA20=
-k8s.io/controller-manager v0.25.0/go.mod h1:QElCivPrZ64NP1Y976pkgyViZUqn6UcvjlXHiAAUGd0=
-k8s.io/cri-api v0.25.0/go.mod h1:J1rAyQkSJ2Q6I+aBMOVgg2/cbbebso6FNa0UagiR0kc=
-k8s.io/csi-translation-lib v0.25.0/go.mod h1:Wb80CDywP4753F6wWkIyOuJIQtQAbhgw985veSgAn/4=
+k8s.io/api v0.23.5 h1:zno3LUiMubxD/V1Zw3ijyKO3wxrhbUF1Ck+VjBvfaoA=
+k8s.io/api v0.23.5/go.mod h1:Na4XuKng8PXJ2JsploYYrivXrINeTaycCGcYgF91Xm8=
+k8s.io/apiextensions-apiserver v0.23.5 h1:5SKzdXyvIJKu+zbfPc3kCbWpbxi+O+zdmAJBm26UJqI=
+k8s.io/apiextensions-apiserver v0.23.5/go.mod h1:ntcPWNXS8ZPKN+zTXuzYMeg731CP0heCTl6gYBxLcuQ=
+k8s.io/apimachinery v0.23.5 h1:Va7dwhp8wgkUPWsEXk6XglXWU4IKYLKNlv8VkX7SDM0=
+k8s.io/apimachinery v0.23.5/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM=
+k8s.io/apiserver v0.23.5 h1:2Ly8oUjz5cnZRn1YwYr+aFgDZzUmEVL9RscXbnIeDSE=
+k8s.io/apiserver v0.23.5/go.mod h1:7wvMtGJ42VRxzgVI7jkbKvMbuCbVbgsWFT7RyXiRNTw=
+k8s.io/cli-runtime v0.23.5 h1:Z7XUpGoJZYZB2uNjQfJjMbyDKyVkoBGye62Ap0sWQHY=
+k8s.io/cli-runtime v0.23.5/go.mod h1:oY6QDF2qo9xndSq32tqcmRp2UyXssdGrLfjAVymgbx4=
+k8s.io/client-go v0.23.5 h1:zUXHmEuqx0RY4+CsnkOn5l0GU+skkRXKGJrhmE2SLd8=
+k8s.io/client-go v0.23.5/go.mod h1:flkeinTO1CirYgzMPRWxUCnV0G4Fbu2vLhYCObnt/r4=
+k8s.io/cloud-provider v0.23.5/go.mod h1:xMZFA6pIYKweqTkWCYVgRSVMAjqOvxVr3u/kmfyxvkU=
+k8s.io/cluster-bootstrap v0.23.5 h1:BhGnjfd2J0W/XqissyT+fX8bZwKVnvaTmjsDmtda3DY=
+k8s.io/cluster-bootstrap v0.23.5/go.mod h1:8/Gz6VTOMmEDDhn8U/nx0McnQR4YETAqiYXIlqR8hdQ=
+k8s.io/code-generator v0.23.5/go.mod h1:S0Q1JVA+kSzTI1oUvbKAxZY/DYbA/ZUb4Uknog12ETk=
+k8s.io/component-base v0.23.5 h1:8qgP5R6jG1BBSXmRYW+dsmitIrpk8F/fPEvgDenMCCE=
+k8s.io/component-base v0.23.5/go.mod h1:c5Nq44KZyt1aLl0IpHX82fhsn84Sb0jjzwjpcA42bY0=
+k8s.io/component-helpers v0.23.5/go.mod h1:5riXJgjTIs+ZB8xnf5M2anZ8iQuq37a0B/0BgoPQuSM=
+k8s.io/controller-manager v0.23.5/go.mod h1:n/KRlUzAtkFcZodZ/w0GlQdmErVKh7lS/wS0bbo7W4I=
+k8s.io/cri-api v0.23.5/go.mod h1:REJE3PSU0h/LOV1APBrupxrEJqnoxZC8KWzkBUHwrK4=
+k8s.io/csi-translation-lib v0.23.5/go.mod h1:8RyFkoHAJrFU7c7MN1ZUjctm3ZhHclKm1FIHNSyGcuw=
+k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
 k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E=
-k8s.io/gengo v0.0.0-20211129171323-c02415ce4185/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E=
 k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
 k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
 k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
 k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
-k8s.io/klog/v2 v2.70.1 h1:7aaoSdahviPmR+XkS7FyxlkkXs6tHISSG03RxleQAVQ=
-k8s.io/klog/v2 v2.70.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
-k8s.io/kube-aggregator v0.25.0/go.mod h1:dfdl4aQkleiWK/U++UDLdDC8g2rsonhkB23zzUeBCgM=
-k8s.io/kube-controller-manager v0.25.0/go.mod h1:SjL1hKSG2z9wajnvjRHZv1zOsdDHjmbZd1ykmaYO6J8=
+k8s.io/klog/v2 v2.60.1 h1:VW25q3bZx9uE3vvdL6M8ezOX79vA2Aq1nEWLqNQclHc=
+k8s.io/klog/v2 v2.60.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
+k8s.io/kube-aggregator v0.23.5/go.mod h1:3ynYx07Co6dzjpKPgipM+1/Mt2Jcm7dY++cRlKLr5s8=
+k8s.io/kube-controller-manager v0.23.5/go.mod h1:Pkg5lIk9YG9Qjj4F7Dn0gi6/k8cEYP63oLdgrlrrtu4=
+k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw=
+k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 h1:E3J9oCLlaobFUqsjG9DfKbP2BmgwBL2p7pn0A3dG9W4=
 k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk=
-k8s.io/kube-openapi v0.0.0-20220401212409-b28bf2818661/go.mod h1:daOouuuwd9JXpv1L7Y34iV3yf6nxzipkKMWWlqlvK9M=
-k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 h1:MQ8BAZPZlWk3S9K4a9NCkIFQtZShWqoha7snGixVgEA=
-k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1/go.mod h1:C/N6wCaBHeBHkHUesQOQy2/MZqGgMAFPqGsGQLdbZBU=
-k8s.io/kube-proxy v0.25.0 h1:QuoKEyXV+NNMXEh8oqlthUlHkmWF+WBnYUMHCf817k0=
-k8s.io/kube-proxy v0.25.0/go.mod h1:uHv1HwMVDYgl1pU2PTDKLRlxtNOf4z2M5YPYC6NP1CU=
-k8s.io/kube-scheduler v0.25.0/go.mod h1:cwiyJeImgFbhmbnImzvuhbiJayNngRNEe3FJkZDPw9Y=
-k8s.io/kubectl v0.25.0/go.mod h1:n16ULWsOl2jmQpzt2o7Dud1t4o0+Y186ICb4O+GwKAU=
-k8s.io/kubelet v0.25.0 h1:eTS5B1u1o63ndExAHKLJytzz/GBy86ROcxYtu0VK3RA=
-k8s.io/kubelet v0.25.0/go.mod h1:J6aQxrZdSsGPrskYrhZdEn6PCnGha+GNvF0g9aWfQnw=
+k8s.io/kube-proxy v0.23.5 h1:WdLf577zVdOGuZNqERdyXk+WjBwbd3Neiu0dMhY21gE=
+k8s.io/kube-proxy v0.23.5/go.mod h1:/yCbRrOHgPCb1g1k4XmMJPmNesfdPhZTGrvwNlNgwo8=
+k8s.io/kube-scheduler v0.23.5/go.mod h1:IJGf4WngeoAHLj4ms4n3Poa29ttmaxCXxIqpgU0ky7E=
+k8s.io/kubectl v0.23.5/go.mod h1:lLgw7cVY8xbd7o637vOXPca/w6HC205KsPCRDYRCxwE=
+k8s.io/kubelet v0.23.5 h1:eCGJ7olStiyF7TYHlUTjpXg2ltw7Bs9OPZcch8HP2Go=
+k8s.io/kubelet v0.23.5/go.mod h1:M0aj0gaX+rOaGfCfqkV6P7QbwtMwqbL6RdwviHmnehU=
 k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk=
 k8s.io/kubernetes v1.23.5 h1:bxpSv2BKc2MqYRfyqQqLVdodLZ2r+NZ/rEdZXyUAvug=
 k8s.io/kubernetes v1.23.5/go.mod h1:avI3LUTUYZugxwh52KMVM7v9ZjB5gYJ6D3FIoZ1SHUo=
-k8s.io/legacy-cloud-providers v0.25.0/go.mod h1:bnmUgHHeBmK3M9JgQzu+ne6UCUVURDzkpF0Y7VeypVE=
-k8s.io/metrics v0.25.0/go.mod h1:HZZrbhuRX+fsDcRc3u59o2FbrKhqD67IGnoFECNmovc=
-k8s.io/mount-utils v0.25.0/go.mod h1:WTYq8Ev/JrnkqK2h1jFUnC8qWGuqzMb9XDC+Lu3WNU0=
-k8s.io/pod-security-admission v0.25.0/go.mod h1:b/UC586Th2LijoNV+ssyyAryUvmaTrEWms5ZzBEkVsA=
-k8s.io/sample-apiserver v0.25.0/go.mod h1:Wyy/yKmXCrWLcc+082Vsn6fxAuwraRw5FQpekHg3go8=
+k8s.io/legacy-cloud-providers v0.23.5/go.mod h1:IENlwY686f1fbakotgNf7gAQuIyCvOUIAXkPPPE/7KU=
+k8s.io/metrics v0.23.5/go.mod h1:WNAtV2a5BYbmDS8+7jSqYYV6E3efuGTpIwJ8PTD1wgs=
+k8s.io/mount-utils v0.23.5/go.mod h1:OTN3LQPiOGMfx/SmVlsnySwsAmh4gYrDYLchlMHtf98=
+k8s.io/pod-security-admission v0.23.5/go.mod h1:aSyWfjev8Zil5DaZBZ+ICAObZmZlRqhnAZHxA9r71UI=
+k8s.io/sample-apiserver v0.23.5/go.mod h1:m4cnT3HgRY5Dt2AjMVKGnb31D6rGY0B+xpKtRJUUC8w=
 k8s.io/system-validators v1.6.0 h1:21qaPNdZ+mQrm4qc5shU0T5Eh49t/miFqZsn4sW8Hr0=
 k8s.io/system-validators v1.6.0/go.mod h1:bPldcLgkIUK22ALflnsXk8pvkTEndYdNuaHH6gRrl0Q=
 k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
+k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 h1:HNSDgDCrr/6Ly3WEGKZftiE7IY19Vz2GdbOCyI4qqhc=
 k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
-k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed h1:jAne/RjBTyawwAy0utX5eqigAwz/lQhTmy+Hr/Cpue4=
-k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 mellium.im/sasl v0.2.1/go.mod h1:ROaEDLQNuf9vjKqE1SrAfnsobm2YKXT1gnN1uDp1PjQ=
 mellium.im/sasl v0.3.0 h1:0qoaTCTo5Py7u/g0cBIQZcMOgG/5LM71nshbXwznBh8=
 mellium.im/sasl v0.3.0/go.mod h1:xm59PUYpZHhgQ9ZqoJ5QaCqzWMi8IeS49dhp6plPCzw=
@@ -1703,21 +1658,20 @@ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8
 rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
-sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.32/go.mod h1:fEO7lRTdivWO2qYVCVG7dEADOMo/MLDCVr8So2g88Uw=
+sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.30/go.mod h1:fEO7lRTdivWO2qYVCVG7dEADOMo/MLDCVr8So2g88Uw=
 sigs.k8s.io/controller-runtime v0.11.0 h1:DqO+c8mywcZLFJWILq4iktoECTyn30Bkj0CwgqMpZWQ=
 sigs.k8s.io/controller-runtime v0.11.0/go.mod h1:KKwLiTooNGu+JmLZGn9Sl3Gjmfj66eMbCQznLP5zcqA=
-sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 h1:iXTIw73aPyC+oRdyqqvVJuloN1p0AC/kzH07hu3NE+k=
-sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
-sigs.k8s.io/kustomize/api v0.12.1 h1:7YM7gW3kYBwtKvoY216ZzY+8hM+lV53LUayghNRJ0vM=
-sigs.k8s.io/kustomize/api v0.12.1/go.mod h1:y3JUhimkZkR6sbLNwfJHxvo1TCLwuwm14sCYnkH6S1s=
-sigs.k8s.io/kustomize/cmd/config v0.10.9/go.mod h1:T0s850zPV3wKfBALA0dyeP/K74jlJcoP8Pr9ZWwE3MQ=
-sigs.k8s.io/kustomize/kustomize/v4 v4.5.7/go.mod h1:VSNKEH9D9d9bLiWEGbS6Xbg/Ih0tgQalmPvntzRxZ/Q=
-sigs.k8s.io/kustomize/kyaml v0.13.9 h1:Qz53EAaFFANyNgyOEJbT/yoIHygK40/ZcvU3rgry2Tk=
-sigs.k8s.io/kustomize/kyaml v0.13.9/go.mod h1:QsRbD0/KcU+wdk0/L0fIp2KLnohkVzs6fQ85/nOXac4=
+sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 h1:fD1pz4yfdADVNfFmcP2aBEtudwUQ1AlLnRBALr33v3s=
+sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs=
+sigs.k8s.io/kustomize/api v0.10.1 h1:KgU7hfYoscuqag84kxtzKdEC3mKMb99DPI3a0eaV1d0=
+sigs.k8s.io/kustomize/api v0.10.1/go.mod h1:2FigT1QN6xKdcnGS2Ppp1uIWrtWN28Ms8A3OZUZhwr8=
+sigs.k8s.io/kustomize/cmd/config v0.10.2/go.mod h1:K2aW7nXJ0AaT+VA/eO0/dzFLxmpFcTzudmAgDwPY1HQ=
+sigs.k8s.io/kustomize/kustomize/v4 v4.4.1/go.mod h1:qOKJMMz2mBP+vcS7vK+mNz4HBLjaQSWRY22EF6Tb7Io=
+sigs.k8s.io/kustomize/kyaml v0.13.0 h1:9c+ETyNfSrVhxvphs+K2dzT3dh5oVPPEqPOE/cUpScY=
+sigs.k8s.io/kustomize/kyaml v0.13.0/go.mod h1:FTJxEZ86ScK184NpGSAQcfEqee0nul8oLCK30D47m4E=
 sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
+sigs.k8s.io/structured-merge-diff/v4 v4.2.1 h1:bKCqE9GvQ5tiVHn5rfn1r+yao3aLQEaLzkkmAkf+A6Y=
 sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4=
-sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE=
-sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E=
 sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
 sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=
 sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=

indexers/indexer.go

@@ -1,12 +0,0 @@
-// Copyright 2022 Clastix Labs
-// SPDX-License-Identifier: Apache-2.0
-
-package indexers
-
-import "sigs.k8s.io/controller-runtime/pkg/client"
-
-type Indexer interface {
-	Object() client.Object
-	Field() string
-	ExtractValue() client.IndexerFunc
-}

indexers/tcp_useddatastore.go

@@ -1,40 +0,0 @@
-// Copyright 2022 Clastix Labs
-// SPDX-License-Identifier: Apache-2.0
-
-package indexers
-
-import (
-	"context"
-
-	controllerruntime "sigs.k8s.io/controller-runtime"
-	"sigs.k8s.io/controller-runtime/pkg/client"
-
-	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
-)
-
-const (
-	TenantControlPlaneUsedDataStoreKey = "status.storage.dataStoreName"
-)
-
-type TenantControlPlaneStatusDataStore struct{}
-
-func (t *TenantControlPlaneStatusDataStore) Object() client.Object {
-	return &kamajiv1alpha1.TenantControlPlane{}
-}
-
-func (t *TenantControlPlaneStatusDataStore) Field() string {
-	return TenantControlPlaneUsedDataStoreKey
-}
-
-func (t *TenantControlPlaneStatusDataStore) ExtractValue() client.IndexerFunc {
-	return func(object client.Object) []string {
-		//nolint:forcetypeassert
-		tcp := object.(*kamajiv1alpha1.TenantControlPlane)
-
-		return []string{tcp.Status.Storage.DataStoreName}
-	}
-}
-
-func (t *TenantControlPlaneStatusDataStore) SetupWithManager(ctx context.Context, mgr controllerruntime.Manager) error {
-	return mgr.GetFieldIndexer().IndexField(ctx, t.Object(), t.Field(), t.ExtractValue())
-}

internal/builders/controlplane/deployment.go

@@ -18,6 +18,7 @@ import (
 	"k8s.io/utils/pointer"
 
 	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+	"github.com/clastix/kamaji/internal/types"
 	"github.com/clastix/kamaji/internal/utilities"
 )
 
@@ -27,6 +28,7 @@ const (
 	apiServerIndex orderedIndex = iota
 	schedulerIndex
 	controllerManagerIndex
+	kineIndex
 )
 
 const (
@@ -37,19 +39,22 @@ const (
 	usrLocalShareCACertificates
 	schedulerKubeconfig
 	controllerManagerKubeconfig
+	kineConfig
+	kineCerts
 )
 
 const (
 	apiServerFlagsAnnotation = "kube-apiserver.kamaji.clastix.io/args"
-	kineContainerName        = "kine"
-	dataStoreCerts           = "kine-config"
+	kineVolumeName           = "kine-config"
 	kineVolumeCertName       = "kine-certs"
 )
 
 type Deployment struct {
-	Address            string
-	KineContainerImage string
-	DataStore          kamajiv1alpha1.DataStore
+	Address                string
+	ETCDEndpoints          []string
+	ETCDCompactionInterval string
+	ETCDStorageType        types.ETCDStorageType
+	KineContainerImage     string
 }
 
 func (d *Deployment) SetContainers(podSpec *corev1.PodSpec, tcp *kamajiv1alpha1.TenantControlPlane, address string) {
@@ -114,24 +119,19 @@ func (d *Deployment) buildPKIVolume(podSpec *corev1.PodSpec, tcp *kamajiv1alpha1
 		},
 	}
 
-	if d.DataStore.Spec.Driver == kamajiv1alpha1.EtcdDriver {
+	if d.ETCDStorageType == types.ETCD {
+		sources = append(sources, corev1.VolumeProjection{
+			Secret: d.secretProjection(tcp.Status.Certificates.ETCD.APIServer.SecretName, constants.APIServerEtcdClientCertName, constants.APIServerEtcdClientKeyName),
+		})
 		sources = append(sources, corev1.VolumeProjection{
 			Secret: &corev1.SecretProjection{
 				LocalObjectReference: corev1.LocalObjectReference{
-					Name: tcp.Status.Storage.Certificate.SecretName,
+					Name: tcp.Status.Certificates.ETCD.CA.SecretName,
 				},
 				Items: []corev1.KeyToPath{
 					{
-						Key:  "ca.crt",
-						Path: "etcd/ca.crt",
-					},
-					{
-						Key:  "server.crt",
-						Path: "etcd/server.crt",
-					},
-					{
-						Key:  "server.key",
-						Path: "etcd/server.key",
+						Key:  constants.CACertName,
+						Path: constants.EtcdCACertName,
 					},
 				},
 			},
@@ -262,7 +262,7 @@ func (d *Deployment) BuildScheduler(podSpec *corev1.PodSpec, tenantControlPlane
 	args["--authorization-kubeconfig"] = kubeconfig
 	args["--bind-address"] = "0.0.0.0"
 	args["--kubeconfig"] = kubeconfig
-	args["--leader-elect"] = "true" //nolint:goconst
+	args["--leader-elect"] = "true" // nolint:goconst
 
 	podSpec.Containers[schedulerIndex].Name = "kube-scheduler"
 	podSpec.Containers[schedulerIndex].Image = fmt.Sprintf("k8s.gcr.io/kube-scheduler:%s", tenantControlPlane.Spec.Kubernetes.Version)
@@ -289,7 +289,6 @@ func (d *Deployment) BuildScheduler(podSpec *corev1.PodSpec, tenantControlPlane
 		SuccessThreshold:    1,
 		FailureThreshold:    3,
 	}
-
 	podSpec.Containers[schedulerIndex].StartupProbe = &corev1.Probe{
 		ProbeHandler: corev1.ProbeHandler{
 			HTTPGet: &corev1.HTTPGetAction{
@@ -384,7 +383,6 @@ func (d *Deployment) buildControllerManager(podSpec *corev1.PodSpec, tenantContr
 			MountPath: "/usr/local/share/ca-certificates",
 		},
 	}
-
 	podSpec.Containers[controllerManagerIndex].LivenessProbe = &corev1.Probe{
 		ProbeHandler: corev1.ProbeHandler{
 			HTTPGet: &corev1.HTTPGetAction{
@@ -399,7 +397,6 @@ func (d *Deployment) buildControllerManager(podSpec *corev1.PodSpec, tenantContr
 		SuccessThreshold:    1,
 		FailureThreshold:    3,
 	}
-
 	podSpec.Containers[controllerManagerIndex].StartupProbe = &corev1.Probe{
 		ProbeHandler: corev1.ProbeHandler{
 			HTTPGet: &corev1.HTTPGetAction{
@@ -536,6 +533,7 @@ func (d *Deployment) buildKubeAPIServerCommand(tenantControlPlane *kamajiv1alpha
 		"--client-ca-file":                     path.Join(v1beta3.DefaultCertificatesDir, constants.CACertName),
 		"--enable-admission-plugins":           strings.Join(tenantControlPlane.Spec.Kubernetes.AdmissionControllers.ToSlice(), ","),
 		"--enable-bootstrap-token-auth":        "true",
+		"--etcd-servers":                       strings.Join(d.ETCDEndpoints, ","),
 		"--service-cluster-ip-range":           tenantControlPlane.Spec.NetworkProfile.ServiceCIDR,
 		"--kubelet-client-certificate":         path.Join(v1beta3.DefaultCertificatesDir, constants.APIServerKubeletClientCertName),
 		"--kubelet-client-key":                 path.Join(v1beta3.DefaultCertificatesDir, constants.APIServerKubeletClientKeyName),
@@ -554,23 +552,13 @@ func (d *Deployment) buildKubeAPIServerCommand(tenantControlPlane *kamajiv1alpha
 		"--tls-private-key-file":               path.Join(v1beta3.DefaultCertificatesDir, constants.APIServerKeyName),
 	}
 
-	switch d.DataStore.Spec.Driver {
-	case kamajiv1alpha1.KineMySQLDriver, kamajiv1alpha1.KinePostgreSQLDriver:
-		desiredArgs["--etcd-servers"] = "http://127.0.0.1:2379"
-	case kamajiv1alpha1.EtcdDriver:
-		httpsEndpoints := make([]string, 0, len(d.DataStore.Spec.Endpoints))
-
-		for _, ep := range d.DataStore.Spec.Endpoints {
-			httpsEndpoints = append(httpsEndpoints, fmt.Sprintf("https://%s", ep))
-		}
-
+	if d.ETCDStorageType == types.ETCD {
+		desiredArgs["--etcd-compaction-interval"] = d.ETCDCompactionInterval
+		desiredArgs["--etcd-cafile"] = path.Join(v1beta3.DefaultCertificatesDir, constants.EtcdCACertName)
+		desiredArgs["--etcd-certfile"] = path.Join(v1beta3.DefaultCertificatesDir, constants.APIServerEtcdClientCertName)
+		desiredArgs["--etcd-keyfile"] = path.Join(v1beta3.DefaultCertificatesDir, constants.APIServerEtcdClientKeyName)
 		desiredArgs["--etcd-prefix"] = fmt.Sprintf("/%s", tenantControlPlane.GetName())
-		desiredArgs["--etcd-servers"] = strings.Join(httpsEndpoints, ",")
-		desiredArgs["--etcd-cafile"] = "/etc/kubernetes/pki/etcd/ca.crt"
-		desiredArgs["--etcd-certfile"] = "/etc/kubernetes/pki/etcd/server.crt"
-		desiredArgs["--etcd-keyfile"] = "/etc/kubernetes/pki/etcd/server.key"
 	}
-
 	// Order matters, here: extraArgs could try to overwrite some arguments managed by Kamaji and that would be crucial.
 	// Adding as first element of the array of maps, we're sure that these overrides will be sanitized by our configuration.
 	return utilities.MergeMaps(extraArgs, current, desiredArgs)
@@ -594,7 +582,18 @@ func (d *Deployment) secretProjection(secretName, certKeyName, keyName string) *
 	}
 }
 
-func (d *Deployment) removeKineVolumes(podSpec *corev1.PodSpec) {
+func (d *Deployment) buildKineVolume(podSpec *corev1.PodSpec, tcp *kamajiv1alpha1.TenantControlPlane) {
+	// Kine is expecting an additional volume for its configuration, and it must be removed before proceeding with the
+	// customized storage that is idempotent
+	if found, index := utilities.HasNamedVolume(podSpec.Volumes, kineVolumeName); found {
+		var volumes []corev1.Volume
+
+		volumes = append(volumes, podSpec.Volumes[:index]...)
+		volumes = append(volumes, podSpec.Volumes[index+1:]...)
+
+		podSpec.Volumes = volumes
+	}
+
 	if found, index := utilities.HasNamedVolume(podSpec.Volumes, kineVolumeCertName); found {
 		var volumes []corev1.Volume
 
@@ -603,44 +602,37 @@ func (d *Deployment) removeKineVolumes(podSpec *corev1.PodSpec) {
 
 		podSpec.Volumes = volumes
 	}
-}
 
-func (d *Deployment) buildKineVolume(podSpec *corev1.PodSpec, tcp *kamajiv1alpha1.TenantControlPlane) {
-	// Adding the volume for chmod'ed Kine certificates.
-	found, index := utilities.HasNamedVolume(podSpec.Volumes, dataStoreCerts)
-	if !found {
-		podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{})
-		index = len(podSpec.Volumes) - 1
-	}
+	if d.ETCDStorageType == types.KineMySQL || d.ETCDStorageType == types.KinePostgreSQL {
+		if index := int(kineConfig) + 1; len(podSpec.Volumes) < index {
+			podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{})
+		}
+		// Adding the volume to read Kine certificates:
+		// these must be subsequently fixed with a chmod due to pg issues with private key.
+		podSpec.Volumes[kineConfig].Name = kineVolumeName
+		podSpec.Volumes[kineConfig].VolumeSource = corev1.VolumeSource{
+			Secret: &corev1.SecretVolumeSource{
+				SecretName:  tcp.Status.Storage.Kine.Certificate.SecretName,
+				DefaultMode: pointer.Int32Ptr(420),
+			},
+		}
+		// Adding the Volume for the certificates with fixed permission
+		if index := int(kineCerts) + 1; len(podSpec.Volumes) < index {
+			podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{})
+		}
 
-	podSpec.Volumes[index].Name = dataStoreCerts
-	podSpec.Volumes[index].VolumeSource = corev1.VolumeSource{
-		Secret: &corev1.SecretVolumeSource{
-			SecretName:  tcp.Status.Storage.Certificate.SecretName,
-			DefaultMode: pointer.Int32Ptr(420),
-		},
-	}
-	if d.DataStore.Spec.Driver == kamajiv1alpha1.EtcdDriver {
-		d.removeKineVolumes(podSpec)
-
-		return
-	}
-	// Adding the volume to read Kine certificates:
-	// these must be subsequently fixed with a chmod due to pg issues with private key.
-	if found, index = utilities.HasNamedVolume(podSpec.Volumes, kineVolumeCertName); !found {
-		podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{})
-		index = len(podSpec.Volumes) - 1
-	}
-
-	podSpec.Volumes[index].Name = kineVolumeCertName
-	podSpec.Volumes[index].VolumeSource = corev1.VolumeSource{
-		EmptyDir: &corev1.EmptyDirVolumeSource{},
+		podSpec.Volumes[kineCerts].Name = kineVolumeCertName
+		podSpec.Volumes[kineCerts].VolumeSource = corev1.VolumeSource{
+			EmptyDir: &corev1.EmptyDirVolumeSource{},
+		}
 	}
 }
 
-func (d *Deployment) removeKineContainers(podSpec *corev1.PodSpec) {
-	found, index := utilities.HasNamedContainer(podSpec.Containers, kineContainerName)
-	if found {
+func (d *Deployment) buildKine(podSpec *corev1.PodSpec, tcp *kamajiv1alpha1.TenantControlPlane) {
+	const kineContainerName = "kine"
+	// Kine is expecting an additional container, and it must be removed before proceeding with the additional one
+	// in order to make this function idempotent.
+	if found, index := utilities.HasNamedContainer(podSpec.Containers, kineContainerName); found {
 		var containers []corev1.Container
 
 		containers = append(containers, podSpec.Containers[:index]...)
@@ -648,22 +640,13 @@ func (d *Deployment) removeKineContainers(podSpec *corev1.PodSpec) {
 
 		podSpec.Containers = containers
 	}
-
-	podSpec.InitContainers = nil
-}
-
-func (d *Deployment) buildKine(podSpec *corev1.PodSpec, tcp *kamajiv1alpha1.TenantControlPlane) {
-	if d.DataStore.Spec.Driver == kamajiv1alpha1.EtcdDriver {
-		d.removeKineContainers(podSpec)
-
+	// In case of bare ETCD we exit without mangling the PodSpec resource.
+	if d.ETCDStorageType == types.ETCD {
 		return
 	}
-	// Kine is expecting an additional container, and it must be removed before proceeding with the additional one
-	// in order to make this function idempotent.
-	found, index := utilities.HasNamedContainer(podSpec.Containers, kineContainerName)
-	if !found {
+
+	if index := int(kineIndex) + 1; len(podSpec.Containers) < index {
 		podSpec.Containers = append(podSpec.Containers, corev1.Container{})
-		index = len(podSpec.Containers) - 1
 	}
 
 	args := map[string]string{}
@@ -672,11 +655,11 @@ func (d *Deployment) buildKine(podSpec *corev1.PodSpec, tcp *kamajiv1alpha1.Tena
 		args = utilities.ArgsFromSliceToMap(tcp.Spec.ControlPlane.Deployment.ExtraArgs.Kine)
 	}
 
-	switch d.DataStore.Spec.Driver {
-	case kamajiv1alpha1.KineMySQLDriver:
-		args["--endpoint"] = "mysql://$(DB_USER):$(DB_PASSWORD)@tcp($(DB_CONNECTION_STRING))/$(DB_SCHEMA)"
-	case kamajiv1alpha1.KinePostgreSQLDriver:
-		args["--endpoint"] = "postgres://$(DB_USER):$(DB_PASSWORD)@$(DB_CONNECTION_STRING)/$(DB_SCHEMA)"
+	switch d.ETCDStorageType {
+	case types.KineMySQL:
+		args["--endpoint"] = "mysql://$(DB_USER):$(DB_PASSWORD)@tcp($(DB_HOST):$(DB_PORT))/$(DB_SCHEMA)"
+	case types.KinePostgreSQL:
+		args["--endpoint"] = "postgres://$(DB_USER):$(DB_PASSWORD)@$(DB_HOST):$(DB_PORT)/$(DB_SCHEMA)"
 	}
 
 	args["--ca-file"] = "/certs/ca.crt"
@@ -697,7 +680,7 @@ func (d *Deployment) buildKine(podSpec *corev1.PodSpec, tcp *kamajiv1alpha1.Tena
 			},
 			VolumeMounts: []corev1.VolumeMount{
 				{
-					Name:      dataStoreCerts,
+					Name:      kineVolumeName,
 					ReadOnly:  true,
 					MountPath: "/kine",
 				},
@@ -710,42 +693,42 @@ func (d *Deployment) buildKine(podSpec *corev1.PodSpec, tcp *kamajiv1alpha1.Tena
 		},
 	}
 
-	podSpec.Containers[index].Name = kineContainerName
-	podSpec.Containers[index].Image = d.KineContainerImage
-	podSpec.Containers[index].Command = []string{"/bin/kine"}
-	podSpec.Containers[index].Args = utilities.ArgsFromMapToSlice(args)
-	podSpec.Containers[index].VolumeMounts = []corev1.VolumeMount{
+	podSpec.Containers[kineIndex].Name = kineContainerName
+	podSpec.Containers[kineIndex].Image = d.KineContainerImage
+	podSpec.Containers[kineIndex].Command = []string{"/bin/kine"}
+	podSpec.Containers[kineIndex].Args = utilities.ArgsFromMapToSlice(args)
+	podSpec.Containers[kineIndex].VolumeMounts = []corev1.VolumeMount{
 		{
 			Name:      kineVolumeCertName,
 			MountPath: "/certs",
 			ReadOnly:  false,
 		},
 	}
-	podSpec.Containers[index].TerminationMessagePath = corev1.TerminationMessagePathDefault
-	podSpec.Containers[index].TerminationMessagePolicy = corev1.TerminationMessageReadFile
-	podSpec.Containers[index].Env = []corev1.EnvVar{
+	podSpec.Containers[kineIndex].TerminationMessagePath = corev1.TerminationMessagePathDefault
+	podSpec.Containers[kineIndex].TerminationMessagePolicy = corev1.TerminationMessageReadFile
+	podSpec.Containers[kineIndex].Env = []corev1.EnvVar{
 		{
 			Name:  "GODEBUG",
 			Value: "x509ignoreCN=0",
 		},
 	}
-	podSpec.Containers[index].EnvFrom = []corev1.EnvFromSource{
+	podSpec.Containers[kineIndex].EnvFrom = []corev1.EnvFromSource{
 		{
 			SecretRef: &corev1.SecretEnvSource{
 				LocalObjectReference: corev1.LocalObjectReference{
-					Name: tcp.Status.Storage.Config.SecretName,
+					Name: tcp.Status.Storage.Kine.Config.SecretName,
 				},
 			},
 		},
 	}
-	podSpec.Containers[index].Ports = []corev1.ContainerPort{
+	podSpec.Containers[kineIndex].Ports = []corev1.ContainerPort{
 		{
 			ContainerPort: 2379,
 			Name:          "server",
 			Protocol:      corev1.ProtocolTCP,
 		},
 	}
-	podSpec.Containers[index].ImagePullPolicy = corev1.PullAlways
+	podSpec.Containers[kineIndex].ImagePullPolicy = corev1.PullAlways
 }
 
 func (d *Deployment) SetSelector(deploymentSpec *appsv1.DeploymentSpec, tcp *kamajiv1alpha1.TenantControlPlane) {
@@ -772,18 +755,6 @@ func (d *Deployment) SetAnnotations(resource *appsv1.Deployment, annotations map
 	resource.SetAnnotations(annotations)
 }
 
-func (d *Deployment) SetTopologySpreadConstraints(spec *appsv1.DeploymentSpec, topologies []corev1.TopologySpreadConstraint) {
-	defaultSelector := spec.Selector
-
-	for index, topology := range topologies {
-		if topology.LabelSelector == nil {
-			topologies[index].LabelSelector = defaultSelector
-		}
-	}
-
-	spec.Template.Spec.TopologySpreadConstraints = topologies
-}
-
 // ResetKubeAPIServerFlags ensures that upon a change of the kube-apiserver extra flags the desired ones are properly
 // applied, also considering that the container could be lately patched by the konnectivity addon resources.
 func (d *Deployment) ResetKubeAPIServerFlags(resource *appsv1.Deployment, tcp *kamajiv1alpha1.TenantControlPlane) {

internal/config/config.go

@@ -21,10 +21,18 @@ var (
 )
 
 const (
-	envPrefix           = "KAMAJI"
-	defaultTmpDirectory = "/tmp/kamaji"
-	defaultKineImage    = "rancher/kine:v0.9.2-amd64"
-	defaultDataStore    = "etcd"
+	envPrefix                        = "KAMAJI"
+	defaultETCDStorageType           = "etcd"
+	defaultETCDCASecretName          = "etcd-certs"
+	defaultETCDCASecretNamespace     = "kamaji-system"
+	defaultETCDEndpoints             = "etcd-server:2379"
+	defaultETCDCompactionInterval    = "0"
+	defaultETCDClientSecretName      = "root-client-certs"
+	defaultETCDClientSecretNamespace = "kamaji-system"
+	defaultTmpDirectory              = "/tmp/kamaji"
+	defaultKineSecretName            = "kine-secret"
+	defaultKineSecretNamespace       = "kamaji-system"
+	defaultKineImage                 = "rancher/kine:v0.9.2-amd64"
 )
 
 func InitConfig() (*viper.Viper, error) {
@@ -36,12 +44,21 @@ func InitConfig() (*viper.Viper, error) {
 	flag.String("health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
 	flag.Bool("leader-elect", false, "Enable leader election for controller manager. "+
 		"Enabling this will ensure there is only one active controller manager.")
+	flag.String("etcd-storage-type", defaultETCDStorageType, "Type of storage for ETCD (i.e etcd, kine-mysql, kine-psql)")
+	flag.String("etcd-ca-secret-name", defaultETCDCASecretName, "Name of the secret which contains CA's certificate and private key.")
+	flag.String("etcd-ca-secret-namespace", defaultETCDCASecretNamespace, "Namespace of the secret which contains CA's certificate and private key.")
+	flag.String("etcd-client-secret-name", defaultETCDClientSecretName, "Name of the secret which contains ETCD client certificates")
+	flag.String("etcd-client-secret-namespace", defaultETCDClientSecretNamespace, "Name of the namespace where the secret which contains ETCD client certificates is")
+	flag.String("etcd-endpoints", defaultETCDEndpoints, "Comma-separated list with ETCD endpoints (i.e. https://etcd-0.etcd.kamaji-system.svc.cluster.local,https://etcd-1.etcd.kamaji-system.svc.cluster.local,https://etcd-2.etcd.kamaji-system.svc.cluster.local)")
+	flag.String("etcd-compaction-interval", defaultETCDCompactionInterval, "ETCD Compaction interval (i.e. \"5m0s\"). (default: \"0\" (disabled))")
 	flag.String("tmp-directory", defaultTmpDirectory, "Directory which will be used to work with temporary files.")
+	flag.String("kine-secret-name", defaultKineSecretName, "Name of the secret which contains the Kine configuration.")
+	flag.String("kine-secret-namespace", defaultKineSecretNamespace, "Name of the namespace where the secret which contains the Kine configuration.")
+	flag.String("kine-host", "", "Host where the DB used by Kine is working.")
+	flag.Int("kine-port", 0, "Port where the DB used by Kine is listening to.")
 	flag.String("kine-image", defaultKineImage, "Container image along with tag to use for the Kine sidecar container (used only if etcd-storage-type is set to one of kine strategies)")
-	flag.String("datastore", defaultDataStore, "The default DataStore that should be used by Kamaji to setup the required storage")
 
 	// Setup zap configuration
-
 	opts := zap.Options{
 		Development: true,
 	}
@@ -67,13 +84,43 @@ func InitConfig() (*viper.Viper, error) {
 	if err := config.BindEnv("leader-elect", fmt.Sprintf("%s_LEADER_ELECTION", envPrefix)); err != nil {
 		return nil, err
 	}
+	if err := config.BindEnv("etcd-storage-type", fmt.Sprintf("%s_ETCD_STORAGE_TYPE", envPrefix)); err != nil {
+		return nil, err
+	}
+	if err := config.BindEnv("etcd-ca-secret-name", fmt.Sprintf("%s_ETCD_CA_SECRET_NAME", envPrefix)); err != nil {
+		return nil, err
+	}
+	if err := config.BindEnv("etcd-ca-secret-namespace", fmt.Sprintf("%s_ETCD_CA_SECRET_NAMESPACE", envPrefix)); err != nil {
+		return nil, err
+	}
+	if err := config.BindEnv("etcd-client-secret-name", fmt.Sprintf("%s_ETCD_CLIENT_SECRET_NAME", envPrefix)); err != nil {
+		return nil, err
+	}
+	if err := config.BindEnv("etcd-client-secret-namespace", fmt.Sprintf("%s_ETCD_CLIENT_SECRET_NAMESPACE", envPrefix)); err != nil {
+		return nil, err
+	}
+	if err := config.BindEnv("etcd-endpoints", fmt.Sprintf("%s_ETCD_ENDPOINTS", envPrefix)); err != nil {
+		return nil, err
+	}
+	if err := config.BindEnv("etcd-compaction-interval", fmt.Sprintf("%s_ETCD_COMPACTION_INTERVAL", envPrefix)); err != nil {
+		return nil, err
+	}
 	if err := config.BindEnv("tmp-directory", fmt.Sprintf("%s_TMP_DIRECTORY", envPrefix)); err != nil {
 		return nil, err
 	}
-	if err := config.BindEnv("kine-image", fmt.Sprintf("%s_KINE_IMAGE", envPrefix)); err != nil {
+	if err := config.BindEnv("kine-secret-name", fmt.Sprintf("%s_KINE_SECRET_NAME", envPrefix)); err != nil {
 		return nil, err
 	}
-	if err := config.BindEnv("datastore", fmt.Sprintf("%s_DATASTORE", envPrefix)); err != nil {
+	if err := config.BindEnv("kine-secret-namespace", fmt.Sprintf("%s_KINE_SECRET_NAMESPACE", envPrefix)); err != nil {
+		return nil, err
+	}
+	if err := config.BindEnv("kine-host", fmt.Sprintf("%s_KINE_HOST", envPrefix)); err != nil {
+		return nil, err
+	}
+	if err := config.BindEnv("kine-port", fmt.Sprintf("%s_KINE_PORT", envPrefix)); err != nil {
+		return nil, err
+	}
+	if err := config.BindEnv("kine-image", fmt.Sprintf("%s_KINE_IMAGE", envPrefix)); err != nil {
 		return nil, err
 	}
 

internal/crypto/crypto.go

@@ -60,7 +60,7 @@ func GetPublickKey(pubKey []byte) (*rsa.PublicKey, error) {
 		return nil, err
 	}
 
-	return pub.(*rsa.PublicKey), nil //nolint:forcetypeassert
+	return pub.(*rsa.PublicKey), nil // nolint:forcetypeassert
 }
 
 func GenerateCertificateKeyPairBytes(template *x509.Certificate, bitSize int, caCert *x509.Certificate, caKey *rsa.PrivateKey) (*bytes.Buffer, *bytes.Buffer, error) {
@@ -76,18 +76,16 @@ func GenerateCertificateKeyPairBytes(template *x509.Certificate, bitSize int, ca
 
 	certPEM := &bytes.Buffer{}
 	if err := pem.Encode(certPEM, &pem.Block{
-		Type:    "CERTIFICATE",
-		Headers: nil,
-		Bytes:   certBytes,
+		Type:  "CERTIFICATE",
+		Bytes: certBytes,
 	}); err != nil {
 		return nil, nil, err
 	}
 
 	certPrivKeyPEM := &bytes.Buffer{}
 	if err := pem.Encode(certPrivKeyPEM, &pem.Block{
-		Type:    "RSA PRIVATE KEY",
-		Headers: nil,
-		Bytes:   x509.MarshalPKCS1PrivateKey(certPrivKey),
+		Type:  "RSA PRIVATE KEY",
+		Bytes: x509.MarshalPKCS1PrivateKey(certPrivKey),
 	}); err != nil {
 		return nil, nil, err
 	}
@@ -134,7 +132,7 @@ func checkCertificateValidity(cert x509.Certificate) bool {
 }
 
 func checkCertificateKeyPair(cert x509.Certificate, privKey rsa.PrivateKey) bool {
-	return checkPublicKeys(*cert.PublicKey.(*rsa.PublicKey), privKey.PublicKey) //nolint:forcetypeassert
+	return checkPublicKeys(*cert.PublicKey.(*rsa.PublicKey), privKey.PublicKey) // nolint:forcetypeassert
 }
 
 func checkPublicKeys(a rsa.PublicKey, b rsa.PublicKey) bool {

internal/datastore/datastore.go

@@ -1,56 +0,0 @@
-// Copyright 2022 Clastix Labs
-// SPDX-License-Identifier: Apache-2.0
-
-package datastore
-
-import (
-	"context"
-	"crypto/tls"
-	"fmt"
-)
-
-type ConnectionEndpoint struct {
-	Host string
-	Port int
-}
-
-func (r ConnectionEndpoint) String() string {
-	return fmt.Sprintf("%s:%d", r.Host, r.Port)
-}
-
-type ConnectionConfig struct {
-	User       string
-	Password   string
-	Endpoints  []ConnectionEndpoint
-	DBName     string
-	TLSConfig  *tls.Config
-	Parameters map[string][]string
-}
-
-func (config ConnectionConfig) getDataSourceNameUserPassword() string {
-	if config.User == "" {
-		return ""
-	}
-
-	if config.Password == "" {
-		return fmt.Sprintf("%s@", config.User)
-	}
-
-	return fmt.Sprintf("%s:%s@", config.User, config.Password)
-}
-
-type Connection interface {
-	CreateUser(ctx context.Context, user, password string) error
-	CreateDB(ctx context.Context, dbName string) error
-	GrantPrivileges(ctx context.Context, user, dbName string) error
-	UserExists(ctx context.Context, user string) (bool, error)
-	DBExists(ctx context.Context, dbName string) (bool, error)
-	GrantPrivilegesExists(ctx context.Context, user, dbName string) (bool, error)
-	DeleteUser(ctx context.Context, user string) error
-	DeleteDB(ctx context.Context, dbName string) error
-	RevokePrivileges(ctx context.Context, user, dbName string) error
-	GetConnectionString() string
-	Close() error
-	Check(ctx context.Context) error
-	Driver() string
-}

internal/datastore/etcd.go

@@ -1,171 +0,0 @@
-// Copyright 2022 Clastix Labs
-// SPDX-License-Identifier: Apache-2.0
-
-package datastore
-
-import (
-	"context"
-	"fmt"
-
-	"github.com/pkg/errors"
-	"go.etcd.io/etcd/api/v3/authpb"
-	"go.etcd.io/etcd/api/v3/v3rpc/rpctypes"
-	etcdclient "go.etcd.io/etcd/client/v3"
-
-	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
-)
-
-const (
-	// rangeEnd is the key following the last key of the range.
-	// If rangeEnd is ‘\0’, the range is all keys greater than or equal to the key argument
-	// source: https://etcd.io/docs/v3.5/learning/api/
-	rangeEnd = "\\0"
-)
-
-func NewETCDConnection(config ConnectionConfig) (Connection, error) {
-	endpoints := make([]string, 0, len(config.Endpoints))
-
-	for _, ep := range config.Endpoints {
-		endpoints = append(endpoints, ep.String())
-	}
-
-	cfg := etcdclient.Config{
-		Endpoints: endpoints,
-		TLS:       config.TLSConfig,
-	}
-
-	client, err := etcdclient.New(cfg)
-	if err != nil {
-		return nil, err
-	}
-
-	return &EtcdClient{
-		Client: *client,
-	}, nil
-}
-
-type EtcdClient struct {
-	Client etcdclient.Client
-}
-
-func (e *EtcdClient) CreateUser(ctx context.Context, user, password string) error {
-	_, err := e.Client.Auth.UserAddWithOptions(ctx, user, password, &etcdclient.UserAddOptions{
-		NoPassword: true,
-	})
-
-	return err
-}
-
-func (e *EtcdClient) CreateDB(ctx context.Context, dbName string) error {
-	return nil
-}
-
-func (e *EtcdClient) GrantPrivileges(ctx context.Context, user, dbName string) error {
-	_, err := e.Client.Auth.RoleAdd(ctx, dbName)
-	if err != nil {
-		return err
-	}
-
-	permission := etcdclient.PermissionType(authpb.READWRITE)
-	key := e.buildKey(dbName)
-	if _, err = e.Client.RoleGrantPermission(ctx, user, key, rangeEnd, permission); err != nil {
-		return err
-	}
-
-	if _, err = e.Client.UserGrantRole(ctx, user, dbName); err != nil {
-		return err
-	}
-
-	return err
-}
-
-func (e *EtcdClient) UserExists(ctx context.Context, user string) (bool, error) {
-	_, err := e.Client.UserGet(ctx, user)
-	if err != nil {
-		if errors.As(err, &rpctypes.ErrGRPCUserNotFound) {
-			return false, nil
-		}
-
-		return false, err
-	}
-
-	return true, nil
-}
-
-func (e *EtcdClient) DBExists(_ context.Context, dbName string) (bool, error) {
-	return true, nil
-}
-
-func (e *EtcdClient) GrantPrivilegesExists(ctx context.Context, username, dbName string) (bool, error) {
-	_, err := e.Client.RoleGet(ctx, dbName)
-	if err != nil {
-		if errors.As(err, &rpctypes.ErrGRPCRoleNotFound) {
-			return false, nil
-		}
-
-		return false, err
-	}
-
-	user, err := e.Client.UserGet(ctx, username)
-	if err != nil {
-		return false, err
-	}
-
-	for _, i := range user.Roles {
-		if i == dbName {
-			return true, nil
-		}
-	}
-
-	return false, nil
-}
-
-func (e *EtcdClient) DeleteUser(ctx context.Context, user string) error {
-	_, err := e.Client.Auth.UserDelete(ctx, user)
-
-	return err
-}
-
-func (e *EtcdClient) DeleteDB(ctx context.Context, dbName string) error {
-	withRange := etcdclient.WithRange(rangeEnd)
-	prefix := e.buildKey(dbName)
-	_, err := e.Client.Delete(ctx, prefix, withRange)
-
-	return err
-}
-
-func (e *EtcdClient) RevokePrivileges(ctx context.Context, user, dbName string) error {
-	_, err := e.Client.Auth.RoleDelete(ctx, dbName)
-
-	return err
-}
-
-func (e *EtcdClient) GetConnectionString() string {
-	// There's no need for connection string in etcd client:
-	// it's not used by Kine
-	return ""
-}
-
-func (e *EtcdClient) Close() error {
-	return e.Client.Close()
-}
-
-func (e *EtcdClient) Check(ctx context.Context) error {
-	_, err := e.Client.AuthStatus(ctx)
-
-	return err
-}
-
-func (e *EtcdClient) Driver() string {
-	return string(kamajiv1alpha1.EtcdDriver)
-}
-
-func (e *EtcdClient) buildKey(roleName string) string {
-	return fmt.Sprintf("/%s/", roleName)
-}
-
-type Permission struct {
-	Type     int    `json:"type,omitempty"`
-	Key      string `json:"key,omitempty"`
-	RangeEnd string `json:"rangeEnd,omitempty"`
-}

internal/etcd/api.go

@@ -0,0 +1,177 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package etcd
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"errors"
+	"fmt"
+	"time"
+
+	"go.etcd.io/etcd/api/v3/authpb"
+	"go.etcd.io/etcd/api/v3/v3rpc/rpctypes"
+	etcdclient "go.etcd.io/etcd/client/v3"
+	"google.golang.org/grpc/codes"
+)
+
+const (
+	etcdTimeout = 10 // seconds
+
+	//  rangeEnd is the key following the last key of the range.
+	//  If rangeEnd is ‘\0’, the range is all keys greater than or equal to the key argument
+	//  source: https://etcd.io/docs/v3.5/learning/api/
+	rangeEnd = "\\0"
+)
+
+func NewClient(config Config) (*etcdclient.Client, error) {
+	cert, err := tls.X509KeyPair(config.ETCDCertificate, config.ETCDPrivateKey)
+	if err != nil {
+		return nil, err
+	}
+
+	pool := x509.NewCertPool()
+	pool.AppendCertsFromPEM(config.ETCDCA)
+
+	cfg := etcdclient.Config{
+		Endpoints: config.Endpoints,
+		TLS: &tls.Config{ // nolint:gosec
+			Certificates: []tls.Certificate{cert},
+			RootCAs:      pool,
+		},
+	}
+
+	return etcdclient.New(cfg)
+}
+
+func GetUser(ctx context.Context, client *etcdclient.Client, user *User) error {
+	ctxWithTimeout, cancel := context.WithTimeout(ctx, etcdTimeout*time.Second)
+	defer cancel()
+
+	response, err := client.UserGet(ctxWithTimeout, user.Name)
+	if err != nil {
+		var etcdError rpctypes.EtcdError
+		if errors.As(err, &etcdError) && etcdError.Code() == codes.FailedPrecondition {
+			return nil
+		}
+
+		return err
+	}
+
+	user.Roles = response.Roles
+	user.Exists = true
+
+	return nil
+}
+
+func AddUser(ctx context.Context, client *etcdclient.Client, username string) error {
+	ctxWithTimeout, cancel := getContextWithTimeout(ctx)
+	defer cancel()
+	opts := etcdclient.UserAddOptions{
+		NoPassword: true,
+	}
+	_, err := client.Auth.UserAddWithOptions(ctxWithTimeout, username, "", &opts)
+
+	return err
+}
+
+func RemoveUser(ctx context.Context, client *etcdclient.Client, username string) error {
+	ctxWithTimeout, cancel := getContextWithTimeout(ctx)
+	defer cancel()
+
+	_, err := client.Auth.UserDelete(ctxWithTimeout, username)
+
+	return err
+}
+
+func GetRole(ctx context.Context, client *etcdclient.Client, role *Role) error {
+	ctxWithTimeout, cancel := getContextWithTimeout(ctx)
+	defer cancel()
+
+	response, err := client.RoleGet(ctxWithTimeout, role.Name)
+	if err != nil {
+		var etcdError rpctypes.EtcdError
+		if errors.As(err, &etcdError) && etcdError.Code() == codes.FailedPrecondition {
+			return nil
+		}
+
+		return err
+	}
+
+	role.Exists = true
+	for _, perm := range response.Perm {
+		permission := Permission{
+			Type:     int(perm.PermType),
+			Key:      string(perm.Key),
+			RangeEnd: string(perm.RangeEnd),
+		}
+		role.Permissions = append(role.Permissions, permission)
+	}
+
+	return nil
+}
+
+func AddRole(ctx context.Context, client *etcdclient.Client, roleName string) error {
+	ctxWithTimeout, cancel := getContextWithTimeout(ctx)
+	defer cancel()
+
+	_, err := client.Auth.RoleAdd(ctxWithTimeout, roleName)
+
+	return err
+}
+
+func RemoveRole(ctx context.Context, client *etcdclient.Client, roleName string) error {
+	ctxWithTimeout, cancel := getContextWithTimeout(ctx)
+	defer cancel()
+
+	_, err := client.Auth.RoleDelete(ctxWithTimeout, roleName)
+
+	return err
+}
+
+func GrantUserRole(ctx context.Context, client *etcdclient.Client, user User, role Role) error {
+	ctxWithTimeout, cancel := getContextWithTimeout(ctx)
+	defer cancel()
+
+	_, err := client.UserGrantRole(ctxWithTimeout, user.Name, role.Name)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func GrantRolePermission(ctx context.Context, client *etcdclient.Client, role Role) error {
+	ctxWithTimeout, cancel := getContextWithTimeout(ctx)
+	defer cancel()
+
+	permission := etcdclient.PermissionType(authpb.READWRITE)
+	key := BuildKey(role.Name)
+	_, err := client.RoleGrantPermission(ctxWithTimeout, role.Name, key, rangeEnd, permission)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func CleanUpPrefix(ctx context.Context, client *etcdclient.Client, name string) error {
+	ctxWithTimeout, cancel := getContextWithTimeout(ctx)
+	defer cancel()
+
+	withRange := etcdclient.WithRange(rangeEnd)
+	prefix := BuildKey(name)
+	_, err := client.Delete(ctxWithTimeout, prefix, withRange)
+
+	return err
+}
+
+func BuildKey(roleName string) string {
+	return fmt.Sprintf("/%s/", roleName)
+}
+
+func getContextWithTimeout(ctx context.Context) (context.Context, context.CancelFunc) {
+	return context.WithTimeout(ctx, etcdTimeout*time.Second)
+}

internal/etcd/types.go

@@ -0,0 +1,71 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package etcd
+
+type Config struct {
+	ETCDCertificate []byte
+	ETCDPrivateKey  []byte
+	ETCDCA          []byte
+	Endpoints       []string
+}
+
+type Permission struct {
+	Type     int    `json:"type,omitempty"`
+	Key      string `json:"key,omitempty"`
+	RangeEnd string `json:"rangeEnd,omitempty"`
+}
+
+func (in *Permission) DeepCopyInto(out *Permission) {
+	*out = *in
+}
+
+func (in *Permission) DeepCopy() *Permission {
+	if in == nil {
+		return nil
+	}
+	out := new(Permission)
+	in.DeepCopyInto(out)
+
+	return out
+}
+
+type Role struct {
+	Name        string       `json:"name"`
+	Permissions []Permission `json:"permissions,omitempty"`
+	Exists      bool         `json:"exists"`
+}
+
+func (in *Role) DeepCopyInto(out *Role) {
+	*out = *in
+}
+
+func (in *Role) DeepCopy() *Role {
+	if in == nil {
+		return nil
+	}
+	out := new(Role)
+	in.DeepCopyInto(out)
+
+	return out
+}
+
+type User struct {
+	Name   string   `json:"name"`
+	Roles  []string `json:"roles,omitempty"`
+	Exists bool     `json:"exists"`
+}
+
+func (in *User) DeepCopyInto(out *User) {
+	*out = *in
+}
+
+func (in *User) DeepCopy() *User {
+	if in == nil {
+		return nil
+	}
+	out := new(User)
+	in.DeepCopyInto(out)
+
+	return out
+}

internal/kubeadm/addon.go

@@ -34,13 +34,6 @@ const (
 )
 
 func AddCoreDNS(client kubernetes.Interface, config *Configuration) error {
-	// We're passing the values from the parameters here because they wouldn't be hashed by the YAML encoder:
-	// the struct kubeadm.ClusterConfiguration hasn't struct tags, and it wouldn't be hashed properly.
-	if opts := config.Parameters.CoreDNSOptions; opts != nil {
-		config.InitConfiguration.DNS.ImageRepository = opts.Repository
-		config.InitConfiguration.DNS.ImageTag = opts.Tag
-	}
-
 	return dns.EnsureDNSAddon(&config.InitConfiguration.ClusterConfiguration, client)
 }
 
@@ -87,7 +80,6 @@ func removeCoreDNSDeployment(ctx context.Context, client kubernetes.Interface) e
 
 func removeCoreDNSConfigMap(ctx context.Context, client kubernetes.Interface) error {
 	name, _ := getCoreDNSConfigMapName(ctx)
-
 	opts := metav1.DeleteOptions{}
 
 	return client.CoreV1().ConfigMaps(kubeSystemNamespace).Delete(ctx, name, opts)
@@ -120,9 +112,7 @@ func AddKubeProxy(client kubernetes.Interface, config *Configuration) error {
 		return err
 	}
 
-	image := fmt.Sprintf("%s/kube-proxy:%s", config.Parameters.KubeProxyOptions.Repository, config.Parameters.KubeProxyOptions.Tag)
-
-	if err := createKubeProxyAddon(client, image); err != nil {
+	if err := createKubeProxyAddon(client); err != nil {
 		return err
 	}
 
@@ -162,7 +152,6 @@ func RemoveKubeProxy(ctx context.Context, client kubernetes.Interface) error {
 
 func removeKubeProxyDaemonSet(ctx context.Context, client kubernetes.Interface) error {
 	name, _ := getKubeProxyDaemonSetName(ctx)
-
 	opts := metav1.DeleteOptions{}
 
 	return client.AppsV1().DaemonSets(kubeSystemNamespace).Delete(ctx, name, opts)
@@ -170,7 +159,6 @@ func removeKubeProxyDaemonSet(ctx context.Context, client kubernetes.Interface)
 
 func removeKubeProxyConfigMap(ctx context.Context, client kubernetes.Interface) error {
 	name, _ := getKubeProxyConfigMapName(ctx)
-
 	opts := metav1.DeleteOptions{}
 
 	return client.CoreV1().ConfigMaps(kubeSystemNamespace).Delete(ctx, name, opts)
@@ -179,7 +167,6 @@ func removeKubeProxyConfigMap(ctx context.Context, client kubernetes.Interface)
 func removeKubeProxyRBAC(ctx context.Context, client kubernetes.Interface) error {
 	// TODO: Currently, kube-proxy is installed using kubeadm phases, therefore, name is the same.
 	name, _ := getKubeProxyRBACName(ctx)
-
 	opts := metav1.DeleteOptions{}
 	var result error
 
@@ -253,7 +240,7 @@ func createKubeProxyConfigMap(client kubernetes.Interface, config *Configuration
 	return apiclient.CreateOrUpdateConfigMap(client, configMap)
 }
 
-func createKubeProxyAddon(client kubernetes.Interface, image string) error {
+func createKubeProxyAddon(client kubernetes.Interface) error {
 	daemonSet := &appsv1.DaemonSet{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      "kube-proxy",
@@ -294,7 +281,7 @@ func createKubeProxyAddon(client kubernetes.Interface, image string) error {
 									},
 								},
 							},
-							Image:           image,
+							Image:           "k8s.gcr.io/kube-proxy:v1.21.2",
 							ImagePullPolicy: corev1.PullIfNotPresent,
 							Name:            "kube-proxy",
 							SecurityContext: &corev1.SecurityContext{
@@ -374,7 +361,6 @@ func createKubeProxyAddon(client kubernetes.Interface, image string) error {
 func getKubeproxyConfigmapContent(config *Configuration) ([]byte, error) {
 	zeroDuration := metav1.Duration{Duration: 0}
 	oneSecondDuration := metav1.Duration{Duration: time.Second}
-
 	kubeProxyConfiguration := kubeproxyconfig.KubeProxyConfiguration{
 		TypeMeta: metav1.TypeMeta{
 			Kind:       "KubeProxyConfiguration",

internal/kubeadm/certificates.go

@@ -7,6 +7,7 @@ import (
 	"crypto"
 	"crypto/x509"
 	"fmt"
+	"io/ioutil"
 	"os"
 	"path/filepath"
 
@@ -146,7 +147,7 @@ func readCertificateFiles(name string, directory string, extensions ...string) (
 	for _, extension := range extensions {
 		fileName := fmt.Sprintf("%s.%s", name, extension)
 		path := filepath.Join(directory, fileName)
-		content, err := os.ReadFile(path)
+		content, err := ioutil.ReadFile(path)
 		if err != nil {
 			return nil, err
 		}
@@ -159,6 +160,6 @@ func readCertificateFiles(name string, directory string, extensions ...string) (
 func deleteCertificateDirectory(certificateDirectory string) {
 	if err := os.RemoveAll(certificateDirectory); err != nil {
 		// TODO(prometherion): we should log rather than printing to stdout
-		fmt.Printf("Error removing %s: %s", certificateDirectory, err.Error()) //nolint:forbidigo
+		fmt.Printf("Error removing %s: %s", certificateDirectory, err.Error()) // nolint:forbidigo
 	}
 }

internal/kubeadm/configuration.go

@@ -90,7 +90,7 @@ func getKubeadmClusterConfiguration(params Parameters) kubeadmapi.ClusterConfigu
 			}, params.TenantControlPlaneCertSANs...),
 			ControlPlaneComponent: kubeadmapi.ControlPlaneComponent{
 				ExtraArgs: map[string]string{
-					"etcd-compaction-interval": "0s",
+					"etcd-compaction-interval": params.ETCDCompactionInterval,
 					"etcd-prefix":              fmt.Sprintf("/%s", params.TenantControlPlaneName),
 				},
 			},

internal/kubeadm/kubeconfig.go

@@ -4,6 +4,7 @@
 package kubeadm
 
 import (
+	"io/ioutil"
 	"os"
 	"path"
 	"path/filepath"
@@ -18,12 +19,12 @@ func buildCertificateDirectoryWithCA(ca CertificatePrivateKeyPair, directory str
 	}
 
 	certPath := path.Join(directory, kubeadmconstants.CACertName)
-	if err := os.WriteFile(certPath, ca.Certificate, os.FileMode(0o600)); err != nil {
+	if err := ioutil.WriteFile(certPath, ca.Certificate, os.FileMode(0o600)); err != nil {
 		return err
 	}
 
 	keyPath := path.Join(directory, kubeadmconstants.CAKeyName)
-	if err := os.WriteFile(keyPath, ca.PrivateKey, os.FileMode(0o600)); err != nil {
+	if err := ioutil.WriteFile(keyPath, ca.PrivateKey, os.FileMode(0o600)); err != nil {
 		return err
 	}
 
@@ -43,7 +44,7 @@ func CreateKubeconfig(kubeconfigName string, ca CertificatePrivateKeyPair, confi
 
 	path := filepath.Join(config.InitConfiguration.CertificatesDir, kubeconfigName)
 
-	return os.ReadFile(path)
+	return ioutil.ReadFile(path)
 }
 
 func IsKubeconfigValid(kubeconfigBytes []byte) bool {

internal/kubeadm/types.go

@@ -44,15 +44,9 @@ type Parameters struct {
 	TenantControlPlaneVersion      string
 	TenantControlPlaneCGroupDriver string
 	ETCDs                          []string
+	ETCDCompactionInterval         string
 	CertificatesDir                string
 	KubeconfigDir                  string
-	KubeProxyOptions               *AddonOptions
-	CoreDNSOptions                 *AddonOptions
-}
-
-type AddonOptions struct {
-	Repository string
-	Tag        string
 }
 
 type KubeletConfiguration struct {

internal/resources/api_server_certificate.go

@@ -25,22 +25,23 @@ type APIServerCertificate struct {
 	resource     *corev1.Secret
 	Client       client.Client
 	Log          logr.Logger
+	Name         string
 	TmpDirectory string
 }
 
-func (r *APIServerCertificate) ShouldStatusBeUpdated(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
+func (r *APIServerCertificate) ShouldStatusBeUpdated(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
 	return tenantControlPlane.Status.Certificates.APIServer.Checksum != r.resource.GetAnnotations()["checksum"]
 }
 
-func (r *APIServerCertificate) ShouldCleanup(_ *kamajiv1alpha1.TenantControlPlane) bool {
+func (r *APIServerCertificate) ShouldCleanup(plane *kamajiv1alpha1.TenantControlPlane) bool {
 	return false
 }
 
-func (r *APIServerCertificate) CleanUp(context.Context, *kamajiv1alpha1.TenantControlPlane) (bool, error) {
+func (r *APIServerCertificate) CleanUp(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) {
 	return false, nil
 }
 
-func (r *APIServerCertificate) Define(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+func (r *APIServerCertificate) Define(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
 	r.resource = &corev1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      r.getPrefixedName(tenantControlPlane),
@@ -52,7 +53,7 @@ func (r *APIServerCertificate) Define(_ context.Context, tenantControlPlane *kam
 }
 
 func (r *APIServerCertificate) getPrefixedName(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) string {
-	return utilities.AddTenantPrefix(r.GetName(), tenantControlPlane)
+	return utilities.AddTenantPrefix(r.Name, tenantControlPlane)
 }
 
 func (r *APIServerCertificate) GetClient() client.Client {
@@ -68,10 +69,10 @@ func (r *APIServerCertificate) CreateOrUpdate(ctx context.Context, tenantControl
 }
 
 func (r *APIServerCertificate) GetName() string {
-	return "api-server-certificate"
+	return r.Name
 }
 
-func (r *APIServerCertificate) UpdateTenantControlPlaneStatus(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+func (r *APIServerCertificate) UpdateTenantControlPlaneStatus(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
 	tenantControlPlane.Status.Certificates.APIServer.LastUpdate = metav1.Now()
 	tenantControlPlane.Status.Certificates.APIServer.SecretName = r.resource.GetName()
 	tenantControlPlane.Status.Certificates.APIServer.Checksum = r.resource.GetAnnotations()["checksum"]

internal/resources/api_server_kubelet_client_certificate.go

@@ -25,22 +25,23 @@ type APIServerKubeletClientCertificate struct {
 	resource     *corev1.Secret
 	Client       client.Client
 	Log          logr.Logger
+	Name         string
 	TmpDirectory string
 }
 
-func (r *APIServerKubeletClientCertificate) ShouldStatusBeUpdated(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
+func (r *APIServerKubeletClientCertificate) ShouldStatusBeUpdated(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
 	return tenantControlPlane.Status.Certificates.APIServerKubeletClient.Checksum != r.resource.GetAnnotations()["checksum"]
 }
 
-func (r *APIServerKubeletClientCertificate) ShouldCleanup(*kamajiv1alpha1.TenantControlPlane) bool {
+func (r *APIServerKubeletClientCertificate) ShouldCleanup(plane *kamajiv1alpha1.TenantControlPlane) bool {
 	return false
 }
 
-func (r *APIServerKubeletClientCertificate) CleanUp(context.Context, *kamajiv1alpha1.TenantControlPlane) (bool, error) {
+func (r *APIServerKubeletClientCertificate) CleanUp(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) {
 	return false, nil
 }
 
-func (r *APIServerKubeletClientCertificate) Define(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+func (r *APIServerKubeletClientCertificate) Define(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
 	r.resource = &corev1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      r.getPrefixedName(tenantControlPlane),
@@ -52,7 +53,7 @@ func (r *APIServerKubeletClientCertificate) Define(_ context.Context, tenantCont
 }
 
 func (r *APIServerKubeletClientCertificate) getPrefixedName(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) string {
-	return utilities.AddTenantPrefix(r.GetName(), tenantControlPlane)
+	return utilities.AddTenantPrefix(r.Name, tenantControlPlane)
 }
 
 func (r *APIServerKubeletClientCertificate) GetClient() client.Client {
@@ -68,10 +69,10 @@ func (r *APIServerKubeletClientCertificate) CreateOrUpdate(ctx context.Context,
 }
 
 func (r *APIServerKubeletClientCertificate) GetName() string {
-	return "api-server-kubelet-client-certificate"
+	return r.Name
 }
 
-func (r *APIServerKubeletClientCertificate) UpdateTenantControlPlaneStatus(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+func (r *APIServerKubeletClientCertificate) UpdateTenantControlPlaneStatus(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
 	tenantControlPlane.Status.Certificates.APIServerKubeletClient.LastUpdate = metav1.Now()
 	tenantControlPlane.Status.Certificates.APIServerKubeletClient.SecretName = r.resource.GetName()
 	tenantControlPlane.Status.Certificates.APIServerKubeletClient.Checksum = r.resource.GetAnnotations()["checksum"]
@@ -100,7 +101,6 @@ func (r *APIServerKubeletClientCertificate) mutate(ctx context.Context, tenantCo
 		}
 
 		namespacedName := k8stypes.NamespacedName{Namespace: tenantControlPlane.GetNamespace(), Name: tenantControlPlane.Status.Certificates.CA.SecretName}
-
 		secretCA := &corev1.Secret{}
 		if err = r.Client.Get(ctx, namespacedName, secretCA); err != nil {
 			return err

internal/resources/ca_certificate.go

@@ -24,23 +24,24 @@ type CACertificate struct {
 	resource     *corev1.Secret
 	Client       client.Client
 	Log          logr.Logger
+	Name         string
 	TmpDirectory string
 }
 
-func (r *CACertificate) ShouldStatusBeUpdated(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
+func (r *CACertificate) ShouldStatusBeUpdated(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
 	return tenantControlPlane.Status.Certificates.CA.SecretName != r.resource.GetName() ||
 		tenantControlPlane.Status.Certificates.CA.Checksum != r.resource.GetAnnotations()["checksum"]
 }
 
-func (r *CACertificate) ShouldCleanup(*kamajiv1alpha1.TenantControlPlane) bool {
+func (r *CACertificate) ShouldCleanup(plane *kamajiv1alpha1.TenantControlPlane) bool {
 	return false
 }
 
-func (r *CACertificate) CleanUp(context.Context, *kamajiv1alpha1.TenantControlPlane) (bool, error) {
+func (r *CACertificate) CleanUp(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) {
 	return false, nil
 }
 
-func (r *CACertificate) Define(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+func (r *CACertificate) Define(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
 	r.resource = &corev1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      r.getPrefixedName(tenantControlPlane),
@@ -52,7 +53,7 @@ func (r *CACertificate) Define(_ context.Context, tenantControlPlane *kamajiv1al
 }
 
 func (r *CACertificate) getPrefixedName(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) string {
-	return utilities.AddTenantPrefix(r.GetName(), tenantControlPlane)
+	return utilities.AddTenantPrefix(r.Name, tenantControlPlane)
 }
 
 func (r *CACertificate) GetClient() client.Client {
@@ -68,10 +69,10 @@ func (r *CACertificate) CreateOrUpdate(ctx context.Context, tenantControlPlane *
 }
 
 func (r *CACertificate) GetName() string {
-	return "ca"
+	return r.Name
 }
 
-func (r *CACertificate) UpdateTenantControlPlaneStatus(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+func (r *CACertificate) UpdateTenantControlPlaneStatus(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
 	tenantControlPlane.Status.Certificates.CA.LastUpdate = metav1.Now()
 	tenantControlPlane.Status.Certificates.CA.SecretName = r.resource.GetName()
 	tenantControlPlane.Status.Certificates.CA.Checksum = r.resource.GetAnnotations()["checksum"]

internal/resources/datastore/datastore_certificate.go

@@ -1,148 +0,0 @@
-// Copyright 2022 Clastix Labs
-// SPDX-License-Identifier: Apache-2.0
-
-package datastore
-
-import (
-	"bytes"
-	"context"
-	"fmt"
-
-	corev1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	ctrl "sigs.k8s.io/controller-runtime"
-	"sigs.k8s.io/controller-runtime/pkg/client"
-	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
-
-	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
-	"github.com/clastix/kamaji/internal/etcd"
-	"github.com/clastix/kamaji/internal/utilities"
-)
-
-type Certificate struct {
-	resource  *corev1.Secret
-	Client    client.Client
-	Name      string
-	DataStore kamajiv1alpha1.DataStore
-}
-
-func (r *Certificate) ShouldStatusBeUpdated(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
-	return tenantControlPlane.Status.Storage.Certificate.Checksum != r.resource.GetAnnotations()["checksum"]
-}
-
-func (r *Certificate) ShouldCleanup(*kamajiv1alpha1.TenantControlPlane) bool {
-	return false
-}
-
-func (r *Certificate) CleanUp(context.Context, *kamajiv1alpha1.TenantControlPlane) (bool, error) {
-	return false, nil
-}
-
-func (r *Certificate) Define(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
-	r.resource = &corev1.Secret{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      r.getPrefixedName(tenantControlPlane),
-			Namespace: tenantControlPlane.GetNamespace(),
-		},
-		Data: map[string][]byte{},
-	}
-
-	return nil
-}
-
-func (r *Certificate) getPrefixedName(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) string {
-	return utilities.AddTenantPrefix(r.GetName(), tenantControlPlane)
-}
-
-func (r *Certificate) GetClient() client.Client {
-	return r.Client
-}
-
-func (r *Certificate) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) {
-	return utilities.CreateOrUpdateWithConflict(ctx, r.Client, r.resource, r.mutate(ctx, tenantControlPlane))
-}
-
-func (r *Certificate) GetName() string {
-	return "datastore-certificate"
-}
-
-func (r *Certificate) UpdateTenantControlPlaneStatus(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
-	tenantControlPlane.Status.Storage.Certificate.SecretName = r.resource.GetName()
-	tenantControlPlane.Status.Storage.Certificate.Checksum = r.resource.GetAnnotations()["checksum"]
-	tenantControlPlane.Status.Storage.Certificate.LastUpdate = metav1.Now()
-
-	return nil
-}
-
-func (r *Certificate) mutate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) controllerutil.MutateFn {
-	return func() error {
-		ca, err := r.DataStore.Spec.TLSConfig.CertificateAuthority.Certificate.GetContent(ctx, r.Client)
-		if err != nil {
-			return err
-		}
-
-		r.resource.Data["ca.crt"] = ca
-
-		if r.resource.GetAnnotations()["checksum"] == utilities.CalculateConfigMapChecksum(r.resource.StringData) {
-			if r.DataStore.Spec.Driver == kamajiv1alpha1.EtcdDriver {
-				if isValid, _ := etcd.IsETCDCertificateAndKeyPairValid(r.resource.Data["server.crt"], r.resource.Data["server.key"]); isValid {
-					return nil
-				}
-			}
-		}
-
-		var crt, key *bytes.Buffer
-
-		switch r.DataStore.Spec.Driver {
-		case kamajiv1alpha1.EtcdDriver:
-			// When dealing with the etcd storage we cannot use the basic authentication, thus the generation of a
-			// certificate used for authentication is mandatory, along with the CA private key.
-			privateKey, err := r.DataStore.Spec.TLSConfig.CertificateAuthority.PrivateKey.GetContent(ctx, r.Client)
-			if err != nil {
-				return err
-			}
-
-			crt, key, err = etcd.GetETCDCACertificateAndKeyPair(tenantControlPlane.GetName(), ca, privateKey)
-			if err != nil {
-				return err
-			}
-		case kamajiv1alpha1.KineMySQLDriver, kamajiv1alpha1.KinePostgreSQLDriver:
-			// For the SQL drivers we just need to copy the certificate, since the basic authentication is used
-			// to connect to the desired schema and database.
-			crtBytes, err := r.DataStore.Spec.TLSConfig.ClientCertificate.Certificate.GetContent(ctx, r.Client)
-			if err != nil {
-				return err
-			}
-			crt = bytes.NewBuffer(crtBytes)
-
-			keyBytes, err := r.DataStore.Spec.TLSConfig.ClientCertificate.PrivateKey.GetContent(ctx, r.Client)
-			if err != nil {
-				return err
-			}
-			key = bytes.NewBuffer(keyBytes)
-		default:
-			return fmt.Errorf("unrecognized driver for Certificate generation")
-		}
-
-		r.resource.Data["server.crt"] = crt.Bytes()
-		r.resource.Data["server.key"] = key.Bytes()
-
-		annotations := r.resource.GetAnnotations()
-		if annotations == nil {
-			annotations = map[string]string{}
-		}
-		annotations["checksum"] = utilities.CalculateConfigMapChecksum(r.resource.StringData)
-		r.resource.SetAnnotations(annotations)
-
-		r.resource.SetLabels(utilities.MergeMaps(
-			utilities.KamajiLabels(),
-			r.resource.GetLabels(),
-			map[string]string{
-				"kamaji.clastix.io/name":      tenantControlPlane.GetName(),
-				"kamaji.clastix.io/component": r.GetName(),
-			},
-		))
-
-		return ctrl.SetControllerReference(tenantControlPlane, r.resource, r.Client.Scheme())
-	}
-}

internal/resources/datastore/datastore_setup.go

@@ -1,237 +0,0 @@
-// Copyright 2022 Clastix Labs
-// SPDX-License-Identifier: Apache-2.0
-
-package datastore
-
-import (
-	"context"
-
-	corev1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/types"
-	"sigs.k8s.io/controller-runtime/pkg/client"
-	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
-
-	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
-	"github.com/clastix/kamaji/internal/datastore"
-	"github.com/clastix/kamaji/internal/resources/utils"
-)
-
-type SetupResource struct {
-	schema   string
-	user     string
-	password string
-}
-
-type Setup struct {
-	resource   SetupResource
-	Client     client.Client
-	Connection datastore.Connection
-	DataStore  kamajiv1alpha1.DataStore
-}
-
-func (r *Setup) ShouldStatusBeUpdated(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
-	return tenantControlPlane.Status.Storage.Driver != string(r.DataStore.Spec.Driver) &&
-		tenantControlPlane.Status.Storage.Setup.Checksum != tenantControlPlane.Status.Storage.Config.Checksum
-}
-
-func (r *Setup) ShouldCleanup(_ *kamajiv1alpha1.TenantControlPlane) bool {
-	return false
-}
-
-func (r *Setup) CleanUp(context.Context, *kamajiv1alpha1.TenantControlPlane) (bool, error) {
-	return false, nil
-}
-
-func (r *Setup) Define(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
-	secret := &corev1.Secret{}
-	namespacedName := types.NamespacedName{
-		Namespace: tenantControlPlane.GetNamespace(),
-		Name:      tenantControlPlane.Status.Storage.Config.SecretName,
-	}
-	if err := r.Client.Get(ctx, namespacedName, secret); err != nil {
-		return err
-	}
-
-	r.resource = SetupResource{
-		schema:   string(secret.Data["DB_SCHEMA"]),
-		user:     string(secret.Data["DB_USER"]),
-		password: string(secret.Data["DB_PASSWORD"]),
-	}
-
-	return nil
-}
-
-func (r *Setup) GetClient() client.Client {
-	return r.Client
-}
-
-func (r *Setup) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) {
-	if tenantControlPlane.Status.Storage.Setup.Checksum != "" &&
-		tenantControlPlane.Status.Storage.Setup.Checksum != tenantControlPlane.Status.Storage.Config.Checksum {
-		if err := r.Delete(ctx, tenantControlPlane); err != nil {
-			return controllerutil.OperationResultNone, err
-		}
-
-		return controllerutil.OperationResultUpdated, nil
-	}
-
-	reconcilationResult := controllerutil.OperationResultNone
-	var operationResult controllerutil.OperationResult
-	var err error
-
-	operationResult, err = r.createDB(ctx, tenantControlPlane)
-	if err != nil {
-		return reconcilationResult, err
-	}
-	reconcilationResult = utils.UpdateOperationResult(reconcilationResult, operationResult)
-
-	operationResult, err = r.createUser(ctx, tenantControlPlane)
-	if err != nil {
-		return reconcilationResult, err
-	}
-	reconcilationResult = utils.UpdateOperationResult(reconcilationResult, operationResult)
-
-	operationResult, err = r.createGrantPrivileges(ctx, tenantControlPlane)
-	if err != nil {
-		return reconcilationResult, err
-	}
-	reconcilationResult = utils.UpdateOperationResult(reconcilationResult, operationResult)
-
-	return reconcilationResult, nil
-}
-
-func (r *Setup) GetName() string {
-	return "datastore-setup"
-}
-
-func (r *Setup) Delete(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
-	if err := r.Define(ctx, tenantControlPlane); err != nil {
-		return err
-	}
-
-	if err := r.revokeGrantPrivileges(ctx, tenantControlPlane); err != nil {
-		return err
-	}
-
-	if err := r.deleteDB(ctx, tenantControlPlane); err != nil {
-		return err
-	}
-
-	if err := r.deleteUser(ctx, tenantControlPlane); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (r *Setup) UpdateTenantControlPlaneStatus(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
-	tenantControlPlane.Status.Storage.Setup.Schema = r.resource.schema
-	tenantControlPlane.Status.Storage.Setup.User = r.resource.user
-	tenantControlPlane.Status.Storage.Setup.LastUpdate = metav1.Now()
-	tenantControlPlane.Status.Storage.Setup.Checksum = tenantControlPlane.Status.Storage.Config.Checksum
-
-	return nil
-}
-
-func (r *Setup) createDB(ctx context.Context, _ *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) {
-	exists, err := r.Connection.DBExists(ctx, r.resource.schema)
-	if err != nil {
-		return controllerutil.OperationResultNone, err
-	}
-
-	if exists {
-		return controllerutil.OperationResultNone, nil
-	}
-
-	if err := r.Connection.CreateDB(ctx, r.resource.schema); err != nil {
-		return controllerutil.OperationResultNone, err
-	}
-
-	return controllerutil.OperationResultCreated, nil
-}
-
-func (r *Setup) deleteDB(ctx context.Context, _ *kamajiv1alpha1.TenantControlPlane) error {
-	exists, err := r.Connection.DBExists(ctx, r.resource.schema)
-	if err != nil {
-		return err
-	}
-
-	if !exists {
-		return nil
-	}
-
-	if err := r.Connection.DeleteDB(ctx, r.resource.schema); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (r *Setup) createUser(ctx context.Context, _ *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) {
-	exists, err := r.Connection.UserExists(ctx, r.resource.user)
-	if err != nil {
-		return controllerutil.OperationResultNone, err
-	}
-
-	if exists {
-		return controllerutil.OperationResultNone, nil
-	}
-
-	if err := r.Connection.CreateUser(ctx, r.resource.user, r.resource.password); err != nil {
-		return controllerutil.OperationResultNone, err
-	}
-
-	return controllerutil.OperationResultCreated, nil
-}
-
-func (r *Setup) deleteUser(ctx context.Context, _ *kamajiv1alpha1.TenantControlPlane) error {
-	exists, err := r.Connection.UserExists(ctx, r.resource.user)
-	if err != nil {
-		return err
-	}
-
-	if !exists {
-		return nil
-	}
-
-	if err := r.Connection.DeleteUser(ctx, r.resource.user); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (r *Setup) createGrantPrivileges(ctx context.Context, _ *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) {
-	exists, err := r.Connection.GrantPrivilegesExists(ctx, r.resource.user, r.resource.schema)
-	if err != nil {
-		return controllerutil.OperationResultNone, err
-	}
-
-	if exists {
-		return controllerutil.OperationResultNone, nil
-	}
-
-	if err := r.Connection.GrantPrivileges(ctx, r.resource.user, r.resource.schema); err != nil {
-		return controllerutil.OperationResultNone, err
-	}
-
-	return controllerutil.OperationResultCreated, nil
-}
-
-func (r *Setup) revokeGrantPrivileges(ctx context.Context, _ *kamajiv1alpha1.TenantControlPlane) error {
-	exists, err := r.Connection.GrantPrivilegesExists(ctx, r.resource.user, r.resource.schema)
-	if err != nil {
-		return err
-	}
-
-	if !exists {
-		return nil
-	}
-
-	if err := r.Connection.RevokePrivileges(ctx, r.resource.user, r.resource.schema); err != nil {
-		return err
-	}
-
-	return nil
-}

internal/resources/datastore/datastore_storage_config.go

@@ -1,112 +0,0 @@
-// Copyright 2022 Clastix Labs
-// SPDX-License-Identifier: Apache-2.0
-
-package datastore
-
-import (
-	"context"
-
-	corev1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	ctrl "sigs.k8s.io/controller-runtime"
-	"sigs.k8s.io/controller-runtime/pkg/client"
-	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
-
-	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
-	"github.com/clastix/kamaji/internal/utilities"
-)
-
-type Config struct {
-	resource   *corev1.Secret
-	Client     client.Client
-	ConnString string
-	DataStore  kamajiv1alpha1.DataStore
-}
-
-func (r *Config) ShouldStatusBeUpdated(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
-	return tenantControlPlane.Status.Storage.Config.Checksum != r.resource.GetAnnotations()["checksum"] ||
-		tenantControlPlane.Status.Storage.DataStoreName != r.DataStore.GetName()
-}
-
-func (r *Config) ShouldCleanup(*kamajiv1alpha1.TenantControlPlane) bool {
-	return false
-}
-
-func (r *Config) CleanUp(context.Context, *kamajiv1alpha1.TenantControlPlane) (bool, error) {
-	return false, nil
-}
-
-func (r *Config) Define(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
-	r.resource = &corev1.Secret{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      r.getPrefixedName(tenantControlPlane),
-			Namespace: tenantControlPlane.GetNamespace(),
-		},
-	}
-
-	return nil
-}
-
-func (r *Config) getPrefixedName(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) string {
-	return utilities.AddTenantPrefix(r.GetName(), tenantControlPlane)
-}
-
-func (r *Config) GetClient() client.Client {
-	return r.Client
-}
-
-func (r *Config) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) {
-	return utilities.CreateOrUpdateWithConflict(ctx, r.Client, r.resource, r.mutate(ctx, tenantControlPlane))
-}
-
-func (r *Config) GetName() string {
-	return "datastore-config"
-}
-
-func (r *Config) UpdateTenantControlPlaneStatus(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
-	tenantControlPlane.Status.Storage.Driver = string(r.DataStore.Spec.Driver)
-	tenantControlPlane.Status.Storage.DataStoreName = r.DataStore.GetName()
-	tenantControlPlane.Status.Storage.Config.SecretName = r.resource.GetName()
-	tenantControlPlane.Status.Storage.Config.Checksum = r.resource.GetAnnotations()["checksum"]
-
-	return nil
-}
-
-func (r *Config) mutate(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) controllerutil.MutateFn {
-	return func() error {
-		var password []byte
-
-		savedHash, ok := r.resource.GetAnnotations()["checksum"]
-		switch {
-		case ok && savedHash == utilities.CalculateConfigMapChecksum(r.resource.StringData):
-			password = r.resource.Data["DB_PASSWORD"]
-		default:
-			password = []byte(utilities.GenerateUUIDString())
-		}
-
-		r.resource.Data = map[string][]byte{
-			"DB_CONNECTION_STRING": []byte(r.ConnString),
-			"DB_SCHEMA":            []byte(tenantControlPlane.GetName()),
-			"DB_USER":              []byte(tenantControlPlane.GetName()),
-			"DB_PASSWORD":          password,
-		}
-
-		annotations := r.resource.GetAnnotations()
-		if annotations == nil {
-			annotations = map[string]string{}
-		}
-
-		annotations["checksum"] = utilities.CalculateConfigMapChecksum(r.resource.StringData)
-		r.resource.SetAnnotations(annotations)
-
-		r.resource.SetLabels(utilities.MergeMaps(
-			utilities.KamajiLabels(),
-			map[string]string{
-				"kamaji.clastix.io/name":      tenantControlPlane.GetName(),
-				"kamaji.clastix.io/component": r.GetName(),
-			},
-		))
-
-		return ctrl.SetControllerReference(tenantControlPlane, r.resource, r.Client.Scheme())
-	}
-}

internal/resources/etcd_ca_certificates.go

@@ -0,0 +1,119 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package resources
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/go-logr/logr"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	k8stypes "k8s.io/apimachinery/pkg/types"
+	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+	"github.com/clastix/kamaji/internal/etcd"
+	"github.com/clastix/kamaji/internal/utilities"
+)
+
+type ETCDCACertificatesResource struct {
+	resource              *corev1.Secret
+	Client                client.Client
+	Log                   logr.Logger
+	Name                  string
+	ETCDCASecretName      string
+	ETCDCASecretNamespace string
+}
+
+func (r *ETCDCACertificatesResource) ShouldStatusBeUpdated(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
+	if tenantControlPlane.Status.Certificates.ETCD == nil {
+		return true
+	}
+
+	return tenantControlPlane.Status.Certificates.ETCD.CA.Checksum != r.resource.GetAnnotations()["checksum"]
+}
+
+func (r *ETCDCACertificatesResource) ShouldCleanup(plane *kamajiv1alpha1.TenantControlPlane) bool {
+	return false
+}
+
+func (r *ETCDCACertificatesResource) CleanUp(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) {
+	return false, nil
+}
+
+func (r *ETCDCACertificatesResource) Define(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+	r.resource = &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      r.getPrefixedName(tenantControlPlane),
+			Namespace: tenantControlPlane.GetNamespace(),
+		},
+	}
+
+	return nil
+}
+
+func (r *ETCDCACertificatesResource) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) {
+	return utilities.CreateOrUpdateWithConflict(ctx, r.Client, r.resource, r.mutate(ctx, tenantControlPlane))
+}
+
+func (r *ETCDCACertificatesResource) GetName() string {
+	return r.Name
+}
+
+func (r *ETCDCACertificatesResource) UpdateTenantControlPlaneStatus(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+	if tenantControlPlane.Status.Certificates.ETCD == nil {
+		tenantControlPlane.Status.Certificates.ETCD = &kamajiv1alpha1.ETCDCertificatesStatus{}
+	}
+
+	tenantControlPlane.Status.Certificates.ETCD.CA.SecretName = r.resource.GetName()
+	tenantControlPlane.Status.Certificates.ETCD.CA.LastUpdate = metav1.Now()
+	tenantControlPlane.Status.Certificates.ETCD.CA.Checksum = r.resource.GetAnnotations()["checksum"]
+
+	return nil
+}
+
+func (r *ETCDCACertificatesResource) getPrefixedName(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) string {
+	return utilities.AddTenantPrefix(r.Name, tenantControlPlane)
+}
+
+func (r *ETCDCACertificatesResource) mutate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) controllerutil.MutateFn {
+	return func() error {
+		if etcdStatus := tenantControlPlane.Status.Certificates.ETCD; etcdStatus != nil && len(etcdStatus.CA.Checksum) > 0 && etcdStatus.CA.Checksum == r.resource.GetAnnotations()["checksum"] {
+			isValid, err := etcd.IsETCDCertificateAndKeyPairValid(r.resource.Data[kubeadmconstants.CACertName], r.resource.Data[kubeadmconstants.CAKeyName])
+			if err != nil {
+				r.Log.Info(fmt.Sprintf("etcd certificates are not valid: %s", err.Error()))
+			}
+
+			if isValid {
+				return nil
+			}
+		}
+
+		etcdCASecretNamespacedName := k8stypes.NamespacedName{Namespace: r.ETCDCASecretNamespace, Name: r.ETCDCASecretName}
+		etcdCASecret := &corev1.Secret{}
+		if err := r.Client.Get(ctx, etcdCASecretNamespacedName, etcdCASecret); err != nil {
+			return err
+		}
+
+		r.resource.Data = map[string][]byte{
+			kubeadmconstants.CACertName: etcdCASecret.Data[kubeadmconstants.CACertName],
+			kubeadmconstants.CAKeyName:  etcdCASecret.Data[kubeadmconstants.CAKeyName],
+		}
+
+		r.resource.SetLabels(utilities.KamajiLabels())
+
+		annotations := r.resource.GetAnnotations()
+		if annotations == nil {
+			annotations = map[string]string{}
+		}
+		annotations["checksum"] = utilities.CalculateConfigMapChecksum(r.resource.StringData)
+		r.resource.SetAnnotations(annotations)
+
+		return ctrl.SetControllerReference(tenantControlPlane, r.resource, r.Client.Scheme())
+	}
+}

internal/resources/etcd_certificates.go

@@ -0,0 +1,130 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package resources
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/go-logr/logr"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	k8stypes "k8s.io/apimachinery/pkg/types"
+	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+	"github.com/clastix/kamaji/internal/etcd"
+	"github.com/clastix/kamaji/internal/utilities"
+)
+
+type ETCDCertificatesResource struct {
+	resource *corev1.Secret
+	Client   client.Client
+	Log      logr.Logger
+	Name     string
+}
+
+func (r *ETCDCertificatesResource) ShouldStatusBeUpdated(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
+	if tenantControlPlane.Status.Certificates.ETCD == nil {
+		return true
+	}
+
+	return tenantControlPlane.Status.Certificates.ETCD.APIServer.Checksum != r.resource.GetAnnotations()["checksum"]
+}
+
+func (r *ETCDCertificatesResource) ShouldCleanup(plane *kamajiv1alpha1.TenantControlPlane) bool {
+	return false
+}
+
+func (r *ETCDCertificatesResource) CleanUp(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) {
+	return false, nil
+}
+
+func (r *ETCDCertificatesResource) Define(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+	r.resource = &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      r.getPrefixedName(tenantControlPlane),
+			Namespace: tenantControlPlane.GetNamespace(),
+		},
+	}
+
+	return nil
+}
+
+func (r *ETCDCertificatesResource) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) {
+	return utilities.CreateOrUpdateWithConflict(ctx, r.Client, r.resource, r.mutate(ctx, tenantControlPlane))
+}
+
+func (r *ETCDCertificatesResource) GetName() string {
+	return r.Name
+}
+
+func (r *ETCDCertificatesResource) UpdateTenantControlPlaneStatus(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+	if tenantControlPlane.Status.Certificates.ETCD == nil {
+		tenantControlPlane.Status.Certificates.ETCD = &kamajiv1alpha1.ETCDCertificatesStatus{}
+	}
+
+	tenantControlPlane.Status.Certificates.ETCD.APIServer.SecretName = r.resource.GetName()
+	tenantControlPlane.Status.Certificates.ETCD.APIServer.LastUpdate = metav1.Now()
+	tenantControlPlane.Status.Certificates.ETCD.APIServer.Checksum = r.resource.GetAnnotations()["checksum"]
+
+	return nil
+}
+
+func (r *ETCDCertificatesResource) getPrefixedName(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) string {
+	return utilities.AddTenantPrefix(r.Name, tenantControlPlane)
+}
+
+func (r *ETCDCertificatesResource) mutate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) controllerutil.MutateFn {
+	return func() error {
+		if tenantControlPlane.Status.Certificates.ETCD == nil {
+			return fmt.Errorf("etcd is still synchronizing latest changes")
+		}
+
+		if checksum := tenantControlPlane.Status.Certificates.ETCD.APIServer.Checksum; len(checksum) > 0 && checksum == r.resource.GetAnnotations()["checksum"] {
+			isValid, err := etcd.IsETCDCertificateAndKeyPairValid(r.resource.Data[kubeadmconstants.APIServerEtcdClientCertName], r.resource.Data[kubeadmconstants.APIServerEtcdClientKeyName])
+			if err != nil {
+				r.Log.Info(fmt.Sprintf("etcd certificates are not valid: %s", err.Error()))
+			}
+
+			if isValid {
+				return nil
+			}
+		}
+
+		etcdCASecretNamespacedName := k8stypes.NamespacedName{Namespace: tenantControlPlane.GetNamespace(), Name: tenantControlPlane.Status.Certificates.ETCD.CA.SecretName}
+		etcdCASecret := &corev1.Secret{}
+		if err := r.Client.Get(ctx, etcdCASecretNamespacedName, etcdCASecret); err != nil {
+			return err
+		}
+
+		cert, privKey, err := etcd.GetETCDCACertificateAndKeyPair(
+			tenantControlPlane.GetName(),
+			etcdCASecret.Data[kubeadmconstants.CACertName],
+			etcdCASecret.Data[kubeadmconstants.CAKeyName],
+		)
+		if err != nil {
+			return err
+		}
+
+		r.resource.Data = map[string][]byte{
+			kubeadmconstants.APIServerEtcdClientCertName: cert.Bytes(),
+			kubeadmconstants.APIServerEtcdClientKeyName:  privKey.Bytes(),
+		}
+
+		r.resource.SetLabels(utilities.KamajiLabels())
+
+		annotations := r.resource.GetAnnotations()
+		if annotations == nil {
+			annotations = map[string]string{}
+		}
+		annotations["checksum"] = utilities.CalculateConfigMapChecksum(r.resource.StringData)
+		r.resource.SetAnnotations(annotations)
+
+		return ctrl.SetControllerReference(tenantControlPlane, r.resource, r.Client.Scheme())
+	}
+}

internal/resources/etcd_setup.go

@@ -0,0 +1,278 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package resources
+
+import (
+	"context"
+
+	"github.com/go-logr/logr"
+	etcdclient "go.etcd.io/etcd/client/v3"
+	corev1 "k8s.io/api/core/v1"
+	k8stypes "k8s.io/apimachinery/pkg/types"
+	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+	"github.com/clastix/kamaji/internal/etcd"
+)
+
+const (
+	caKeyName = kubeadmconstants.CACertName
+)
+
+type etcdSetupResource struct {
+	role etcd.Role
+	user etcd.User
+}
+
+type ETCDSetupResource struct {
+	resource              *etcdSetupResource
+	Client                client.Client
+	Log                   logr.Logger
+	Name                  string
+	Endpoints             []string
+	ETCDClientCertsSecret k8stypes.NamespacedName
+	ETCDCACertsSecret     k8stypes.NamespacedName
+}
+
+func (r *ETCDSetupResource) ShouldStatusBeUpdated(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
+	if tenantControlPlane.Status.Storage.ETCD == nil {
+		return true
+	}
+
+	return tenantControlPlane.Status.Storage.ETCD.Role.Name != r.resource.role.Name ||
+		tenantControlPlane.Status.Storage.ETCD.User.Name != r.resource.user.Name
+}
+
+func (r *ETCDSetupResource) ShouldCleanup(plane *kamajiv1alpha1.TenantControlPlane) bool {
+	return false
+}
+
+func (r *ETCDSetupResource) CleanUp(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) {
+	return false, nil
+}
+
+func (r *ETCDSetupResource) Define(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+	r.resource = &etcdSetupResource{
+		role: etcd.Role{Name: tenantControlPlane.Name, Exists: false},
+		user: etcd.User{Name: tenantControlPlane.Name, Exists: false},
+	}
+
+	return nil
+}
+
+func (r *ETCDSetupResource) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) {
+	return r.reconcile(ctx)
+}
+
+func (r *ETCDSetupResource) Delete(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+	if err := r.Define(ctx, tenantControlPlane); err != nil {
+		return err
+	}
+
+	client, err := r.getETCDClient(ctx)
+	if err != nil {
+		return err
+	}
+
+	if err = r.deleteData(ctx, client, tenantControlPlane); err != nil {
+		return err
+	}
+
+	if err = r.deleteUser(ctx, client, tenantControlPlane); err != nil {
+		return err
+	}
+
+	if err = r.deleteRole(ctx, client, tenantControlPlane); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (r *ETCDSetupResource) GetName() string {
+	return r.Name
+}
+
+func (r *ETCDSetupResource) UpdateTenantControlPlaneStatus(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+	if tenantControlPlane.Status.Storage.ETCD == nil {
+		tenantControlPlane.Status.Storage.ETCD = &kamajiv1alpha1.ETCDStatus{}
+	}
+	tenantControlPlane.Status.Storage.ETCD.Role = r.resource.role
+	tenantControlPlane.Status.Storage.ETCD.User = r.resource.user
+
+	return nil
+}
+
+func (r *ETCDSetupResource) reconcile(ctx context.Context) (controllerutil.OperationResult, error) {
+	reconcilationResult := controllerutil.OperationResultNone
+	var operationResult controllerutil.OperationResult
+
+	client, err := r.getETCDClient(ctx)
+	if err != nil {
+		return reconcilationResult, err
+	}
+	defer client.Close()
+
+	operationResult, err = r.reconcileUser(ctx, client)
+	if err != nil {
+		return reconcilationResult, err
+	}
+	reconcilationResult = updateOperationResult(reconcilationResult, operationResult)
+
+	operationResult, err = r.reconcileRole(ctx, client)
+	if err != nil {
+		return controllerutil.OperationResultNone, err
+	}
+	reconcilationResult = updateOperationResult(reconcilationResult, operationResult)
+
+	operationResult, err = r.grantUserRole(ctx, client)
+	if err != nil {
+		return controllerutil.OperationResultNone, err
+	}
+	reconcilationResult = updateOperationResult(reconcilationResult, operationResult)
+
+	operationResult, err = r.grantRolePermissions(ctx, client)
+	if err != nil {
+		return controllerutil.OperationResultNone, err
+	}
+	reconcilationResult = updateOperationResult(reconcilationResult, operationResult)
+
+	return reconcilationResult, nil
+}
+
+func (r *ETCDSetupResource) getETCDClient(ctx context.Context) (*etcdclient.Client, error) {
+	var certsClientSecret corev1.Secret
+	if err := r.Client.Get(ctx, r.ETCDClientCertsSecret, &certsClientSecret); err != nil {
+		return nil, err
+	}
+
+	var certsCASecret corev1.Secret
+	if err := r.Client.Get(ctx, r.ETCDCACertsSecret, &certsCASecret); err != nil {
+		return nil, err
+	}
+
+	config := etcd.Config{
+		ETCDCertificate: certsClientSecret.Data[corev1.TLSCertKey],
+		ETCDPrivateKey:  certsClientSecret.Data[corev1.TLSPrivateKeyKey],
+		ETCDCA:          certsCASecret.Data[caKeyName],
+		Endpoints:       r.Endpoints,
+	}
+
+	return etcd.NewClient(config)
+}
+
+func (r *ETCDSetupResource) reconcileUser(ctx context.Context, client *etcdclient.Client) (controllerutil.OperationResult, error) {
+	if err := etcd.GetUser(ctx, client, &r.resource.user); err != nil {
+		return controllerutil.OperationResultNone, err
+	}
+
+	if !r.resource.user.Exists {
+		if err := etcd.AddUser(ctx, client, r.resource.user.Name); err != nil {
+			return controllerutil.OperationResultNone, err
+		}
+
+		return controllerutil.OperationResultCreated, nil
+	}
+
+	return controllerutil.OperationResultNone, nil
+}
+
+func (r *ETCDSetupResource) reconcileRole(ctx context.Context, client *etcdclient.Client) (controllerutil.OperationResult, error) {
+	if err := etcd.GetRole(ctx, client, &r.resource.role); err != nil {
+		return controllerutil.OperationResultNone, err
+	}
+
+	if !r.resource.role.Exists {
+		if err := etcd.AddRole(ctx, client, r.resource.role.Name); err != nil {
+			return controllerutil.OperationResultNone, err
+		}
+
+		return controllerutil.OperationResultCreated, nil
+	}
+
+	return controllerutil.OperationResultNone, nil
+}
+
+func (r *ETCDSetupResource) grantUserRole(ctx context.Context, client *etcdclient.Client) (controllerutil.OperationResult, error) {
+	if err := etcd.GetUser(ctx, client, &r.resource.user); err != nil {
+		return controllerutil.OperationResultNone, err
+	}
+
+	if len(r.resource.user.Roles) > 0 && isRole(r.resource.user.Roles, r.resource.role.Name) {
+		return controllerutil.OperationResultNone, nil
+	}
+
+	if err := etcd.GrantUserRole(ctx, client, r.resource.user, r.resource.role); err != nil {
+		return controllerutil.OperationResultNone, err
+	}
+
+	return controllerutil.OperationResultUpdated, nil
+}
+
+func (r *ETCDSetupResource) grantRolePermissions(ctx context.Context, client *etcdclient.Client) (controllerutil.OperationResult, error) {
+	if err := etcd.GetRole(ctx, client, &r.resource.role); err != nil {
+		return controllerutil.OperationResultNone, err
+	}
+
+	if len(r.resource.role.Permissions) > 0 && isPermission(r.resource.role.Permissions, r.resource.role.Name) {
+		return controllerutil.OperationResultNone, nil
+	}
+
+	if err := etcd.GrantRolePermission(ctx, client, r.resource.role); err != nil {
+		return controllerutil.OperationResultNone, err
+	}
+
+	return controllerutil.OperationResultUpdated, nil
+}
+
+func (r *ETCDSetupResource) deleteData(ctx context.Context, client *etcdclient.Client, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+	return etcd.CleanUpPrefix(ctx, client, tenantControlPlane.GetName())
+}
+
+func (r *ETCDSetupResource) deleteUser(ctx context.Context, client *etcdclient.Client, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+	if err := etcd.GetUser(ctx, client, &r.resource.user); err != nil {
+		return err
+	}
+
+	if !r.resource.user.Exists {
+		return nil
+	}
+
+	return etcd.RemoveUser(ctx, client, tenantControlPlane.GetName())
+}
+
+func (r *ETCDSetupResource) deleteRole(ctx context.Context, client *etcdclient.Client, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+	if err := etcd.GetRole(ctx, client, &r.resource.role); err != nil {
+		return err
+	}
+
+	if !r.resource.role.Exists {
+		return nil
+	}
+
+	return etcd.RemoveRole(ctx, client, tenantControlPlane.GetName())
+}
+
+func isRole(s []string, x string) bool {
+	for _, o := range s {
+		if o == x {
+			return true
+		}
+	}
+
+	return false
+}
+
+func isPermission(s []etcd.Permission, role string) bool {
+	key := etcd.BuildKey(role)
+	for _, o := range s {
+		if o.Key == key {
+			return true
+		}
+	}
+
+	return false
+}

internal/resources/front-proxy-client-certificate.go

@@ -25,22 +25,23 @@ type FrontProxyClientCertificate struct {
 	resource     *corev1.Secret
 	Client       client.Client
 	Log          logr.Logger
+	Name         string
 	TmpDirectory string
 }
 
-func (r *FrontProxyClientCertificate) ShouldStatusBeUpdated(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
+func (r *FrontProxyClientCertificate) ShouldStatusBeUpdated(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
 	return tenantControlPlane.Status.Certificates.FrontProxyClient.Checksum != r.resource.GetAnnotations()["checksum"]
 }
 
-func (r *FrontProxyClientCertificate) ShouldCleanup(*kamajiv1alpha1.TenantControlPlane) bool {
+func (r *FrontProxyClientCertificate) ShouldCleanup(plane *kamajiv1alpha1.TenantControlPlane) bool {
 	return false
 }
 
-func (r *FrontProxyClientCertificate) CleanUp(context.Context, *kamajiv1alpha1.TenantControlPlane) (bool, error) {
+func (r *FrontProxyClientCertificate) CleanUp(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) {
 	return false, nil
 }
 
-func (r *FrontProxyClientCertificate) Define(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+func (r *FrontProxyClientCertificate) Define(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
 	r.resource = &corev1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      r.getPrefixedName(tenantControlPlane),
@@ -52,7 +53,7 @@ func (r *FrontProxyClientCertificate) Define(_ context.Context, tenantControlPla
 }
 
 func (r *FrontProxyClientCertificate) getPrefixedName(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) string {
-	return utilities.AddTenantPrefix(r.GetName(), tenantControlPlane)
+	return utilities.AddTenantPrefix(r.Name, tenantControlPlane)
 }
 
 func (r *FrontProxyClientCertificate) GetClient() client.Client {
@@ -68,10 +69,10 @@ func (r *FrontProxyClientCertificate) CreateOrUpdate(ctx context.Context, tenant
 }
 
 func (r *FrontProxyClientCertificate) GetName() string {
-	return "front-proxy-client-certificate"
+	return r.Name
 }
 
-func (r *FrontProxyClientCertificate) UpdateTenantControlPlaneStatus(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+func (r *FrontProxyClientCertificate) UpdateTenantControlPlaneStatus(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
 	tenantControlPlane.Status.Certificates.FrontProxyClient.LastUpdate = metav1.Now()
 	tenantControlPlane.Status.Certificates.FrontProxyClient.SecretName = r.resource.GetName()
 	tenantControlPlane.Status.Certificates.FrontProxyClient.Checksum = r.resource.GetAnnotations()["checksum"]

internal/resources/front_proxy_ca_certificate.go

@@ -24,22 +24,23 @@ type FrontProxyCACertificate struct {
 	resource     *corev1.Secret
 	Client       client.Client
 	Log          logr.Logger
+	Name         string
 	TmpDirectory string
 }
 
-func (r *FrontProxyCACertificate) ShouldStatusBeUpdated(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
+func (r *FrontProxyCACertificate) ShouldStatusBeUpdated(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
 	return tenantControlPlane.Status.Certificates.FrontProxyCA.Checksum != r.resource.GetAnnotations()["checksum"]
 }
 
-func (r *FrontProxyCACertificate) ShouldCleanup(*kamajiv1alpha1.TenantControlPlane) bool {
+func (r *FrontProxyCACertificate) ShouldCleanup(plane *kamajiv1alpha1.TenantControlPlane) bool {
 	return false
 }
 
-func (r *FrontProxyCACertificate) CleanUp(context.Context, *kamajiv1alpha1.TenantControlPlane) (bool, error) {
+func (r *FrontProxyCACertificate) CleanUp(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) {
 	return false, nil
 }
 
-func (r *FrontProxyCACertificate) Define(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+func (r *FrontProxyCACertificate) Define(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
 	r.resource = &corev1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      r.getPrefixedName(tenantControlPlane),
@@ -47,11 +48,13 @@ func (r *FrontProxyCACertificate) Define(_ context.Context, tenantControlPlane *
 		},
 	}
 
+	r.Name = "front-proxy-ca-certificate"
+
 	return nil
 }
 
 func (r *FrontProxyCACertificate) getPrefixedName(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) string {
-	return utilities.AddTenantPrefix(r.GetName(), tenantControlPlane)
+	return utilities.AddTenantPrefix(r.Name, tenantControlPlane)
 }
 
 func (r *FrontProxyCACertificate) GetClient() client.Client {
@@ -67,10 +70,10 @@ func (r *FrontProxyCACertificate) CreateOrUpdate(ctx context.Context, tenantCont
 }
 
 func (r *FrontProxyCACertificate) GetName() string {
-	return "front-proxy-ca-certificate"
+	return r.Name
 }
 
-func (r *FrontProxyCACertificate) UpdateTenantControlPlaneStatus(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+func (r *FrontProxyCACertificate) UpdateTenantControlPlaneStatus(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
 	tenantControlPlane.Status.Certificates.FrontProxyCA.LastUpdate = metav1.Now()
 	tenantControlPlane.Status.Certificates.FrontProxyCA.SecretName = r.resource.GetName()
 	tenantControlPlane.Status.Certificates.FrontProxyCA.Checksum = r.resource.GetAnnotations()["checksum"]

internal/resources/k8s_deployment_resource.go

@@ -14,15 +14,18 @@ import (
 
 	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
 	builder "github.com/clastix/kamaji/internal/builders/controlplane"
+	"github.com/clastix/kamaji/internal/types"
 	"github.com/clastix/kamaji/internal/utilities"
 )
 
 type KubernetesDeploymentResource struct {
-	resource           *appsv1.Deployment
-	Client             client.Client
-	DataStore          kamajiv1alpha1.DataStore
-	Name               string
-	KineContainerImage string
+	resource               *appsv1.Deployment
+	Client                 client.Client
+	ETCDStorageType        types.ETCDStorageType
+	ETCDEndpoints          []string
+	ETCDCompactionInterval string
+	Name                   string
+	KineContainerImage     string
 }
 
 func (r *KubernetesDeploymentResource) isStatusEqual(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
@@ -62,16 +65,17 @@ func (r *KubernetesDeploymentResource) mutate(ctx context.Context, tenantControl
 		}
 
 		d := builder.Deployment{
-			Address:            address,
-			DataStore:          r.DataStore,
-			KineContainerImage: r.KineContainerImage,
+			Address:                address,
+			ETCDEndpoints:          r.ETCDEndpoints,
+			ETCDCompactionInterval: r.ETCDCompactionInterval,
+			ETCDStorageType:        r.ETCDStorageType,
+			KineContainerImage:     r.KineContainerImage,
 		}
 		d.SetLabels(r.resource, utilities.MergeMaps(utilities.CommonLabels(tenantControlPlane.GetName()), tenantControlPlane.Spec.ControlPlane.Deployment.AdditionalMetadata.Labels))
 		d.SetAnnotations(r.resource, utilities.MergeMaps(r.resource.Annotations, tenantControlPlane.Spec.ControlPlane.Deployment.AdditionalMetadata.Annotations))
 		d.SetTemplateLabels(&r.resource.Spec.Template, r.deploymentTemplateLabels(ctx, tenantControlPlane))
 		d.SetStrategy(&r.resource.Spec)
 		d.SetSelector(&r.resource.Spec, tenantControlPlane)
-		d.SetTopologySpreadConstraints(&r.resource.Spec, tenantControlPlane.Spec.ControlPlane.Deployment.TopologySpreadConstraints)
 		d.SetReplicas(&r.resource.Spec, tenantControlPlane)
 		d.ResetKubeAPIServerFlags(r.resource, tenantControlPlane)
 		d.SetContainers(&r.resource.Spec.Template.Spec, tenantControlPlane, address)
@@ -113,10 +117,10 @@ func (r *KubernetesDeploymentResource) UpdateTenantControlPlaneStatus(_ context.
 }
 
 func (r *KubernetesDeploymentResource) deploymentTemplateLabels(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (labels map[string]string) {
-	hash := func(ctx context.Context, namespace, secretName string) string {
-		h, _ := utilities.SecretHashValue(ctx, r.Client, namespace, secretName)
+	hash := func(ctx context.Context, namespace, secretName string) (hash string) {
+		hash, _ = utilities.SecretHashValue(ctx, r.Client, namespace, secretName)
 
-		return h
+		return
 	}
 
 	labels = map[string]string{
@@ -131,6 +135,11 @@ func (r *KubernetesDeploymentResource) deploymentTemplateLabels(ctx context.Cont
 		"component.kamaji.clastix.io/scheduler-kubeconfig":                  hash(ctx, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.KubeConfig.Scheduler.SecretName),
 	}
 
+	if r.ETCDStorageType == types.ETCD {
+		labels["component.kamaji.clastix.io/etcd-ca-certificates"] = hash(ctx, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.Certificates.ETCD.CA.SecretName)
+		labels["component.kamaji.clastix.io/etcd-certificates"] = hash(ctx, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.Certificates.ETCD.APIServer.SecretName)
+	}
+
 	return labels
 }
 

internal/resources/k8s_ingress_resource.go

@@ -24,7 +24,7 @@ type KubernetesIngressResource struct {
 	Name     string
 }
 
-func (r *KubernetesIngressResource) ShouldStatusBeUpdated(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
+func (r *KubernetesIngressResource) ShouldStatusBeUpdated(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
 	return !(tenantControlPlane.Status.Kubernetes.Ingress.Name == r.resource.GetName() &&
 		tenantControlPlane.Status.Kubernetes.Ingress.Namespace == r.resource.GetNamespace())
 }
@@ -33,7 +33,7 @@ func (r *KubernetesIngressResource) ShouldCleanup(tenantControlPlane *kamajiv1al
 	return tenantControlPlane.Spec.ControlPlane.Ingress == nil
 }
 
-func (r *KubernetesIngressResource) CleanUp(ctx context.Context, _ *kamajiv1alpha1.TenantControlPlane) (bool, error) {
+func (r *KubernetesIngressResource) CleanUp(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) {
 	if err := r.Client.Delete(ctx, r.resource); err != nil {
 		if !k8serrors.IsNotFound(err) {
 			return false, err
@@ -45,7 +45,7 @@ func (r *KubernetesIngressResource) CleanUp(ctx context.Context, _ *kamajiv1alph
 	return true, nil
 }
 
-func (r *KubernetesIngressResource) UpdateTenantControlPlaneStatus(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+func (r *KubernetesIngressResource) UpdateTenantControlPlaneStatus(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
 	if tenantControlPlane.Spec.ControlPlane.Ingress != nil {
 		tenantControlPlane.Status.Kubernetes.Ingress.IngressStatus = r.resource.Status
 		tenantControlPlane.Status.Kubernetes.Ingress.Name = r.resource.GetName()
@@ -59,7 +59,7 @@ func (r *KubernetesIngressResource) UpdateTenantControlPlaneStatus(_ context.Con
 	return nil
 }
 
-func (r *KubernetesIngressResource) Define(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+func (r *KubernetesIngressResource) Define(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
 	r.resource = &networkingv1.Ingress{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      tenantControlPlane.GetName(),

internal/resources/k8s_service_resource.go

@@ -22,19 +22,20 @@ import (
 type KubernetesServiceResource struct {
 	resource *corev1.Service
 	Client   client.Client
+	Name     string
 }
 
-func (r *KubernetesServiceResource) ShouldStatusBeUpdated(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
+func (r *KubernetesServiceResource) ShouldStatusBeUpdated(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
 	return tenantControlPlane.Status.Kubernetes.Service.Name != r.resource.GetName() ||
 		tenantControlPlane.Status.Kubernetes.Service.Namespace != r.resource.GetNamespace() ||
 		tenantControlPlane.Status.Kubernetes.Service.Port != r.resource.Spec.Ports[0].Port
 }
 
-func (r *KubernetesServiceResource) ShouldCleanup(*kamajiv1alpha1.TenantControlPlane) bool {
+func (r *KubernetesServiceResource) ShouldCleanup(plane *kamajiv1alpha1.TenantControlPlane) bool {
 	return false
 }
 
-func (r *KubernetesServiceResource) CleanUp(context.Context, *kamajiv1alpha1.TenantControlPlane) (bool, error) {
+func (r *KubernetesServiceResource) CleanUp(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) {
 	return false, nil
 }
 
@@ -54,7 +55,7 @@ func (r *KubernetesServiceResource) UpdateTenantControlPlaneStatus(ctx context.C
 	return nil
 }
 
-func (r *KubernetesServiceResource) Define(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+func (r *KubernetesServiceResource) Define(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
 	r.resource = &corev1.Service{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      tenantControlPlane.GetName(),
@@ -62,6 +63,8 @@ func (r *KubernetesServiceResource) Define(_ context.Context, tenantControlPlane
 		},
 	}
 
+	r.Name = "service"
+
 	return nil
 }
 
@@ -121,5 +124,5 @@ func (r *KubernetesServiceResource) mutate(ctx context.Context, tenantControlPla
 }
 
 func (r *KubernetesServiceResource) GetName() string {
-	return "service"
+	return r.Name
 }

internal/resources/konnectivity/agent.go

@@ -70,7 +70,7 @@ func (r *Agent) Define(ctx context.Context, tenantControlPlane *kamajiv1alpha1.T
 }
 
 func (r *Agent) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) {
-	return controllerutil.CreateOrUpdate(ctx, r.tenantClient, r.resource, r.mutate(tenantControlPlane))
+	return controllerutil.CreateOrUpdate(ctx, r.tenantClient, r.resource, r.mutate(ctx, tenantControlPlane))
 }
 
 func (r *Agent) GetName() string {
@@ -96,8 +96,8 @@ func (r *Agent) UpdateTenantControlPlaneStatus(ctx context.Context, tenantContro
 	return nil
 }
 
-func (r *Agent) mutate(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) controllerutil.MutateFn {
-	return func() error {
+func (r *Agent) mutate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) controllerutil.MutateFn {
+	return func() (err error) {
 		address, _, err := tenantControlPlane.AssignedControlPlaneAddress()
 		if err != nil {
 			return err

internal/resources/konnectivity/cluster_role_binding_resource.go

@@ -62,7 +62,7 @@ func (r *ClusterRoleBindingResource) Define(ctx context.Context, tenantControlPl
 }
 
 func (r *ClusterRoleBindingResource) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) {
-	return controllerutil.CreateOrUpdate(ctx, r.tenantClient, r.resource, r.mutate())
+	return controllerutil.CreateOrUpdate(ctx, r.tenantClient, r.resource, r.mutate(ctx, tenantControlPlane))
 }
 
 func (r *ClusterRoleBindingResource) GetName() string {
@@ -86,7 +86,7 @@ func (r *ClusterRoleBindingResource) UpdateTenantControlPlaneStatus(ctx context.
 	return nil
 }
 
-func (r *ClusterRoleBindingResource) mutate() controllerutil.MutateFn {
+func (r *ClusterRoleBindingResource) mutate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) controllerutil.MutateFn {
 	return func() error {
 		r.resource.SetLabels(utilities.MergeMaps(
 			utilities.KamajiLabels(),

internal/resources/konnectivity/deployment_resource.go

@@ -18,6 +18,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/log"
 
 	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+	"github.com/clastix/kamaji/internal/types"
 	"github.com/clastix/kamaji/internal/utilities"
 )
 
@@ -32,10 +33,16 @@ const (
 )
 
 type KubernetesDeploymentResource struct {
-	resource      *appsv1.Deployment
-	Client        client.Client
-	ETCDEndpoints []string
-	Name          string
+	resource               *appsv1.Deployment
+	Client                 client.Client
+	ETCDStorageType        types.ETCDStorageType
+	ETCDEndpoints          []string
+	ETCDCompactionInterval string
+	Name                   string
+}
+
+func (r *KubernetesDeploymentResource) isStatusEqual(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
+	return r.resource.Status.String() == tenantControlPlane.Status.Kubernetes.Deployment.DeploymentStatus.String()
 }
 
 func (r *KubernetesDeploymentResource) ShouldStatusBeUpdated(context.Context, *kamajiv1alpha1.TenantControlPlane) bool {
@@ -103,7 +110,7 @@ func (r *KubernetesDeploymentResource) Define(ctx context.Context, tenantControl
 	return nil
 }
 
-func (r *KubernetesDeploymentResource) syncContainer(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) {
+func (r *KubernetesDeploymentResource) syncContainer(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
 	found, index := utilities.HasNamedContainer(r.resource.Spec.Template.Spec.Containers, konnectivityServerName)
 	if !found {
 		r.resource.Spec.Template.Spec.Containers = append(r.resource.Spec.Template.Spec.Containers, corev1.Container{})
@@ -188,6 +195,8 @@ func (r *KubernetesDeploymentResource) syncContainer(tenantControlPlane *kamajiv
 	if resources := tenantControlPlane.Spec.Addons.Konnectivity.Resources; resources != nil {
 		r.resource.Spec.Template.Spec.Containers[index].Resources = *resources
 	}
+
+	return nil
 }
 
 func (r *KubernetesDeploymentResource) mutate(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) controllerutil.MutateFn {
@@ -202,13 +211,15 @@ func (r *KubernetesDeploymentResource) mutate(_ context.Context, tenantControlPl
 			return fmt.Errorf("the Deployment resource is not ready to be mangled for Konnectivity server enrichment")
 		}
 
-		r.syncContainer(tenantControlPlane)
-
+		if err = r.syncContainer(tenantControlPlane); err != nil {
+			return errors.Wrap(err, "cannot sync konnectivity-server container")
+		}
 		if err = r.patchKubeAPIServerContainer(); err != nil {
 			return errors.Wrap(err, "cannot sync patch kube-apiserver container")
 		}
-
-		r.syncVolumes(tenantControlPlane)
+		if err = r.syncVolumes(tenantControlPlane); err != nil {
+			return errors.Wrap(err, "cannot patch required konnectivity volumes")
+		}
 
 		return nil
 	}
@@ -237,7 +248,9 @@ func (r *KubernetesDeploymentResource) patchKubeAPIServerContainer() error {
 	// Adding the egress selector config file flag
 	args := utilities.ArgsFromSliceToMap(r.resource.Spec.Template.Spec.Containers[index].Args)
 
-	utilities.ArgsAddFlagValue(args, "--egress-selector-config-file", konnectivityEgressSelectorConfigurationPath)
+	if utilities.ArgsAddFlagValue(args, "--egress-selector-config-file", konnectivityEgressSelectorConfigurationPath) {
+		// LOG
+	}
 
 	r.resource.Spec.Template.Spec.Containers[index].Args = utilities.ArgsFromMapToSlice(args)
 
@@ -264,7 +277,7 @@ func (r *KubernetesDeploymentResource) patchKubeAPIServerContainer() error {
 	return nil
 }
 
-func (r *KubernetesDeploymentResource) syncVolumes(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) {
+func (r *KubernetesDeploymentResource) syncVolumes(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
 	found, index := false, 0
 	// Defining volumes for the UDS socket
 	found, index = utilities.HasNamedVolume(r.resource.Spec.Template.Spec.Volumes, konnectivityUDSVolume)
@@ -309,4 +322,6 @@ func (r *KubernetesDeploymentResource) syncVolumes(tenantControlPlane *kamajiv1a
 			DefaultMode: pointer.Int32Ptr(420),
 		},
 	}
+
+	return nil
 }

internal/resources/konnectivity/egress_selector_configuration_resource.go

@@ -77,7 +77,7 @@ func (r *EgressSelectorConfigurationResource) UpdateTenantControlPlaneStatus(ctx
 	return nil
 }
 
-func (r *EgressSelectorConfigurationResource) mutate(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) func() error {
+func (r *EgressSelectorConfigurationResource) mutate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) func() error {
 	return func() error {
 		r.resource.SetLabels(utilities.MergeMaps(r.resource.GetLabels(), utilities.KamajiLabels()))
 

internal/resources/konnectivity/service_account_resource.go

@@ -61,8 +61,8 @@ func (r *ServiceAccountResource) Define(ctx context.Context, tenantControlPlane
 	return nil
 }
 
-func (r *ServiceAccountResource) CreateOrUpdate(ctx context.Context, _ *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) {
-	return controllerutil.CreateOrUpdate(ctx, r.tenantClient, r.resource, r.mutate())
+func (r *ServiceAccountResource) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) {
+	return controllerutil.CreateOrUpdate(ctx, r.tenantClient, r.resource, r.mutate(ctx, tenantControlPlane))
 }
 
 func (r *ServiceAccountResource) GetName() string {
@@ -87,7 +87,7 @@ func (r *ServiceAccountResource) UpdateTenantControlPlaneStatus(ctx context.Cont
 	return nil
 }
 
-func (r *ServiceAccountResource) mutate() controllerutil.MutateFn {
+func (r *ServiceAccountResource) mutate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) controllerutil.MutateFn {
 	return func() error {
 		r.resource.SetLabels(utilities.MergeMaps(
 			utilities.KamajiLabels(),

internal/resources/konnectivity/service_resource.go

@@ -125,7 +125,7 @@ func (r *ServiceResource) CreateOrUpdate(ctx context.Context, tenantControlPlane
 }
 
 func (r *ServiceResource) mutate(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) func() error {
-	return func() error {
+	return func() (err error) {
 		switch len(r.resource.Spec.Ports) {
 		case 0:
 			return fmt.Errorf("current state of the Service is not ready to be mangled for Konnectivity")

internal/resources/kubeadm_addons.go

@@ -55,17 +55,17 @@ func (r *KubeadmAddonResource) SetKubeadmConfigChecksum(checksum string) {
 	r.kubeadmConfigChecksum = checksum
 }
 
-func (r *KubeadmAddonResource) ShouldStatusBeUpdated(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
+func (r *KubeadmAddonResource) ShouldStatusBeUpdated(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
 	return !r.isStatusEqual(tenantControlPlane)
 }
 
 func (r *KubeadmAddonResource) ShouldCleanup(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
-	ok, err := r.getSpec(tenantControlPlane)
+	spec, err := r.getSpec(tenantControlPlane)
 	if err != nil {
 		return false
 	}
 
-	return ok
+	return spec == nil
 }
 
 func (r *KubeadmAddonResource) CleanUp(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) {
@@ -90,7 +90,7 @@ func (r *KubeadmAddonResource) CleanUp(ctx context.Context, tenantControlPlane *
 	return true, nil
 }
 
-func (r *KubeadmAddonResource) Define(context.Context, *kamajiv1alpha1.TenantControlPlane) error {
+func (r *KubeadmAddonResource) Define(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
 	return nil
 }
 
@@ -129,7 +129,7 @@ func (r *KubeadmAddonResource) GetName() string {
 	return r.Name
 }
 
-func (r *KubeadmAddonResource) UpdateTenantControlPlaneStatus(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+func (r *KubeadmAddonResource) UpdateTenantControlPlaneStatus(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
 	status, err := r.GetStatus(tenantControlPlane)
 	if err != nil {
 		return err
@@ -151,14 +151,14 @@ func (r *KubeadmAddonResource) GetStatus(tenantControlPlane *kamajiv1alpha1.Tena
 	}
 }
 
-func (r *KubeadmAddonResource) getSpec(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) {
+func (r *KubeadmAddonResource) getSpec(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (*kamajiv1alpha1.AddonSpec, error) {
 	switch r.KubeadmAddon {
 	case AddonCoreDNS:
-		return tenantControlPlane.Spec.Addons.CoreDNS == nil, nil
+		return tenantControlPlane.Spec.Addons.CoreDNS, nil
 	case AddonKubeProxy:
-		return tenantControlPlane.Spec.Addons.KubeProxy == nil, nil
+		return tenantControlPlane.Spec.Addons.KubeProxy, nil
 	default:
-		return false, fmt.Errorf("%s has no spec", r.KubeadmAddon)
+		return nil, fmt.Errorf("%s has no spec", r.KubeadmAddon)
 	}
 }
 

internal/resources/kubeadm_config.go

@@ -19,25 +19,27 @@ import (
 )
 
 type KubeadmConfigResource struct {
-	resource     *corev1.ConfigMap
-	Client       client.Client
-	ETCDs        []string
-	TmpDirectory string
+	resource               *corev1.ConfigMap
+	Client                 client.Client
+	Name                   string
+	ETCDs                  []string
+	ETCDCompactionInterval string
+	TmpDirectory           string
 }
 
-func (r *KubeadmConfigResource) ShouldStatusBeUpdated(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
+func (r *KubeadmConfigResource) ShouldStatusBeUpdated(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
 	return tenantControlPlane.Status.KubeadmConfig.Checksum != r.resource.GetAnnotations()["checksum"]
 }
 
-func (r *KubeadmConfigResource) ShouldCleanup(*kamajiv1alpha1.TenantControlPlane) bool {
+func (r *KubeadmConfigResource) ShouldCleanup(plane *kamajiv1alpha1.TenantControlPlane) bool {
 	return false
 }
 
-func (r *KubeadmConfigResource) CleanUp(context.Context, *kamajiv1alpha1.TenantControlPlane) (bool, error) {
+func (r *KubeadmConfigResource) CleanUp(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) {
 	return false, nil
 }
 
-func (r *KubeadmConfigResource) Define(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+func (r *KubeadmConfigResource) Define(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
 	r.resource = &corev1.ConfigMap{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      r.getPrefixedName(tenantControlPlane),
@@ -49,7 +51,7 @@ func (r *KubeadmConfigResource) Define(_ context.Context, tenantControlPlane *ka
 }
 
 func (r *KubeadmConfigResource) getPrefixedName(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) string {
-	return utilities.AddTenantPrefix(r.GetName(), tenantControlPlane)
+	return utilities.AddTenantPrefix(r.Name, tenantControlPlane)
 }
 
 func (r *KubeadmConfigResource) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) {
@@ -57,7 +59,7 @@ func (r *KubeadmConfigResource) CreateOrUpdate(ctx context.Context, tenantContro
 }
 
 func (r *KubeadmConfigResource) GetName() string {
-	return "kubeadmconfig"
+	return r.Name
 }
 
 func (r *KubeadmConfigResource) UpdateTenantControlPlaneStatus(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
@@ -96,6 +98,7 @@ func (r *KubeadmConfigResource) mutate(tenantControlPlane *kamajiv1alpha1.Tenant
 			TenantControlPlaneServiceCIDR: tenantControlPlane.Spec.NetworkProfile.ServiceCIDR,
 			TenantControlPlaneVersion:     tenantControlPlane.Spec.Kubernetes.Version,
 			ETCDs:                         r.ETCDs,
+			ETCDCompactionInterval:        r.ETCDCompactionInterval,
 			CertificatesDir:               r.TmpDirectory,
 		}
 

internal/resources/kubeadm_phases.go

@@ -15,7 +15,6 @@ import (
 
 	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
 	"github.com/clastix/kamaji/internal/kubeadm"
-	"github.com/clastix/kamaji/internal/resources/utils"
 )
 
 type kubeadmPhase int
@@ -56,19 +55,19 @@ func (r *KubeadmPhase) SetKubeadmConfigChecksum(checksum string) {
 	r.checksum = checksum
 }
 
-func (r *KubeadmPhase) ShouldStatusBeUpdated(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
+func (r *KubeadmPhase) ShouldStatusBeUpdated(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
 	return !r.isStatusEqual(tenantControlPlane)
 }
 
-func (r *KubeadmPhase) ShouldCleanup(*kamajiv1alpha1.TenantControlPlane) bool {
+func (r *KubeadmPhase) ShouldCleanup(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
 	return false
 }
 
-func (r *KubeadmPhase) CleanUp(context.Context, *kamajiv1alpha1.TenantControlPlane) (bool, error) {
+func (r *KubeadmPhase) CleanUp(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) {
 	return false, nil
 }
 
-func (r *KubeadmPhase) Define(context.Context, *kamajiv1alpha1.TenantControlPlane) error {
+func (r *KubeadmPhase) Define(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
 	return nil
 }
 
@@ -105,7 +104,7 @@ func enrichBootstrapToken(bootstrapToken *bootstraptokenv1.BootstrapToken) {
 	}
 
 	if bootstrapToken.Token.ID == "" {
-		bootstrapToken.Token.ID = fmt.Sprintf("%s.%s", utils.RandomString(6), utils.RandomString(16))
+		bootstrapToken.Token.ID = fmt.Sprintf("%s.%s", randomString(6), randomString(16))
 	}
 }
 
@@ -121,7 +120,7 @@ func (r *KubeadmPhase) GetName() string {
 	return r.Name
 }
 
-func (r *KubeadmPhase) UpdateTenantControlPlaneStatus(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+func (r *KubeadmPhase) UpdateTenantControlPlaneStatus(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
 	status, err := r.GetStatus(tenantControlPlane)
 	if err != nil {
 		return err

internal/resources/kubeadm_upgrade.go

@@ -19,6 +19,7 @@ import (
 )
 
 type KubernetesUpgrade struct {
+	Name    string
 	Client  client.Client
 	upgrade upgrade.Upgrade
 
@@ -46,25 +47,25 @@ func (k *KubernetesUpgrade) CleanUp(context.Context, *kamajiv1alpha1.TenantContr
 	return false, nil
 }
 
-func (k *KubernetesUpgrade) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) {
+func (k *KubernetesUpgrade) CreateOrUpdate(ctx context.Context, plane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) {
 	// A new installation, no need to upgrade
-	if len(tenantControlPlane.Status.Kubernetes.Version.Version) == 0 {
+	if len(plane.Status.Kubernetes.Version.Version) == 0 {
 		k.inProgress = false
 
 		return controllerutil.OperationResultNone, nil
 	}
 	// No version change, no need to upgrade
-	if tenantControlPlane.Status.Kubernetes.Version.Version == tenantControlPlane.Spec.Kubernetes.Version {
+	if plane.Status.Kubernetes.Version.Version == plane.Spec.Kubernetes.Version {
 		k.inProgress = false
 
 		return controllerutil.OperationResultNone, nil
 	}
 	// An upgrade is in progress, let it go
-	if status := tenantControlPlane.Status.Kubernetes.Version.Status; status != nil && *status == kamajiv1alpha1.VersionUpgrading {
+	if status := plane.Status.Kubernetes.Version.Status; status != nil && *status == kamajiv1alpha1.VersionUpgrading {
 		return controllerutil.OperationResultNone, nil
 	}
 	// Checking if the upgrade is allowed, or not
-	restClient, err := utilities.GetTenantRESTClient(ctx, k.Client, tenantControlPlane)
+	restClient, err := utilities.GetTenantRESTClient(ctx, k.Client, plane)
 	if err != nil {
 		return controllerutil.OperationResultNone, errors.Wrap(err, "cannot create REST client required for Kubernetes upgrade plan")
 	}
@@ -85,14 +86,14 @@ func (k *KubernetesUpgrade) CreateOrUpdate(ctx context.Context, tenantControlPla
 }
 
 func (k *KubernetesUpgrade) GetName() string {
-	return "upgrade"
+	return k.Name
 }
 
 func (k *KubernetesUpgrade) ShouldStatusBeUpdated(context.Context, *kamajiv1alpha1.TenantControlPlane) bool {
 	return k.inProgress
 }
 
-func (k *KubernetesUpgrade) UpdateTenantControlPlaneStatus(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+func (k *KubernetesUpgrade) UpdateTenantControlPlaneStatus(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
 	if k.inProgress {
 		tenantControlPlane.Status.Kubernetes.Version.Status = &kamajiv1alpha1.VersionUpgrading
 	}

internal/resources/kubeadm_utils.go

@@ -41,35 +41,6 @@ func KubeadmPhaseCreate(ctx context.Context, r KubeadmPhaseResource, tenantContr
 		TenantControlPlaneCGroupDriver: tenantControlPlane.Spec.Kubernetes.Kubelet.CGroupFS.String(),
 	}
 
-	// If CoreDNS addon is enabled and with an override, adding these to the kubeadm init configuration
-	if coreDNS := tenantControlPlane.Spec.Addons.CoreDNS; coreDNS != nil {
-		config.Parameters.CoreDNSOptions = &kubeadm.AddonOptions{}
-
-		if len(coreDNS.ImageRepository) > 0 {
-			config.Parameters.CoreDNSOptions.Repository = coreDNS.ImageRepository
-		}
-
-		if len(coreDNS.ImageRepository) > 0 {
-			config.Parameters.CoreDNSOptions.Tag = coreDNS.ImageTag
-		}
-	}
-	// If the kube-proxy addon is enabled and with overrides, adding it to the kubeadm parameters
-	if kubeProxy := tenantControlPlane.Spec.Addons.KubeProxy; kubeProxy != nil {
-		config.Parameters.KubeProxyOptions = &kubeadm.AddonOptions{}
-
-		if len(kubeProxy.ImageRepository) > 0 {
-			config.Parameters.KubeProxyOptions.Repository = kubeProxy.ImageRepository
-		} else {
-			config.Parameters.KubeProxyOptions.Repository = "k8s.gcr.io"
-		}
-
-		if len(kubeProxy.ImageTag) > 0 {
-			config.Parameters.KubeProxyOptions.Tag = kubeProxy.ImageTag
-		} else {
-			config.Parameters.KubeProxyOptions.Tag = tenantControlPlane.Spec.Kubernetes.Version
-		}
-	}
-
 	checksum := config.Checksum()
 
 	status, err := r.GetStatus(tenantControlPlane)

internal/resources/kubeconfig.go

@@ -61,7 +61,7 @@ func (r *KubeconfigResource) Define(ctx context.Context, tenantControlPlane *kam
 }
 
 func (r *KubeconfigResource) getPrefixedName(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) string {
-	return utilities.AddTenantPrefix(r.GetName(), tenantControlPlane)
+	return utilities.AddTenantPrefix(r.Name, tenantControlPlane)
 }
 
 func (r *KubeconfigResource) GetClient() client.Client {
@@ -146,7 +146,6 @@ func (r *KubeconfigResource) mutate(ctx context.Context, tenantControlPlane *kam
 
 		kubeconfig, err := kubeadm.CreateKubeconfig(
 			r.KubeConfigFileName,
-
 			kubeadm.CertificatePrivateKeyPair{
 				Certificate: apiServerCertificatesSecret.Data[kubeadmconstants.CACertName],
 				PrivateKey:  apiServerCertificatesSecret.Data[kubeadmconstants.CAKeyName],

internal/resources/resource.go

@@ -27,7 +27,6 @@ type Resource interface {
 }
 
 type DeleteableResource interface {
-	GetName() string
 	Define(context.Context, *kamajiv1alpha1.TenantControlPlane) error
 	Delete(context.Context, *kamajiv1alpha1.TenantControlPlane) error
 }

internal/resources/sa_certificate.go

@@ -28,19 +28,19 @@ type SACertificate struct {
 	TmpDirectory string
 }
 
-func (r *SACertificate) ShouldStatusBeUpdated(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
+func (r *SACertificate) ShouldStatusBeUpdated(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
 	return tenantControlPlane.Status.Certificates.SA.SecretName != r.resource.GetName()
 }
 
-func (r *SACertificate) ShouldCleanup(*kamajiv1alpha1.TenantControlPlane) bool {
+func (r *SACertificate) ShouldCleanup(plane *kamajiv1alpha1.TenantControlPlane) bool {
 	return false
 }
 
-func (r *SACertificate) CleanUp(context.Context, *kamajiv1alpha1.TenantControlPlane) (bool, error) {
+func (r *SACertificate) CleanUp(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) {
 	return false, nil
 }
 
-func (r *SACertificate) Define(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+func (r *SACertificate) Define(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
 	r.resource = &corev1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      r.getPrefixedName(tenantControlPlane),
@@ -52,7 +52,7 @@ func (r *SACertificate) Define(_ context.Context, tenantControlPlane *kamajiv1al
 }
 
 func (r *SACertificate) getPrefixedName(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) string {
-	return utilities.AddTenantPrefix(r.GetName(), tenantControlPlane)
+	return utilities.AddTenantPrefix(r.Name, tenantControlPlane)
 }
 
 func (r *SACertificate) GetClient() client.Client {
@@ -68,7 +68,7 @@ func (r *SACertificate) CreateOrUpdate(ctx context.Context, tenantControlPlane *
 }
 
 func (r *SACertificate) GetName() string {
-	return "sa-certificate"
+	return r.Name
 }
 
 func (r *SACertificate) UpdateTenantControlPlaneStatus(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {

internal/resources/sql_certificate.go

@@ -0,0 +1,143 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package resources
+
+import (
+	"context"
+	"fmt"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	k8stypes "k8s.io/apimachinery/pkg/types"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+	"github.com/clastix/kamaji/internal/types"
+	"github.com/clastix/kamaji/internal/utilities"
+)
+
+type SQLCertificate struct {
+	resource                 *corev1.Secret
+	Client                   client.Client
+	Name                     string
+	StorageType              types.ETCDStorageType
+	SQLConfigSecretName      string
+	SQLConfigSecretNamespace string
+}
+
+func (r *SQLCertificate) ShouldStatusBeUpdated(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
+	return tenantControlPlane.Status.Storage.Kine.Certificate.Checksum != r.resource.GetAnnotations()["checksum"]
+}
+
+func (r *SQLCertificate) ShouldCleanup(plane *kamajiv1alpha1.TenantControlPlane) bool {
+	return false
+}
+
+func (r *SQLCertificate) CleanUp(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) {
+	return false, nil
+}
+
+func (r *SQLCertificate) Define(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+	r.resource = &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      r.getPrefixedName(tenantControlPlane),
+			Namespace: tenantControlPlane.GetNamespace(),
+		},
+		Data: map[string][]byte{},
+	}
+
+	return nil
+}
+
+func (r *SQLCertificate) getPrefixedName(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) string {
+	return utilities.AddTenantPrefix(r.Name, tenantControlPlane)
+}
+
+func (r *SQLCertificate) GetClient() client.Client {
+	return r.Client
+}
+
+func (r *SQLCertificate) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) {
+	return utilities.CreateOrUpdateWithConflict(ctx, r.Client, r.resource, r.mutate(ctx, tenantControlPlane))
+}
+
+func (r *SQLCertificate) GetName() string {
+	return r.Name
+}
+
+func (r *SQLCertificate) UpdateTenantControlPlaneStatus(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+	if tenantControlPlane.Status.Storage.Kine == nil {
+		tenantControlPlane.Status.Storage.Kine = &kamajiv1alpha1.KineStatus{}
+	}
+
+	tenantControlPlane.Status.Storage.Kine.Certificate.SecretName = r.resource.GetName()
+	tenantControlPlane.Status.Storage.Kine.Certificate.Checksum = r.resource.GetAnnotations()["checksum"]
+	tenantControlPlane.Status.Storage.Kine.Certificate.LastUpdate = metav1.Now()
+
+	return nil
+}
+
+func (r *SQLCertificate) mutate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) controllerutil.MutateFn {
+	return func() error {
+		sqlConfig := &corev1.Secret{}
+		namespacedName := k8stypes.NamespacedName{Namespace: r.SQLConfigSecretNamespace, Name: r.SQLConfigSecretName}
+		if err := r.Client.Get(ctx, namespacedName, sqlConfig); err != nil {
+			return err
+		}
+
+		checksum, err := r.buildSecret(ctx, *sqlConfig)
+		if err != nil {
+			return err
+		}
+
+		annotations := r.resource.GetAnnotations()
+		if annotations == nil {
+			annotations = map[string]string{}
+		}
+
+		annotations["checksum"] = checksum
+
+		r.resource.SetAnnotations(annotations)
+
+		r.resource.SetLabels(utilities.MergeMaps(
+			utilities.KamajiLabels(),
+			r.resource.GetLabels(),
+			map[string]string{
+				"kamaji.clastix.io/name":      tenantControlPlane.GetName(),
+				"kamaji.clastix.io/component": r.GetName(),
+			},
+		))
+
+		return ctrl.SetControllerReference(tenantControlPlane, r.resource, r.Client.Scheme())
+	}
+}
+
+func (r *SQLCertificate) buildSecret(ctx context.Context, sqlConfig corev1.Secret) (checksum string, err error) {
+	switch r.StorageType {
+	case types.KineMySQL, types.KinePostgreSQL:
+		keys := []string{"ca.crt", "server.crt", "server.key"}
+
+		return r.buildKineSecret(ctx, keys, sqlConfig)
+	default:
+		return "", fmt.Errorf("storage type %s is not implemented", r.StorageType)
+	}
+}
+
+func (r *SQLCertificate) buildKineSecret(ctx context.Context, keys []string, sqlConfig corev1.Secret) (string, error) {
+	checksumMap := map[string]string{}
+
+	for _, key := range keys {
+		value, ok := sqlConfig.Data[key]
+		if !ok {
+			return "", fmt.Errorf("%s is not in sql config secret", key)
+		}
+
+		r.resource.Data[key] = value
+		checksumMap[key] = string(value)
+	}
+
+	return utilities.CalculateConfigMapChecksum(checksumMap), nil
+}

internal/resources/sql_setup.go

@@ -0,0 +1,246 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package resources
+
+import (
+	"context"
+	"fmt"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+	"github.com/clastix/kamaji/internal/sql"
+)
+
+type sqlSetupResource struct {
+	schema   string
+	user     string
+	password string
+}
+
+type SQLSetup struct {
+	resource     sqlSetupResource
+	Client       client.Client
+	DBConnection sql.DBConnection
+	Name         string
+	Driver       string
+}
+
+func (r *SQLSetup) ShouldStatusBeUpdated(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
+	if tenantControlPlane.Status.Storage.Kine == nil {
+		return true
+	}
+
+	return tenantControlPlane.Status.Storage.Kine.Driver != r.Driver &&
+		tenantControlPlane.Status.Storage.Kine.Setup.Checksum != tenantControlPlane.Status.Storage.Kine.Config.Checksum
+}
+
+func (r *SQLSetup) ShouldCleanup(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
+	return false
+}
+
+func (r *SQLSetup) CleanUp(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) {
+	return false, nil
+}
+
+func (r *SQLSetup) Define(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+	secret := &corev1.Secret{}
+	namespacedName := types.NamespacedName{
+		Namespace: tenantControlPlane.GetNamespace(),
+		Name:      tenantControlPlane.Status.Storage.Kine.Config.SecretName,
+	}
+	if err := r.Client.Get(ctx, namespacedName, secret); err != nil {
+		return err
+	}
+
+	r.resource = sqlSetupResource{
+		schema:   string(secret.Data["DB_SCHEMA"]),
+		user:     string(secret.Data["DB_USER"]),
+		password: string(secret.Data["DB_PASSWORD"]),
+	}
+
+	return nil
+}
+
+func (r *SQLSetup) GetClient() client.Client {
+	return r.Client
+}
+
+func (r *SQLSetup) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) {
+	if tenantControlPlane.Status.Storage.Kine.Setup.Checksum != "" &&
+		tenantControlPlane.Status.Storage.Kine.Setup.Checksum != tenantControlPlane.Status.Storage.Kine.Config.Checksum {
+		if err := r.Delete(ctx, tenantControlPlane); err != nil {
+			return controllerutil.OperationResultNone, err
+		}
+
+		return controllerutil.OperationResultUpdated, nil
+	}
+
+	reconcilationResult := controllerutil.OperationResultNone
+	var operationResult controllerutil.OperationResult
+	var err error
+
+	operationResult, err = r.createDB(ctx, tenantControlPlane)
+	if err != nil {
+		return reconcilationResult, err
+	}
+	reconcilationResult = updateOperationResult(reconcilationResult, operationResult)
+
+	operationResult, err = r.createUser(ctx, tenantControlPlane)
+	if err != nil {
+		return reconcilationResult, err
+	}
+	reconcilationResult = updateOperationResult(reconcilationResult, operationResult)
+
+	operationResult, err = r.createGrantPrivileges(ctx, tenantControlPlane)
+	if err != nil {
+		return reconcilationResult, err
+	}
+	reconcilationResult = updateOperationResult(reconcilationResult, operationResult)
+
+	return reconcilationResult, nil
+}
+
+func (r *SQLSetup) GetName() string {
+	return r.Name
+}
+
+func (r *SQLSetup) Delete(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+	if err := r.Define(ctx, tenantControlPlane); err != nil {
+		return err
+	}
+
+	if err := r.revokeGrantPrivileges(ctx, tenantControlPlane); err != nil {
+		return err
+	}
+
+	if err := r.deleteDB(ctx, tenantControlPlane); err != nil {
+		return err
+	}
+
+	if err := r.deleteUser(ctx, tenantControlPlane); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (r *SQLSetup) UpdateTenantControlPlaneStatus(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+	if tenantControlPlane.Status.Storage.Kine == nil {
+		return fmt.Errorf("sql configuration is not ready")
+	}
+
+	tenantControlPlane.Status.Storage.Kine.Setup.Schema = r.resource.schema
+	tenantControlPlane.Status.Storage.Kine.Setup.User = r.resource.user
+	tenantControlPlane.Status.Storage.Kine.Setup.LastUpdate = metav1.Now()
+	tenantControlPlane.Status.Storage.Kine.Setup.Checksum = tenantControlPlane.Status.Storage.Kine.Config.Checksum
+
+	return nil
+}
+
+func (r *SQLSetup) createDB(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) {
+	exists, err := r.DBConnection.DBExists(ctx, r.resource.schema)
+	if err != nil {
+		return controllerutil.OperationResultNone, err
+	}
+
+	if exists {
+		return controllerutil.OperationResultNone, nil
+	}
+
+	if err := r.DBConnection.CreateDB(ctx, r.resource.schema); err != nil {
+		return controllerutil.OperationResultNone, err
+	}
+
+	return controllerutil.OperationResultCreated, nil
+}
+
+func (r *SQLSetup) deleteDB(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+	exists, err := r.DBConnection.DBExists(ctx, r.resource.schema)
+	if err != nil {
+		return err
+	}
+
+	if !exists {
+		return nil
+	}
+
+	if err := r.DBConnection.DeleteDB(ctx, r.resource.schema); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (r *SQLSetup) createUser(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) {
+	exists, err := r.DBConnection.UserExists(ctx, r.resource.user)
+	if err != nil {
+		return controllerutil.OperationResultNone, err
+	}
+
+	if exists {
+		return controllerutil.OperationResultNone, nil
+	}
+
+	if err := r.DBConnection.CreateUser(ctx, r.resource.user, r.resource.password); err != nil {
+		return controllerutil.OperationResultNone, err
+	}
+
+	return controllerutil.OperationResultCreated, nil
+}
+
+func (r *SQLSetup) deleteUser(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+	exists, err := r.DBConnection.UserExists(ctx, r.resource.user)
+	if err != nil {
+		return err
+	}
+
+	if !exists {
+		return nil
+	}
+
+	if err := r.DBConnection.DeleteUser(ctx, r.resource.user); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (r *SQLSetup) createGrantPrivileges(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) {
+	exists, err := r.DBConnection.GrantPrivilegesExists(ctx, r.resource.user, r.resource.schema)
+	if err != nil {
+		return controllerutil.OperationResultNone, err
+	}
+
+	if exists {
+		return controllerutil.OperationResultNone, nil
+	}
+
+	if err := r.DBConnection.GrantPrivileges(ctx, r.resource.user, r.resource.schema); err != nil {
+		return controllerutil.OperationResultNone, err
+	}
+
+	return controllerutil.OperationResultCreated, nil
+}
+
+func (r *SQLSetup) revokeGrantPrivileges(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+	exists, err := r.DBConnection.GrantPrivilegesExists(ctx, r.resource.user, r.resource.schema)
+	if err != nil {
+		return err
+	}
+
+	if !exists {
+		return nil
+	}
+
+	if err := r.DBConnection.RevokePrivileges(ctx, r.resource.user, r.resource.schema); err != nil {
+		return err
+	}
+
+	return nil
+}

internal/resources/sql_storage_config.go

@@ -0,0 +1,123 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package resources
+
+import (
+	"context"
+	"strconv"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+	"github.com/clastix/kamaji/internal/utilities"
+)
+
+type SQLStorageConfig struct {
+	resource *corev1.Secret
+	Client   client.Client
+	Name     string
+	Host     string
+	Port     int
+	Driver   string
+}
+
+func (r *SQLStorageConfig) ShouldStatusBeUpdated(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
+	if tenantControlPlane.Status.Storage.Kine == nil {
+		return true
+	}
+
+	return tenantControlPlane.Status.Storage.Kine.Config.Checksum != r.resource.GetAnnotations()["checksum"] ||
+		tenantControlPlane.Status.Storage.Kine.Driver != r.Driver
+}
+
+func (r *SQLStorageConfig) ShouldCleanup(plane *kamajiv1alpha1.TenantControlPlane) bool {
+	return false
+}
+
+func (r *SQLStorageConfig) CleanUp(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) {
+	return false, nil
+}
+
+func (r *SQLStorageConfig) Define(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+	r.resource = &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      r.getPrefixedName(tenantControlPlane),
+			Namespace: tenantControlPlane.GetNamespace(),
+		},
+	}
+
+	return nil
+}
+
+func (r *SQLStorageConfig) getPrefixedName(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) string {
+	return utilities.AddTenantPrefix(r.Name, tenantControlPlane)
+}
+
+func (r *SQLStorageConfig) GetClient() client.Client {
+	return r.Client
+}
+
+func (r *SQLStorageConfig) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) {
+	return utilities.CreateOrUpdateWithConflict(ctx, r.Client, r.resource, r.mutate(ctx, tenantControlPlane))
+}
+
+func (r *SQLStorageConfig) GetName() string {
+	return r.Name
+}
+
+func (r *SQLStorageConfig) UpdateTenantControlPlaneStatus(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+	if tenantControlPlane.Status.Storage.Kine == nil {
+		tenantControlPlane.Status.Storage.Kine = &kamajiv1alpha1.KineStatus{}
+	}
+
+	tenantControlPlane.Status.Storage.Kine.Driver = r.Driver
+	tenantControlPlane.Status.Storage.Kine.Config.SecretName = r.resource.GetName()
+	tenantControlPlane.Status.Storage.Kine.Config.Checksum = r.resource.GetAnnotations()["checksum"]
+
+	return nil
+}
+
+func (r *SQLStorageConfig) mutate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) controllerutil.MutateFn {
+	return func() error {
+		var password []byte
+
+		savedHash, ok := r.resource.GetAnnotations()["checksum"]
+		switch {
+		case ok && savedHash == utilities.CalculateConfigMapChecksum(r.resource.StringData):
+			password = r.resource.Data["DB_PASSWORD"]
+		default:
+			password = []byte(utilities.GenerateUUIDString())
+		}
+
+		r.resource.Data = map[string][]byte{
+			"DB_HOST":     []byte(r.Host),
+			"DB_PORT":     []byte(strconv.Itoa(r.Port)),
+			"DB_SCHEMA":   []byte(tenantControlPlane.GetName()),
+			"DB_USER":     []byte(tenantControlPlane.GetName()),
+			"DB_PASSWORD": password,
+		}
+
+		annotations := r.resource.GetAnnotations()
+		if annotations == nil {
+			annotations = map[string]string{}
+		}
+
+		annotations["checksum"] = utilities.CalculateConfigMapChecksum(r.resource.StringData)
+		r.resource.SetAnnotations(annotations)
+
+		r.resource.SetLabels(utilities.MergeMaps(
+			utilities.KamajiLabels(),
+			map[string]string{
+				"kamaji.clastix.io/name":      tenantControlPlane.GetName(),
+				"kamaji.clastix.io/component": r.GetName(),
+			},
+		))
+
+		return ctrl.SetControllerReference(tenantControlPlane, r.resource, r.Client.Scheme())
+	}
+}