feat: making kine container image configurable via kamaji flag

A new CLI flag (`--kine-container`) has been introduced, with the
default value of `rancher/kine:v0.9.2-amd64`. It can be overridden also
using the kamaji configuration file (`kamaji.yaml`) using the key
`kine-image`.

.github/workflows/e2e.yaml

@@ -12,6 +12,7 @@ on:
       - 'go.*'
       - 'main.go'
       - 'Makefile'
+      - 'internal/**'
   pull_request:
     branches: [ "*" ]
     paths:
@@ -23,6 +24,7 @@ on:
       - 'go.*'
       - 'main.go'
       - 'Makefile'
+      - 'internal/**'
 
 jobs:
   kind:

.github/workflows/helm.yaml

@@ -0,0 +1,36 @@
+name: Helm Chart
+
+on:
+  push:
+    branches: [ "*" ]
+    tags: [ "helm-v" ]
+  pull_request:
+    branches: [ "*" ]
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: azure/setup-helm@v1
+        with:
+          version: 3.3.4
+      - name: Linting Chart
+        run: helm lint ./helm/kamaji
+  release:
+    if: startsWith(github.ref, 'refs/tags/helm-v')
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Publish Helm chart
+        uses: stefanprodan/helm-gh-pages@master
+        with:
+          token: ${{ secrets.BOT_GITHUB_TOKEN }}
+          charts_dir: helm
+          charts_url: https://clastix.github.io/charts
+          owner: clastix
+          repository: charts
+          branch: gh-pages
+          target_dir: .
+          commit_username: prometherion
+          commit_email: dario@tranchitella.eu

.golangci.yml

@@ -4,6 +4,10 @@ linters-settings:
       - standard
       - default
       - prefix(github.com/clastix/kamaji)
+  goheader:
+    template: |-
+      Copyright 2022 Clastix Labs
+      SPDX-License-Identifier: Apache-2.0
 
 linters:
   disable:

Dockerfile

@@ -2,6 +2,12 @@
 FROM golang:1.18 as builder
 
 ARG TARGETARCH
+ARG GIT_HEAD_COMMIT
+ARG GIT_TAG_COMMIT
+ARG GIT_LAST_TAG
+ARG GIT_MODIFIED
+ARG GIT_REPO
+ARG BUILD_DATE
 
 WORKDIR /workspace
 # Copy the Go Modules manifests
@@ -18,7 +24,9 @@ COPY controllers/ controllers/
 COPY internal/ internal/
 
 # Build
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=$TARGETARCH go build -a -o manager main.go
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=$TARGETARCH go build \
+    -ldflags "-X github.com/clastix/kamaji/internal.GitRepo=$GIT_REPO -X github.com/clastix/kamaji/internal.GitTag=$GIT_LAST_TAG -X github.com/clastix/kamaji/internal.GitCommit=$GIT_HEAD_COMMIT -X github.com/clastix/kamaji/internal.GitDirty=$GIT_MODIFIED -X github.com/clastix/kamaji/internal.BuildTime=$BUILD_DATE" \
+    -a -o manager main.go
 
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details

Makefile

@@ -97,6 +97,7 @@ kustomize: ## Download kustomize locally if necessary.
 
 manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects.
 	$(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=manager-role webhook paths="./..." output:crd:artifacts:config=config/crd/bases
+	cp config/crd/bases/kamaji.clastix.io_tenantcontrolplanes.yaml helm/kamaji/crds/tenantcontrolplane.yaml
 
 generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations.
 	$(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..."
@@ -106,6 +107,15 @@ test:
 
 ##@ Build
 
+# Get information about git current status
+GIT_HEAD_COMMIT ?= $$(git rev-parse --short HEAD)
+GIT_TAG_COMMIT  ?= $$(git rev-parse --short $(VERSION))
+GIT_MODIFIED_1  ?= $$(git diff $(GIT_HEAD_COMMIT) $(GIT_TAG_COMMIT) --quiet && echo "" || echo ".dev")
+GIT_MODIFIED_2  ?= $$(git diff --quiet && echo "" || echo ".dirty")
+GIT_MODIFIED    ?= $$(echo "$(GIT_MODIFIED_1)$(GIT_MODIFIED_2)")
+GIT_REPO        ?= $$(git config --get remote.origin.url)
+BUILD_DATE      ?= $$(git log -1 --format="%at" | xargs -I{} date -d @{} +%Y-%m-%dT%H:%M:%S)
+
 build: generate fmt vet ## Build manager binary.
 	go build -o bin/manager main.go
 
@@ -113,7 +123,12 @@ run: manifests generate fmt vet ## Run a controller from your host.
 	go run ./main.go
 
 docker-build: ## Build docker image with the manager.
-	docker build -t ${IMG} .
+	docker build -t ${IMG} . --build-arg GIT_HEAD_COMMIT=$(GIT_HEAD_COMMIT) \
+		--build-arg GIT_TAG_COMMIT=$(GIT_TAG_COMMIT) \
+		--build-arg GIT_MODIFIED=$(GIT_MODIFIED) \
+		--build-arg GIT_REPO=$(GIT_REPO) \
+		--build-arg GIT_LAST_TAG=$(VERSION) \
+		--build-arg BUILD_DATE=$(BUILD_DATE)
 
 docker-push: ## Push docker image with the manager.
 	docker push ${IMG}
@@ -223,7 +238,7 @@ endef
 
 .PHONY: env
 env:
-	@make -C deploy/kind kamaji
+	@make -C deploy/kind kind ingress-nginx
 
 ##@ e2e
 

README.md

@@ -42,6 +42,10 @@ A dedicated `etcd` cluster for each tenant cluster doesn’t scale well for a ma
 
 With this solution, the resiliency is guaranteed by the usual `etcd` mechanism, and the pods' count remains under control, so it solves the main goal of resiliency and costs optimization. The trade-off here is that we have to operate an external `etcd` cluster and manage the access to be sure that each tenant cluster uses only its data. Also, there are limits in size in `etcd`, defaulted to 2GB and configurable to a maximum of 8GB. We’re solving this issue by pooling multiple `etcd` and sharding the tenant control planes.
 
+## Getting started
+
+Please refer to the [Getting Started guide](./docs/getting-started-with-kamaji.md) to deploy a minimal setup of Kamaji on KinD.
+
 ## Use cases
 Kamaji project has been initially started as a solution for actual and common problems such as minimizing the Total Cost of Ownership while running Kubernetes at large scale. However, it can open a wider range of use cases. Here are a few:
 

api/interfaces.go

@@ -1,6 +0,0 @@
-package api
-
-type KubeadmConfigResourceVersionDependant interface {
-	GetKubeadmConfigResourceVersion() string
-	SetKubeadmConfigResourceVersion(string)
-}

api/v1alpha1/tenantcontrolplane_addons_funcs.go

@@ -0,0 +1,17 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package v1alpha1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func (in AddonStatus) GetChecksum() string {
+	return in.Checksum
+}
+
+func (in *AddonStatus) SetChecksum(checksum string) {
+	in.LastUpdate = metav1.Now()
+	in.Checksum = checksum
+}

api/v1alpha1/tenantcontrolplane_funcs.go

@@ -5,6 +5,9 @@ package v1alpha1
 
 import (
 	"context"
+	"fmt"
+	"net"
+	"strconv"
 
 	"github.com/pkg/errors"
 	corev1 "k8s.io/api/core/v1"
@@ -14,7 +17,30 @@ import (
 	kamajierrors "github.com/clastix/kamaji/internal/errors"
 )
 
-func (in *TenantControlPlane) GetAddress(ctx context.Context, client client.Client) (string, error) {
+// AssignedControlPlaneAddress returns the announced address and port of a Tenant Control Plane.
+// In case of non-well formed values, or missing announcement, an error is returned.
+func (in *TenantControlPlane) AssignedControlPlaneAddress() (string, int32, error) {
+	if len(in.Status.ControlPlaneEndpoint) == 0 {
+		return "", 0, fmt.Errorf("the Tenant Control Plane is not yet exposed")
+	}
+
+	address, portString, err := net.SplitHostPort(in.Status.ControlPlaneEndpoint)
+	if err != nil {
+		return "", 0, errors.Wrap(err, "cannot split host port from Tenant Control Plane endpoint")
+	}
+
+	port, err := strconv.Atoi(portString)
+	if err != nil {
+		return "", 0, errors.Wrap(err, "cannot convert Tenant Control Plane port from endpoint")
+	}
+
+	return address, int32(port), nil
+}
+
+// DeclaredControlPlaneAddress returns the desired Tenant Control Plane address.
+// In case of dynamic allocation, e.g. using a Load Balancer, it queries the API Server looking for the allocated IP.
+// When an IP has not been yet assigned, or it is expected, an error is returned.
+func (in *TenantControlPlane) DeclaredControlPlaneAddress(ctx context.Context, client client.Client) (string, error) {
 	var loadBalancerStatus corev1.LoadBalancerStatus
 
 	svc := &corev1.Service{}
@@ -27,6 +53,8 @@ func (in *TenantControlPlane) GetAddress(ctx context.Context, client client.Clie
 	case len(in.Spec.NetworkProfile.Address) > 0:
 		// Returning the hard-coded value in the specification in case of non LoadBalanced resources
 		return in.Spec.NetworkProfile.Address, nil
+	case svc.Spec.Type == corev1.ServiceTypeClusterIP:
+		return svc.Spec.ClusterIP, nil
 	case svc.Spec.Type == corev1.ServiceTypeLoadBalancer:
 		loadBalancerStatus = svc.Status.LoadBalancer
 		if len(loadBalancerStatus.Ingress) == 0 {

api/v1alpha1/tenantcontrolplane_interfaces.go

@@ -0,0 +1,12 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package v1alpha1
+
+// KubeadmConfigChecksumDependant is the interface used to retrieve the checksum of the kubeadm phases and addons
+// configuration, required to validate the changes and, upon from that, perform the required reconciliation.
+// +kubebuilder:object:generate=false
+type KubeadmConfigChecksumDependant interface {
+	GetChecksum() string
+	SetChecksum(string)
+}

api/v1alpha1/tenantcontrolplane_kubeadmphase_funcs.go

@@ -0,0 +1,17 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package v1alpha1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func (in KubeadmPhaseStatus) GetChecksum() string {
+	return in.Checksum
+}
+
+func (in *KubeadmPhaseStatus) SetChecksum(checksum string) {
+	in.LastUpdate = metav1.Now()
+	in.Checksum = checksum
+}

api/v1alpha1/tenantcontrolplane_status.go

@@ -0,0 +1,242 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package v1alpha1
+
+import (
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	networkingv1 "k8s.io/api/networking/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/clastix/kamaji/internal/etcd"
+)
+
+// APIServerCertificatesStatus defines the observed state of ETCD Certificate for API server.
+type APIServerCertificatesStatus struct {
+	SecretName string      `json:"secretName,omitempty"`
+	LastUpdate metav1.Time `json:"lastUpdate,omitempty"`
+}
+
+// ETCDCertificateStatus defines the observed state of ETCD Certificate for API server.
+type ETCDCertificateStatus struct {
+	SecretName string      `json:"secretName,omitempty"`
+	LastUpdate metav1.Time `json:"lastUpdate,omitempty"`
+}
+
+// ETCDCertificatesStatus defines the observed state of ETCD Certificate for API server.
+type ETCDCertificatesStatus struct {
+	APIServer ETCDCertificateStatus `json:"apiServer,omitempty"`
+	CA        ETCDCertificateStatus `json:"ca,omitempty"`
+}
+
+// CertificatePrivateKeyPairStatus defines the status.
+type CertificatePrivateKeyPairStatus struct {
+	SecretName      string      `json:"secretName,omitempty"`
+	LastUpdate      metav1.Time `json:"lastUpdate,omitempty"`
+	ResourceVersion string      `json:"resourceVersion,omitempty"`
+}
+
+// PublicKeyPrivateKeyPairStatus defines the status.
+type PublicKeyPrivateKeyPairStatus struct {
+	SecretName string      `json:"secretName,omitempty"`
+	LastUpdate metav1.Time `json:"lastUpdate,omitempty"`
+}
+
+// CertificatesStatus defines the observed state of ETCD Certificates.
+type CertificatesStatus struct {
+	CA                     CertificatePrivateKeyPairStatus `json:"ca,omitempty"`
+	APIServer              CertificatePrivateKeyPairStatus `json:"apiServer,omitempty"`
+	APIServerKubeletClient CertificatePrivateKeyPairStatus `json:"apiServerKubeletClient,omitempty"`
+	FrontProxyCA           CertificatePrivateKeyPairStatus `json:"frontProxyCA,omitempty"`
+	FrontProxyClient       CertificatePrivateKeyPairStatus `json:"frontProxyClient,omitempty"`
+	SA                     PublicKeyPrivateKeyPairStatus   `json:"sa,omitempty"`
+	ETCD                   *ETCDCertificatesStatus         `json:"etcd,omitempty"`
+}
+
+// ETCDStatus defines the observed state of ETCDStatus.
+type ETCDStatus struct {
+	Role etcd.Role `json:"role,omitempty"`
+	User etcd.User `json:"user,omitempty"`
+}
+
+type SQLCertificateStatus struct {
+	SecretName      string      `json:"secretName,omitempty"`
+	ResourceVersion string      `json:"resourceVersion,omitempty"`
+	LastUpdate      metav1.Time `json:"lastUpdate,omitempty"`
+}
+
+type SQLConfigStatus struct {
+	SecretName      string `json:"secretName,omitempty"`
+	ResourceVersion string `json:"resourceVersion,omitempty"`
+}
+
+type SQLSetupStatus struct {
+	Schema                   string      `json:"schema,omitempty"`
+	User                     string      `json:"user,omitempty"`
+	LastUpdate               metav1.Time `json:"lastUpdate,omitempty"`
+	SQLConfigResourceVersion string      `json:"sqlConfigResourceVersion,omitempty"`
+}
+
+type KineMySQLStatus struct {
+	Config      SQLConfigStatus      `json:"config,omitempty"`
+	Setup       SQLSetupStatus       `json:"setup,omitempty"`
+	Certificate SQLCertificateStatus `json:"certificate,omitempty"`
+}
+
+// StorageStatus defines the observed state of StorageStatus.
+type StorageStatus struct {
+	ETCD      *ETCDStatus      `json:"etcd,omitempty"`
+	KineMySQL *KineMySQLStatus `json:"kineMySQL,omitempty"`
+}
+
+// KubeconfigStatus contains information about the generated kubeconfig.
+type KubeconfigStatus struct {
+	SecretName string      `json:"secretName,omitempty"`
+	LastUpdate metav1.Time `json:"lastUpdate,omitempty"`
+	Checksum   string      `json:"checksum,omitempty"`
+}
+
+// KubeconfigsStatus stores information about all the generated kubeconfig resources.
+type KubeconfigsStatus struct {
+	Admin             KubeconfigStatus `json:"admin,omitempty"`
+	ControllerManager KubeconfigStatus `json:"controllerManager,omitempty"`
+	Scheduler         KubeconfigStatus `json:"scheduler,omitempty"`
+}
+
+// KubeadmConfigStatus contains the status of the configuration required by kubeadm.
+type KubeadmConfigStatus struct {
+	ConfigmapName string      `json:"configmapName,omitempty"`
+	LastUpdate    metav1.Time `json:"lastUpdate,omitempty"`
+	// Checksum of the kubeadm configuration to detect changes
+	Checksum string `json:"checksum,omitempty"`
+}
+
+// KubeadmPhaseStatus contains the status of a kubeadm phase action.
+type KubeadmPhaseStatus struct {
+	Checksum   string      `json:"checksum,omitempty"`
+	LastUpdate metav1.Time `json:"lastUpdate,omitempty"`
+}
+
+// KubeadmPhasesStatus contains the status of the different kubeadm phases action.
+type KubeadmPhasesStatus struct {
+	UploadConfigKubeadm KubeadmPhaseStatus `json:"uploadConfigKubeadm"`
+	UploadConfigKubelet KubeadmPhaseStatus `json:"uploadConfigKubelet"`
+	BootstrapToken      KubeadmPhaseStatus `json:"bootstrapToken"`
+}
+
+type ExternalKubernetesObjectStatus struct {
+	Name      string `json:"name,omitempty"`
+	Namespace string `json:"namespace,omitempty"`
+	// Resource version of k8s object
+	RV string `json:"resourceVersion,omitempty"`
+	// Last time when k8s object was updated
+	LastUpdate metav1.Time `json:"lastUpdate,omitempty"`
+}
+
+// KonnectivityStatus defines the status of Konnectivity as Addon.
+type KonnectivityStatus struct {
+	Enabled                     bool                            `json:"enabled"`
+	EgressSelectorConfiguration string                          `json:"egressSelectorConfiguration,omitempty"`
+	Certificate                 CertificatePrivateKeyPairStatus `json:"certificate,omitempty"`
+	Kubeconfig                  KubeconfigStatus                `json:"kubeconfig,omitempty"`
+	ServiceAccount              ExternalKubernetesObjectStatus  `json:"sa,omitempty"`
+	ClusterRoleBinding          ExternalKubernetesObjectStatus  `json:"clusterrolebinding,omitempty"`
+	Agent                       ExternalKubernetesObjectStatus  `json:"agent,omitempty"`
+	Service                     KubernetesServiceStatus         `json:"service,omitempty"`
+}
+
+// AddonStatus defines the observed state of an Addon.
+type AddonStatus struct {
+	Enabled    bool        `json:"enabled"`
+	Checksum   string      `json:"checksum,omitempty"`
+	LastUpdate metav1.Time `json:"lastUpdate,omitempty"`
+}
+
+// AddonsStatus defines the observed state of the different Addons.
+type AddonsStatus struct {
+	CoreDNS   AddonStatus `json:"coreDNS,omitempty"`
+	KubeProxy AddonStatus `json:"kubeProxy,omitempty"`
+
+	Konnectivity KonnectivityStatus `json:"konnectivity,omitempty"`
+}
+
+// TenantControlPlaneStatus defines the observed state of TenantControlPlane.
+type TenantControlPlaneStatus struct {
+	// Storage Status contains information about Kubernetes storage system
+	Storage StorageStatus `json:"storage,omitempty"`
+	// Certificates contains information about the different certificates
+	// that are necessary to run a kubernetes control plane
+	Certificates CertificatesStatus `json:"certificates,omitempty"`
+	// KubeConfig contains information about the kubenconfigs that control plane pieces need
+	KubeConfig KubeconfigsStatus `json:"kubeconfig,omitempty"`
+	// Kubernetes contains information about the reconciliation of the required Kubernetes resources deployed in the admin cluster
+	Kubernetes KubernetesStatus `json:"kubernetesResources,omitempty"`
+	// KubeadmConfig contains the status of the configuration required by kubeadm
+	KubeadmConfig KubeadmConfigStatus `json:"kubeadmconfig,omitempty"`
+	// KubeadmPhase contains the status of the kubeadm phases action
+	KubeadmPhase KubeadmPhasesStatus `json:"kubeadmPhase,omitempty"`
+	// ControlPlaneEndpoint contains the status of the kubernetes control plane
+	ControlPlaneEndpoint string `json:"controlPlaneEndpoint,omitempty"`
+	// Addons contains the status of the different Addons
+	Addons AddonsStatus `json:"addons,omitempty"`
+}
+
+// KubernetesStatus defines the status of the resources deployed in the management cluster,
+// such as Deployment and Service.
+type KubernetesStatus struct {
+	// KubernetesVersion contains the information regarding the running Kubernetes version, and its upgrade status.
+	Version    KubernetesVersion          `json:"version,omitempty"`
+	Deployment KubernetesDeploymentStatus `json:"deployment,omitempty"`
+	Service    KubernetesServiceStatus    `json:"service,omitempty"`
+	Ingress    KubernetesIngressStatus    `json:"ingress,omitempty"`
+}
+
+// +kubebuilder:validation:Enum=Provisioning;Upgrading;Ready;NotReady
+type KubernetesVersionStatus string
+
+var (
+	VersionProvisioning KubernetesVersionStatus = "Provisioning"
+	VersionUpgrading    KubernetesVersionStatus = "Upgrading"
+	VersionReady        KubernetesVersionStatus = "Ready"
+	VersionNotReady     KubernetesVersionStatus = "NotReady"
+)
+
+type KubernetesVersion struct {
+	// Version is the running Kubernetes version of the Tenant Control Plane.
+	Version string `json:"version,omitempty"`
+	// +kubebuilder:default=Provisioning
+	// Status returns the current status of the Kubernetes version, such as its provisioning state, or completed upgrade.
+	Status *KubernetesVersionStatus `json:"status"`
+}
+
+// KubernetesDeploymentStatus defines the status for the Tenant Control Plane Deployment in the management cluster.
+type KubernetesDeploymentStatus struct {
+	appsv1.DeploymentStatus `json:",inline"`
+	// The name of the Deployment for the given cluster.
+	Name string `json:"name"`
+	// The namespace which the Deployment for the given cluster is deployed.
+	Namespace string `json:"namespace"`
+	// Last time when deployment was updated
+	LastUpdate metav1.Time `json:"lastUpdate,omitempty"`
+}
+
+// KubernetesServiceStatus defines the status for the Tenant Control Plane Service in the management cluster.
+type KubernetesServiceStatus struct {
+	corev1.ServiceStatus `json:",inline"`
+	// The name of the Service for the given cluster.
+	Name string `json:"name"`
+	// The namespace which the Service for the given cluster is deployed.
+	Namespace string `json:"namespace"`
+	// The port where the service is running
+	Port int32 `json:"port"`
+}
+
+// KubernetesIngressStatus defines the status for the Tenant Control Plane Ingress in the management cluster.
+type KubernetesIngressStatus struct {
+	networkingv1.IngressStatus `json:",inline"`
+	// The name of the Ingress for the given cluster.
+	Name string `json:"name"`
+	// The namespace which the Ingress for the given cluster is deployed.
+	Namespace string `json:"namespace"`
+}

api/v1alpha1/tenantcontrolplane_types.go

@@ -4,12 +4,8 @@
 package v1alpha1
 
 import (
-	appv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
-	networkingv1 "k8s.io/api/networking/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-
-	"github.com/clastix/kamaji/internal/etcd"
 )
 
 // NetworkProfileSpec defines the desired state of NetworkProfile.
@@ -22,15 +18,18 @@ type NetworkProfileSpec struct {
 	AllowAddressAsExternalIP bool `json:"allowAddressAsExternalIP,omitempty"`
 	// Port where API server of will be exposed
 	// +kubebuilder:default=6443
-	Port int32 `json:"port"`
-
-	// Domain of the tenant control plane
-	Domain string `json:"domain"`
+	Port int32 `json:"port,omitempty"`
+	// CertSANs sets extra Subject Alternative Names (SANs) for the API Server signing certificate.
+	// Use this field to add additional hostnames when exposing the Tenant Control Plane with third solutions.
+	CertSANs []string `json:"certSANs,omitempty"`
 	// Kubernetes Service
-	ServiceCIDR string `json:"serviceCidr"`
+	// +kubebuilder:default="10.96.0.0/16"
+	ServiceCIDR string `json:"serviceCidr,omitempty"`
 	// CIDR for Kubernetes Pods
-	PodCIDR       string   `json:"podCidr"`
-	DNSServiceIPs []string `json:"dnsServiceIPs"`
+	// +kubebuilder:default="10.244.0.0/16"
+	PodCIDR string `json:"podCidr,omitempty"`
+	// +kubebuilder:default={"10.96.0.10"}
+	DNSServiceIPs []string `json:"dnsServiceIPs,omitempty"`
 }
 
 type KubeletSpec struct {
@@ -78,10 +77,31 @@ type IngressSpec struct {
 	Hostname string `json:"hostname,omitempty"`
 }
 
+type ControlPlaneComponentsResources struct {
+	APIServer         *corev1.ResourceRequirements `json:"apiServer,omitempty"`
+	ControllerManager *corev1.ResourceRequirements `json:"controllerManager,omitempty"`
+	Scheduler         *corev1.ResourceRequirements `json:"scheduler,omitempty"`
+}
+
 type DeploymentSpec struct {
 	// +kubebuilder:default=2
-	Replicas           int32              `json:"replicas,omitempty"`
-	AdditionalMetadata AdditionalMetadata `json:"additionalMetadata,omitempty"`
+	Replicas int32 `json:"replicas,omitempty"`
+	// Resources defines the amount of memory and CPU to allocate to each component of the Control Plane
+	// (kube-apiserver, controller-manager, and scheduler).
+	Resources *ControlPlaneComponentsResources `json:"resources,omitempty"`
+	// ExtraArgs allows adding additional arguments to the Control Plane components,
+	// such as kube-apiserver, controller-manager, and scheduler.
+	ExtraArgs          *ControlPlaneExtraArgs `json:"extraArgs,omitempty"`
+	AdditionalMetadata AdditionalMetadata     `json:"additionalMetadata,omitempty"`
+}
+
+// ControlPlaneExtraArgs allows specifying additional arguments to the Control Plane components.
+type ControlPlaneExtraArgs struct {
+	APIServer         []string `json:"apiServer,omitempty"`
+	ControllerManager []string `json:"controllerManager,omitempty"`
+	Scheduler         []string `json:"scheduler,omitempty"`
+	// Available only if Kamaji is running using Kine as backing storage.
+	Kine []string `json:"kine,omitempty"`
 }
 
 type ServiceSpec struct {
@@ -91,226 +111,43 @@ type ServiceSpec struct {
 }
 
 // AddonSpec defines the spec for every addon.
-type AddonSpec struct {
-	// +kubebuilder:default=true
-	Enabled *bool `json:"enabled,omitempty"`
+type AddonSpec struct{}
+
+// KonnectivitySpec defines the spec for Konnectivity.
+type KonnectivitySpec struct {
+	// Port of Konnectivity proxy server.
+	ProxyPort int32 `json:"proxyPort"`
+	// Version for Konnectivity server and agent.
+	// +kubebuilder:default=v0.0.31
+	Version string `json:"version,omitempty"`
+	// ServerImage defines the container image for Konnectivity's server.
+	// +kubebuilder:default=us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-server
+	ServerImage string `json:"serverImage,omitempty"`
+	// AgentImage defines the container image for Konnectivity's agent.
+	// +kubebuilder:default=us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-agent
+	AgentImage string `json:"agentImage,omitempty"`
+	// Resources define the amount of CPU and memory to allocate to the Konnectivity server.
+	Resources *corev1.ResourceRequirements `json:"resources,omitempty"`
 }
 
 // AddonsSpec defines the enabled addons and their features.
 type AddonsSpec struct {
-	// +kubebuilder:default={enabled: true}
-	CoreDNS AddonSpec `json:"coreDNS,omitempty"`
-	// +kubebuilder:default={enabled: true}
-	KubeProxy AddonSpec `json:"kubeProxy,omitempty"`
+	CoreDNS      *AddonSpec        `json:"coreDNS,omitempty"`
+	Konnectivity *KonnectivitySpec `json:"konnectivity,omitempty"`
+	KubeProxy    *AddonSpec        `json:"kubeProxy,omitempty"`
 }
 
 // TenantControlPlaneSpec defines the desired state of TenantControlPlane.
 type TenantControlPlaneSpec struct {
 	ControlPlane ControlPlane `json:"controlPlane"`
-
 	// Kubernetes specification for tenant control plane
 	Kubernetes KubernetesSpec `json:"kubernetes"`
-
 	// NetworkProfile specifies how the network is
 	NetworkProfile NetworkProfileSpec `json:"networkProfile,omitempty"`
-
 	// Addons contain which addons are enabled
-	// +kubebuilder:default={coreDNS: {enabled: true}, kubeProxy: {enabled: true}}
 	Addons AddonsSpec `json:"addons,omitempty"`
 }
 
-// ETCDAPIServerCertificate defines the observed state of ETCD Certificate for API server.
-type APIServerCertificatesStatus struct {
-	SecretName string      `json:"secretName,omitempty"`
-	LastUpdate metav1.Time `json:"lastUpdate,omitempty"`
-}
-
-// ETCDAPIServerCertificate defines the observed state of ETCD Certificate for API server.
-type ETCDCertificateStatus struct {
-	SecretName string      `json:"secretName,omitempty"`
-	LastUpdate metav1.Time `json:"lastUpdate,omitempty"`
-}
-
-// ETCDAPIServerCertificate defines the observed state of ETCD Certificate for API server.
-type ETCDCertificatesStatus struct {
-	APIServer ETCDCertificateStatus `json:"apiServer,omitempty"`
-	CA        ETCDCertificateStatus `json:"ca,omitempty"`
-}
-
-// CertificatePrivateKeyPair defines the status.
-type CertificatePrivateKeyPairStatus struct {
-	SecretName string      `json:"secretName,omitempty"`
-	LastUpdate metav1.Time `json:"lastUpdate,omitempty"`
-}
-
-// CertificatePrivateKeyPair defines the status.
-type PublicKeyPrivateKeyPairStatus struct {
-	SecretName string      `json:"secretName,omitempty"`
-	LastUpdate metav1.Time `json:"lastUpdate,omitempty"`
-}
-
-// ETCDCertificates defines the observed state of ETCD Certificates.
-type CertificatesStatus struct {
-	CA                     CertificatePrivateKeyPairStatus `json:"ca,omitempty"`
-	APIServer              CertificatePrivateKeyPairStatus `json:"apiServer,omitempty"`
-	APIServerKubeletClient CertificatePrivateKeyPairStatus `json:"apiServerKubeletClient,omitempty"`
-	FrontProxyCA           CertificatePrivateKeyPairStatus `json:"frontProxyCA,omitempty"`
-	FrontProxyClient       CertificatePrivateKeyPairStatus `json:"frontProxyClient,omitempty"`
-	SA                     PublicKeyPrivateKeyPairStatus   `json:"sa,omitempty"`
-	ETCD                   *ETCDCertificatesStatus         `json:"etcd,omitempty"`
-}
-
-// ETCDStatus defines the observed state of ETCDStatus.
-type ETCDStatus struct {
-	Role etcd.Role `json:"role,omitempty"`
-	User etcd.User `json:"user,omitempty"`
-}
-
-// StorageStatus defines the observed state of StorageStatus.
-type StorageStatus struct {
-	ETCD *ETCDStatus `json:"etcd,omitempty"`
-}
-
-// TenantControlPlaneKubeconfigsStatus contains information about a the generated kubeconfig.
-type KubeconfigStatus struct {
-	SecretName string      `json:"secretName,omitempty"`
-	LastUpdate metav1.Time `json:"lastUpdate,omitempty"`
-}
-
-// TenantControlPlaneKubeconfigsStatus stores information about all the generated kubeconfigs.
-type KubeconfigsStatus struct {
-	Admin             KubeconfigStatus `json:"admin,omitempty"`
-	ControllerManager KubeconfigStatus `json:"controlerManager,omitempty"`
-	Scheduler         KubeconfigStatus `json:"scheduler,omitempty"`
-}
-
-// KubeadmConfigStatus contains the status of the configuration required by kubeadm.
-type KubeadmConfigStatus struct {
-	ConfigmapName   string      `json:"configmapName,omitempty"`
-	LastUpdate      metav1.Time `json:"lastUpdate,omitempty"`
-	ResourceVersion string      `json:"resourceVersion"`
-}
-
-// KubeadmPhasesStatus contains the status of of a kubeadm phase action.
-type KubeadmPhaseStatus struct {
-	KubeadmConfigResourceVersion string      `json:"kubeadmConfigResourceVersion,omitempty"`
-	LastUpdate                   metav1.Time `json:"lastUpdate,omitempty"`
-}
-
-func (d KubeadmPhaseStatus) GetKubeadmConfigResourceVersion() string {
-	return d.KubeadmConfigResourceVersion
-}
-
-func (d *KubeadmPhaseStatus) SetKubeadmConfigResourceVersion(rv string) {
-	d.KubeadmConfigResourceVersion = rv
-}
-
-// KubeadmPhasesStatus contains the status of the different kubeadm phases action.
-type KubeadmPhasesStatus struct {
-	UploadConfigKubeadm KubeadmPhaseStatus `json:"uploadConfigKubeadm"`
-	UploadConfigKubelet KubeadmPhaseStatus `json:"uploadConfigKubelet"`
-	BootstrapToken      KubeadmPhaseStatus `json:"bootstrapToken"`
-}
-
-// AddonStatus defines the observed state of an Addon.
-type AddonStatus struct {
-	Enabled                      bool        `json:"enabled"`
-	KubeadmConfigResourceVersion string      `json:"kubeadmConfigResourceVersion,omitempty"`
-	LastUpdate                   metav1.Time `json:"lastUpdate,omitempty"`
-}
-
-func (d AddonStatus) GetKubeadmConfigResourceVersion() string {
-	return d.KubeadmConfigResourceVersion
-}
-
-func (d *AddonStatus) SetKubeadmConfigResourceVersion(rv string) {
-	d.KubeadmConfigResourceVersion = rv
-}
-
-// AddonsStatus defines the observed state of the different Addons.
-type AddonsStatus struct {
-	CoreDNS   AddonStatus `json:"coreDNS,omitempty"`
-	KubeProxy AddonStatus `json:"kubeProxy,omitempty"`
-}
-
-// TenantControlPlaneStatus defines the observed state of TenantControlPlane.
-type TenantControlPlaneStatus struct {
-	// Storage Status contains information about Kubernetes storage system
-	Storage StorageStatus `json:"storage,omitempty"`
-	// Certificates contains information about the different certificates
-	// that are necessary to run a kubernetes control plane
-	Certificates CertificatesStatus `json:"certificates,omitempty"`
-	// KubeConfig contains information about the kubenconfigs that control plane pieces need
-	KubeConfig KubeconfigsStatus `json:"kubeconfig,omitempty"`
-	// Kubernetes contains information about the reconciliation of the required Kubernetes resources deployed in the admin cluster
-	Kubernetes KubernetesStatus `json:"kubernetesResources,omitempty"`
-	// KubeadmConfig contains the status of the configuration required by kubeadm
-	KubeadmConfig KubeadmConfigStatus `json:"kubeadmconfig,omitempty"`
-	// KubeadmPhase contains the status of the kubeadm phases action
-	KubeadmPhase KubeadmPhasesStatus `json:"kubeadmPhase,omitempty"`
-	// ControlPlaneEndpoint contains the status of the kubernetes control plane
-	ControlPlaneEndpoint string `json:"controlPlaneEndpoint,omitempty"`
-	// Addons contains the status of the different Addons
-	Addons AddonsStatus `json:"addons,omitempty"`
-}
-
-// KubernetesStatus defines the status of the resources deployed in the management cluster,
-// such as Deployment and Service.
-type KubernetesStatus struct {
-	// KubernetesVersion contains the information regarding the running Kubernetes version, and its upgrade status.
-	Version    KubernetesVersion          `json:"version,omitempty"`
-	Deployment KubernetesDeploymentStatus `json:"deployment,omitempty"`
-	Service    KubernetesServiceStatus    `json:"service,omitempty"`
-	Ingress    KubernetesIngressStatus    `json:"ingress,omitempty"`
-}
-
-// +kubebuilder:validation:Enum=Provisioning;Upgrading;Ready;NotReady
-type KubernetesVersionStatus string
-
-var (
-	VersionProvisioning KubernetesVersionStatus = "Provisioning"
-	VersionUpgrading    KubernetesVersionStatus = "Upgrading"
-	VersionReady        KubernetesVersionStatus = "Ready"
-	VersionNotReady     KubernetesVersionStatus = "NotReady"
-)
-
-type KubernetesVersion struct {
-	// Version is the running Kubernetes version of the Tenant Control Plane.
-	Version string `json:"version,omitempty"`
-	// +kubebuilder:default=Provisioning
-	// Status returns the current status of the Kubernetes version, such as its provisioning state, or completed upgrade.
-	Status *KubernetesVersionStatus `json:"status"`
-}
-
-// KubernetesDeploymentStatus defines the status for the Tenant Control Plane Deployment in the management cluster.
-type KubernetesDeploymentStatus struct {
-	appv1.DeploymentStatus `json:",inline"`
-	// The name of the Deployment for the given cluster.
-	Name string `json:"name"`
-	// The namespace which the Deployment for the given cluster is deployed.
-	Namespace string `json:"namespace"`
-}
-
-// KubernetesServiceStatus defines the status for the Tenant Control Plane Service in the management cluster.
-type KubernetesServiceStatus struct {
-	corev1.ServiceStatus `json:",inline"`
-	// The name of the Service for the given cluster.
-	Name string `json:"name"`
-	// The namespace which the Service for the given cluster is deployed.
-	Namespace string `json:"namespace"`
-	// The port where the service is running
-	Port int32 `json:"port"`
-}
-
-// KubernetesIngressStatus defines the status for the Tenant Control Plane Ingress in the management cluster.
-type KubernetesIngressStatus struct {
-	networkingv1.IngressStatus `json:",inline"`
-	// The name of the Ingress for the given cluster.
-	Name string `json:"name"`
-	// The namespace which the Ingress for the given cluster is deployed.
-	Namespace string `json:"namespace"`
-}
-
 // +kubebuilder:object:root=true
 // +kubebuilder:subresource:status
 // +kubebuilder:resource:shortName=tcp

api/v1alpha1/zz_generated.deepcopy.go

@@ -9,6 +9,7 @@
 package v1alpha1
 
 import (
+	"k8s.io/api/core/v1"
 	runtime "k8s.io/apimachinery/pkg/runtime"
 )
 
@@ -60,11 +61,6 @@ func (in *AdditionalMetadata) DeepCopy() *AdditionalMetadata {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *AddonSpec) DeepCopyInto(out *AddonSpec) {
 	*out = *in
-	if in.Enabled != nil {
-		in, out := &in.Enabled, &out.Enabled
-		*out = new(bool)
-		**out = **in
-	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AddonSpec.
@@ -96,8 +92,21 @@ func (in *AddonStatus) DeepCopy() *AddonStatus {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *AddonsSpec) DeepCopyInto(out *AddonsSpec) {
 	*out = *in
-	in.CoreDNS.DeepCopyInto(&out.CoreDNS)
-	in.KubeProxy.DeepCopyInto(&out.KubeProxy)
+	if in.CoreDNS != nil {
+		in, out := &in.CoreDNS, &out.CoreDNS
+		*out = new(AddonSpec)
+		**out = **in
+	}
+	if in.Konnectivity != nil {
+		in, out := &in.Konnectivity, &out.Konnectivity
+		*out = new(KonnectivitySpec)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.KubeProxy != nil {
+		in, out := &in.KubeProxy, &out.KubeProxy
+		*out = new(AddonSpec)
+		**out = **in
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AddonsSpec.
@@ -115,6 +124,7 @@ func (in *AddonsStatus) DeepCopyInto(out *AddonsStatus) {
 	*out = *in
 	in.CoreDNS.DeepCopyInto(&out.CoreDNS)
 	in.KubeProxy.DeepCopyInto(&out.KubeProxy)
+	in.Konnectivity.DeepCopyInto(&out.Konnectivity)
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AddonsStatus.
@@ -206,9 +216,84 @@ func (in *ControlPlane) DeepCopy() *ControlPlane {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ControlPlaneComponentsResources) DeepCopyInto(out *ControlPlaneComponentsResources) {
+	*out = *in
+	if in.APIServer != nil {
+		in, out := &in.APIServer, &out.APIServer
+		*out = new(v1.ResourceRequirements)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.ControllerManager != nil {
+		in, out := &in.ControllerManager, &out.ControllerManager
+		*out = new(v1.ResourceRequirements)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.Scheduler != nil {
+		in, out := &in.Scheduler, &out.Scheduler
+		*out = new(v1.ResourceRequirements)
+		(*in).DeepCopyInto(*out)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ControlPlaneComponentsResources.
+func (in *ControlPlaneComponentsResources) DeepCopy() *ControlPlaneComponentsResources {
+	if in == nil {
+		return nil
+	}
+	out := new(ControlPlaneComponentsResources)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ControlPlaneExtraArgs) DeepCopyInto(out *ControlPlaneExtraArgs) {
+	*out = *in
+	if in.APIServer != nil {
+		in, out := &in.APIServer, &out.APIServer
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
+	if in.ControllerManager != nil {
+		in, out := &in.ControllerManager, &out.ControllerManager
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
+	if in.Scheduler != nil {
+		in, out := &in.Scheduler, &out.Scheduler
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
+	if in.Kine != nil {
+		in, out := &in.Kine, &out.Kine
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ControlPlaneExtraArgs.
+func (in *ControlPlaneExtraArgs) DeepCopy() *ControlPlaneExtraArgs {
+	if in == nil {
+		return nil
+	}
+	out := new(ControlPlaneExtraArgs)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *DeploymentSpec) DeepCopyInto(out *DeploymentSpec) {
 	*out = *in
+	if in.Resources != nil {
+		in, out := &in.Resources, &out.Resources
+		*out = new(ControlPlaneComponentsResources)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.ExtraArgs != nil {
+		in, out := &in.ExtraArgs, &out.ExtraArgs
+		*out = new(ControlPlaneExtraArgs)
+		(*in).DeepCopyInto(*out)
+	}
 	in.AdditionalMetadata.DeepCopyInto(&out.AdditionalMetadata)
 }
 
@@ -272,6 +357,22 @@ func (in *ETCDStatus) DeepCopy() *ETCDStatus {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ExternalKubernetesObjectStatus) DeepCopyInto(out *ExternalKubernetesObjectStatus) {
+	*out = *in
+	in.LastUpdate.DeepCopyInto(&out.LastUpdate)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExternalKubernetesObjectStatus.
+func (in *ExternalKubernetesObjectStatus) DeepCopy() *ExternalKubernetesObjectStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(ExternalKubernetesObjectStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *IngressSpec) DeepCopyInto(out *IngressSpec) {
 	*out = *in
@@ -288,6 +389,65 @@ func (in *IngressSpec) DeepCopy() *IngressSpec {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *KineMySQLStatus) DeepCopyInto(out *KineMySQLStatus) {
+	*out = *in
+	out.Config = in.Config
+	in.Setup.DeepCopyInto(&out.Setup)
+	in.Certificate.DeepCopyInto(&out.Certificate)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KineMySQLStatus.
+func (in *KineMySQLStatus) DeepCopy() *KineMySQLStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(KineMySQLStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *KonnectivitySpec) DeepCopyInto(out *KonnectivitySpec) {
+	*out = *in
+	if in.Resources != nil {
+		in, out := &in.Resources, &out.Resources
+		*out = new(v1.ResourceRequirements)
+		(*in).DeepCopyInto(*out)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KonnectivitySpec.
+func (in *KonnectivitySpec) DeepCopy() *KonnectivitySpec {
+	if in == nil {
+		return nil
+	}
+	out := new(KonnectivitySpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *KonnectivityStatus) DeepCopyInto(out *KonnectivityStatus) {
+	*out = *in
+	in.Certificate.DeepCopyInto(&out.Certificate)
+	in.Kubeconfig.DeepCopyInto(&out.Kubeconfig)
+	in.ServiceAccount.DeepCopyInto(&out.ServiceAccount)
+	in.ClusterRoleBinding.DeepCopyInto(&out.ClusterRoleBinding)
+	in.Agent.DeepCopyInto(&out.Agent)
+	in.Service.DeepCopyInto(&out.Service)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KonnectivityStatus.
+func (in *KonnectivityStatus) DeepCopy() *KonnectivityStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(KonnectivityStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *KubeadmConfigStatus) DeepCopyInto(out *KubeadmConfigStatus) {
 	*out = *in
@@ -391,6 +551,7 @@ func (in *KubeletSpec) DeepCopy() *KubeletSpec {
 func (in *KubernetesDeploymentStatus) DeepCopyInto(out *KubernetesDeploymentStatus) {
 	*out = *in
 	in.DeploymentStatus.DeepCopyInto(&out.DeploymentStatus)
+	in.LastUpdate.DeepCopyInto(&out.LastUpdate)
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubernetesDeploymentStatus.
@@ -498,6 +659,11 @@ func (in *KubernetesVersion) DeepCopy() *KubernetesVersion {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *NetworkProfileSpec) DeepCopyInto(out *NetworkProfileSpec) {
 	*out = *in
+	if in.CertSANs != nil {
+		in, out := &in.CertSANs, &out.CertSANs
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
 	if in.DNSServiceIPs != nil {
 		in, out := &in.DNSServiceIPs, &out.DNSServiceIPs
 		*out = make([]string, len(*in))
@@ -531,6 +697,53 @@ func (in *PublicKeyPrivateKeyPairStatus) DeepCopy() *PublicKeyPrivateKeyPairStat
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *SQLCertificateStatus) DeepCopyInto(out *SQLCertificateStatus) {
+	*out = *in
+	in.LastUpdate.DeepCopyInto(&out.LastUpdate)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SQLCertificateStatus.
+func (in *SQLCertificateStatus) DeepCopy() *SQLCertificateStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(SQLCertificateStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *SQLConfigStatus) DeepCopyInto(out *SQLConfigStatus) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SQLConfigStatus.
+func (in *SQLConfigStatus) DeepCopy() *SQLConfigStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(SQLConfigStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *SQLSetupStatus) DeepCopyInto(out *SQLSetupStatus) {
+	*out = *in
+	in.LastUpdate.DeepCopyInto(&out.LastUpdate)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SQLSetupStatus.
+func (in *SQLSetupStatus) DeepCopy() *SQLSetupStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(SQLSetupStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ServiceSpec) DeepCopyInto(out *ServiceSpec) {
 	*out = *in
@@ -555,6 +768,11 @@ func (in *StorageStatus) DeepCopyInto(out *StorageStatus) {
 		*out = new(ETCDStatus)
 		(*in).DeepCopyInto(*out)
 	}
+	if in.KineMySQL != nil {
+		in, out := &in.KineMySQL, &out.KineMySQL
+		*out = new(KineMySQLStatus)
+		(*in).DeepCopyInto(*out)
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StorageStatus.

config/crd/bases/kamaji.clastix.io_tenantcontrolplanes.yaml

@@ -61,30 +61,65 @@ spec:
             description: TenantControlPlaneSpec defines the desired state of TenantControlPlane.
             properties:
               addons:
-                default:
-                  coreDNS:
-                    enabled: true
-                  kubeProxy:
-                    enabled: true
                 description: Addons contain which addons are enabled
                 properties:
                   coreDNS:
-                    default:
-                      enabled: true
                     description: AddonSpec defines the spec for every addon.
+                    type: object
+                  konnectivity:
+                    description: KonnectivitySpec defines the spec for Konnectivity.
                     properties:
-                      enabled:
-                        default: true
-                        type: boolean
+                      agentImage:
+                        default: us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-agent
+                        description: AgentImage defines the container image for Konnectivity's
+                          agent.
+                        type: string
+                      proxyPort:
+                        description: Port of Konnectivity proxy server.
+                        format: int32
+                        type: integer
+                      resources:
+                        description: Resources define the amount of CPU and memory
+                          to allocate to the Konnectivity server.
+                        properties:
+                          limits:
+                            additionalProperties:
+                              anyOf:
+                              - type: integer
+                              - type: string
+                              pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                              x-kubernetes-int-or-string: true
+                            description: 'Limits describes the maximum amount of compute
+                              resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                            type: object
+                          requests:
+                            additionalProperties:
+                              anyOf:
+                              - type: integer
+                              - type: string
+                              pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                              x-kubernetes-int-or-string: true
+                            description: 'Requests describes the minimum amount of
+                              compute resources required. If Requests is omitted for
+                              a container, it defaults to Limits if that is explicitly
+                              specified, otherwise to an implementation-defined value.
+                              More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                            type: object
+                        type: object
+                      serverImage:
+                        default: us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-server
+                        description: ServerImage defines the container image for Konnectivity's
+                          server.
+                        type: string
+                      version:
+                        default: v0.0.31
+                        description: Version for Konnectivity server and agent.
+                        type: string
+                    required:
+                    - proxyPort
                     type: object
                   kubeProxy:
-                    default:
-                      enabled: true
                     description: AddonSpec defines the spec for every addon.
-                    properties:
-                      enabled:
-                        default: true
-                        type: boolean
                     type: object
                 type: object
               controlPlane:
@@ -110,10 +145,124 @@ spec:
                               type: string
                             type: object
                         type: object
+                      extraArgs:
+                        description: ExtraArgs allows adding additional arguments
+                          to the Control Plane components, such as kube-apiserver,
+                          controller-manager, and scheduler.
+                        properties:
+                          apiServer:
+                            items:
+                              type: string
+                            type: array
+                          controllerManager:
+                            items:
+                              type: string
+                            type: array
+                          kine:
+                            description: Available only if Kamaji is running using
+                              Kine as backing storage.
+                            items:
+                              type: string
+                            type: array
+                          scheduler:
+                            items:
+                              type: string
+                            type: array
+                        type: object
                       replicas:
                         default: 2
                         format: int32
                         type: integer
+                      resources:
+                        description: Resources defines the amount of memory and CPU
+                          to allocate to each component of the Control Plane (kube-apiserver,
+                          controller-manager, and scheduler).
+                        properties:
+                          apiServer:
+                            description: ResourceRequirements describes the compute
+                              resource requirements.
+                            properties:
+                              limits:
+                                additionalProperties:
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
+                                description: 'Limits describes the maximum amount
+                                  of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                                type: object
+                              requests:
+                                additionalProperties:
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
+                                description: 'Requests describes the minimum amount
+                                  of compute resources required. If Requests is omitted
+                                  for a container, it defaults to Limits if that is
+                                  explicitly specified, otherwise to an implementation-defined
+                                  value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                                type: object
+                            type: object
+                          controllerManager:
+                            description: ResourceRequirements describes the compute
+                              resource requirements.
+                            properties:
+                              limits:
+                                additionalProperties:
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
+                                description: 'Limits describes the maximum amount
+                                  of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                                type: object
+                              requests:
+                                additionalProperties:
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
+                                description: 'Requests describes the minimum amount
+                                  of compute resources required. If Requests is omitted
+                                  for a container, it defaults to Limits if that is
+                                  explicitly specified, otherwise to an implementation-defined
+                                  value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                                type: object
+                            type: object
+                          scheduler:
+                            description: ResourceRequirements describes the compute
+                              resource requirements.
+                            properties:
+                              limits:
+                                additionalProperties:
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
+                                description: 'Limits describes the maximum amount
+                                  of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                                type: object
+                              requests:
+                                additionalProperties:
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
+                                description: 'Requests describes the minimum amount
+                                  of compute resources required. If Requests is omitted
+                                  for a container, it defaults to Limits if that is
+                                  explicitly specified, otherwise to an implementation-defined
+                                  value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                                type: object
+                            type: object
+                        type: object
                     type: object
                   ingress:
                     description: Defining the options for an Optional Ingress which
@@ -273,14 +422,22 @@ spec:
                       in the section of ExternalIPs of the Kubernetes Service (only
                       ClusterIP or NodePort)
                     type: boolean
-                  dnsServiceIPs:
+                  certSANs:
+                    description: CertSANs sets extra Subject Alternative Names (SANs)
+                      for the API Server signing certificate. Use this field to add
+                      additional hostnames when exposing the Tenant Control Plane
+                      with third solutions.
+                    items:
+                      type: string
+                    type: array
+                  dnsServiceIPs:
+                    default:
+                    - 10.96.0.10
                     items:
                       type: string
                     type: array
-                  domain:
-                    description: Domain of the tenant control plane
-                    type: string
                   podCidr:
+                    default: 10.244.0.0/16
                     description: CIDR for Kubernetes Pods
                     type: string
                   port:
@@ -289,14 +446,9 @@ spec:
                     format: int32
                     type: integer
                   serviceCidr:
+                    default: 10.96.0.0/16
                     description: Kubernetes Service
                     type: string
-                required:
-                - dnsServiceIPs
-                - domain
-                - podCidr
-                - port
-                - serviceCidr
                 type: object
             required:
             - controlPlane
@@ -311,23 +463,267 @@ spec:
                   coreDNS:
                     description: AddonStatus defines the observed state of an Addon.
                     properties:
+                      checksum:
+                        type: string
                       enabled:
                         type: boolean
-                      kubeadmConfigResourceVersion:
-                        type: string
                       lastUpdate:
                         format: date-time
                         type: string
                     required:
                     - enabled
                     type: object
+                  konnectivity:
+                    description: KonnectivityStatus defines the status of Konnectivity
+                      as Addon.
+                    properties:
+                      agent:
+                        properties:
+                          lastUpdate:
+                            description: Last time when k8s object was updated
+                            format: date-time
+                            type: string
+                          name:
+                            type: string
+                          namespace:
+                            type: string
+                          resourceVersion:
+                            description: Resource version of k8s object
+                            type: string
+                        type: object
+                      certificate:
+                        description: CertificatePrivateKeyPairStatus defines the status.
+                        properties:
+                          lastUpdate:
+                            format: date-time
+                            type: string
+                          resourceVersion:
+                            type: string
+                          secretName:
+                            type: string
+                        type: object
+                      clusterrolebinding:
+                        properties:
+                          lastUpdate:
+                            description: Last time when k8s object was updated
+                            format: date-time
+                            type: string
+                          name:
+                            type: string
+                          namespace:
+                            type: string
+                          resourceVersion:
+                            description: Resource version of k8s object
+                            type: string
+                        type: object
+                      egressSelectorConfiguration:
+                        type: string
+                      enabled:
+                        type: boolean
+                      kubeconfig:
+                        description: KubeconfigStatus contains information about the
+                          generated kubeconfig.
+                        properties:
+                          checksum:
+                            type: string
+                          lastUpdate:
+                            format: date-time
+                            type: string
+                          secretName:
+                            type: string
+                        type: object
+                      sa:
+                        properties:
+                          lastUpdate:
+                            description: Last time when k8s object was updated
+                            format: date-time
+                            type: string
+                          name:
+                            type: string
+                          namespace:
+                            type: string
+                          resourceVersion:
+                            description: Resource version of k8s object
+                            type: string
+                        type: object
+                      service:
+                        description: KubernetesServiceStatus defines the status for
+                          the Tenant Control Plane Service in the management cluster.
+                        properties:
+                          conditions:
+                            description: Current service state
+                            items:
+                              description: "Condition contains details for one aspect
+                                of the current state of this API Resource. --- This
+                                struct is intended for direct use as an array at the
+                                field path .status.conditions.  For example, type
+                                FooStatus struct{     // Represents the observations
+                                of a foo's current state.     // Known .status.conditions.type
+                                are: \"Available\", \"Progressing\", and \"Degraded\"
+                                \    // +patchMergeKey=type     // +patchStrategy=merge
+                                \    // +listType=map     // +listMapKey=type     Conditions
+                                []metav1.Condition `json:\"conditions,omitempty\"
+                                patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"`
+                                \n     // other fields }"
+                              properties:
+                                lastTransitionTime:
+                                  description: lastTransitionTime is the last time
+                                    the condition transitioned from one status to
+                                    another. This should be when the underlying condition
+                                    changed.  If that is not known, then using the
+                                    time when the API field changed is acceptable.
+                                  format: date-time
+                                  type: string
+                                message:
+                                  description: message is a human readable message
+                                    indicating details about the transition. This
+                                    may be an empty string.
+                                  maxLength: 32768
+                                  type: string
+                                observedGeneration:
+                                  description: observedGeneration represents the .metadata.generation
+                                    that the condition was set based upon. For instance,
+                                    if .metadata.generation is currently 12, but the
+                                    .status.conditions[x].observedGeneration is 9,
+                                    the condition is out of date with respect to the
+                                    current state of the instance.
+                                  format: int64
+                                  minimum: 0
+                                  type: integer
+                                reason:
+                                  description: reason contains a programmatic identifier
+                                    indicating the reason for the condition's last
+                                    transition. Producers of specific condition types
+                                    may define expected values and meanings for this
+                                    field, and whether the values are considered a
+                                    guaranteed API. The value should be a CamelCase
+                                    string. This field may not be empty.
+                                  maxLength: 1024
+                                  minLength: 1
+                                  pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
+                                  type: string
+                                status:
+                                  description: status of the condition, one of True,
+                                    False, Unknown.
+                                  enum:
+                                  - "True"
+                                  - "False"
+                                  - Unknown
+                                  type: string
+                                type:
+                                  description: type of condition in CamelCase or in
+                                    foo.example.com/CamelCase. --- Many .condition.type
+                                    values are consistent across resources like Available,
+                                    but because arbitrary conditions can be useful
+                                    (see .node.status.conditions), the ability to
+                                    deconflict is important. The regex it matches
+                                    is (dns1123SubdomainFmt/)?(qualifiedNameFmt)
+                                  maxLength: 316
+                                  pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
+                                  type: string
+                              required:
+                              - lastTransitionTime
+                              - message
+                              - reason
+                              - status
+                              - type
+                              type: object
+                            type: array
+                            x-kubernetes-list-map-keys:
+                            - type
+                            x-kubernetes-list-type: map
+                          loadBalancer:
+                            description: LoadBalancer contains the current status
+                              of the load-balancer, if one is present.
+                            properties:
+                              ingress:
+                                description: Ingress is a list containing ingress
+                                  points for the load-balancer. Traffic intended for
+                                  the service should be sent to these ingress points.
+                                items:
+                                  description: 'LoadBalancerIngress represents the
+                                    status of a load-balancer ingress point: traffic
+                                    intended for the service should be sent to an
+                                    ingress point.'
+                                  properties:
+                                    hostname:
+                                      description: Hostname is set for load-balancer
+                                        ingress points that are DNS based (typically
+                                        AWS load-balancers)
+                                      type: string
+                                    ip:
+                                      description: IP is set for load-balancer ingress
+                                        points that are IP based (typically GCE or
+                                        OpenStack load-balancers)
+                                      type: string
+                                    ports:
+                                      description: Ports is a list of records of service
+                                        ports If used, every port defined in the service
+                                        should have an entry in it
+                                      items:
+                                        properties:
+                                          error:
+                                            description: 'Error is to record the problem
+                                              with the service port The format of
+                                              the error shall comply with the following
+                                              rules: - built-in error values shall
+                                              be specified in this file and those
+                                              shall use   CamelCase names - cloud
+                                              provider specific error values must
+                                              have names that comply with the   format
+                                              foo.example.com/CamelCase. --- The regex
+                                              it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)'
+                                            maxLength: 316
+                                            pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
+                                            type: string
+                                          port:
+                                            description: Port is the port number of
+                                              the service port of which status is
+                                              recorded here
+                                            format: int32
+                                            type: integer
+                                          protocol:
+                                            default: TCP
+                                            description: 'Protocol is the protocol
+                                              of the service port of which status
+                                              is recorded here The supported values
+                                              are: "TCP", "UDP", "SCTP"'
+                                            type: string
+                                        required:
+                                        - port
+                                        - protocol
+                                        type: object
+                                      type: array
+                                      x-kubernetes-list-type: atomic
+                                  type: object
+                                type: array
+                            type: object
+                          name:
+                            description: The name of the Service for the given cluster.
+                            type: string
+                          namespace:
+                            description: The namespace which the Service for the given
+                              cluster is deployed.
+                            type: string
+                          port:
+                            description: The port where the service is running
+                            format: int32
+                            type: integer
+                        required:
+                        - name
+                        - namespace
+                        - port
+                        type: object
+                    required:
+                    - enabled
+                    type: object
                   kubeProxy:
                     description: AddonStatus defines the observed state of an Addon.
                     properties:
+                      checksum:
+                        type: string
                       enabled:
                         type: boolean
-                      kubeadmConfigResourceVersion:
-                        type: string
                       lastUpdate:
                         format: date-time
                         type: string
@@ -340,39 +736,45 @@ spec:
                   certificates that are necessary to run a kubernetes control plane
                 properties:
                   apiServer:
-                    description: CertificatePrivateKeyPair defines the status.
+                    description: CertificatePrivateKeyPairStatus defines the status.
                     properties:
                       lastUpdate:
                         format: date-time
                         type: string
+                      resourceVersion:
+                        type: string
                       secretName:
                         type: string
                     type: object
                   apiServerKubeletClient:
-                    description: CertificatePrivateKeyPair defines the status.
+                    description: CertificatePrivateKeyPairStatus defines the status.
                     properties:
                       lastUpdate:
                         format: date-time
                         type: string
+                      resourceVersion:
+                        type: string
                       secretName:
                         type: string
                     type: object
                   ca:
-                    description: CertificatePrivateKeyPair defines the status.
+                    description: CertificatePrivateKeyPairStatus defines the status.
                     properties:
                       lastUpdate:
                         format: date-time
                         type: string
+                      resourceVersion:
+                        type: string
                       secretName:
                         type: string
                     type: object
                   etcd:
-                    description: ETCDAPIServerCertificate defines the observed state
+                    description: ETCDCertificatesStatus defines the observed state
                       of ETCD Certificate for API server.
                     properties:
                       apiServer:
-                        description: ETCDAPIServerCertificate defines the observed
-                          state of ETCD Certificate for API server.
+                        description: ETCDCertificateStatus defines the observed state
+                          of ETCD Certificate for API server.
                         properties:
                           lastUpdate:
                             format: date-time
@@ -381,8 +783,8 @@ spec:
                             type: string
                         type: object
                       ca:
-                        description: ETCDAPIServerCertificate defines the observed
-                          state of ETCD Certificate for API server.
+                        description: ETCDCertificateStatus defines the observed state
+                          of ETCD Certificate for API server.
                         properties:
                           lastUpdate:
                             format: date-time
@@ -392,25 +794,29 @@ spec:
                         type: object
                     type: object
                   frontProxyCA:
-                    description: CertificatePrivateKeyPair defines the status.
+                    description: CertificatePrivateKeyPairStatus defines the status.
                     properties:
                       lastUpdate:
                         format: date-time
                         type: string
+                      resourceVersion:
+                        type: string
                       secretName:
                         type: string
                     type: object
                   frontProxyClient:
-                    description: CertificatePrivateKeyPair defines the status.
+                    description: CertificatePrivateKeyPairStatus defines the status.
                     properties:
                       lastUpdate:
                         format: date-time
                         type: string
+                      resourceVersion:
+                        type: string
                       secretName:
                         type: string
                     type: object
                   sa:
-                    description: CertificatePrivateKeyPair defines the status.
+                    description: PublicKeyPrivateKeyPairStatus defines the status.
                     properties:
                       lastUpdate:
                         format: date-time
@@ -428,30 +834,30 @@ spec:
                   action
                 properties:
                   bootstrapToken:
-                    description: KubeadmPhasesStatus contains the status of of a kubeadm
+                    description: KubeadmPhaseStatus contains the status of a kubeadm
                       phase action.
                     properties:
-                      kubeadmConfigResourceVersion:
+                      checksum:
                         type: string
                       lastUpdate:
                         format: date-time
                         type: string
                     type: object
                   uploadConfigKubeadm:
-                    description: KubeadmPhasesStatus contains the status of of a kubeadm
+                    description: KubeadmPhaseStatus contains the status of a kubeadm
                       phase action.
                     properties:
-                      kubeadmConfigResourceVersion:
+                      checksum:
                         type: string
                       lastUpdate:
                         format: date-time
                         type: string
                     type: object
                   uploadConfigKubelet:
-                    description: KubeadmPhasesStatus contains the status of of a kubeadm
+                    description: KubeadmPhaseStatus contains the status of a kubeadm
                       phase action.
                     properties:
-                      kubeadmConfigResourceVersion:
+                      checksum:
                         type: string
                       lastUpdate:
                         format: date-time
@@ -466,34 +872,37 @@ spec:
                 description: KubeadmConfig contains the status of the configuration
                   required by kubeadm
                 properties:
+                  checksum:
+                    description: Checksum of the kubeadm configuration to detect changes
+                    type: string
                   configmapName:
                     type: string
                   lastUpdate:
                     format: date-time
                     type: string
-                  resourceVersion:
-                    type: string
-                required:
-                - resourceVersion
                 type: object
               kubeconfig:
                 description: KubeConfig contains information about the kubenconfigs
                   that control plane pieces need
                 properties:
                   admin:
-                    description: TenantControlPlaneKubeconfigsStatus contains information
-                      about a the generated kubeconfig.
+                    description: KubeconfigStatus contains information about the generated
+                      kubeconfig.
                     properties:
+                      checksum:
+                        type: string
                       lastUpdate:
                         format: date-time
                         type: string
                       secretName:
                         type: string
                     type: object
-                  controlerManager:
-                    description: TenantControlPlaneKubeconfigsStatus contains information
-                      about a the generated kubeconfig.
+                  controllerManager:
+                    description: KubeconfigStatus contains information about the generated
+                      kubeconfig.
                     properties:
+                      checksum:
+                        type: string
                       lastUpdate:
                         format: date-time
                         type: string
@@ -501,9 +910,11 @@ spec:
                         type: string
                     type: object
                   scheduler:
-                    description: TenantControlPlaneKubeconfigsStatus contains information
-                      about a the generated kubeconfig.
+                    description: KubeconfigStatus contains information about the generated
+                      kubeconfig.
                     properties:
+                      checksum:
+                        type: string
                       lastUpdate:
                         format: date-time
                         type: string
@@ -566,6 +977,10 @@ spec:
                           - type
                           type: object
                         type: array
+                      lastUpdate:
+                        description: Last time when deployment was updated
+                        format: date-time
+                        type: string
                       name:
                         description: The name of the Deployment for the given cluster.
                         type: string
@@ -909,6 +1324,38 @@ spec:
                         - name
                         type: object
                     type: object
+                  kineMySQL:
+                    properties:
+                      certificate:
+                        properties:
+                          lastUpdate:
+                            format: date-time
+                            type: string
+                          resourceVersion:
+                            type: string
+                          secretName:
+                            type: string
+                        type: object
+                      config:
+                        properties:
+                          resourceVersion:
+                            type: string
+                          secretName:
+                            type: string
+                        type: object
+                      setup:
+                        properties:
+                          lastUpdate:
+                            format: date-time
+                            type: string
+                          schema:
+                            type: string
+                          sqlConfigResourceVersion:
+                            type: string
+                          user:
+                            type: string
+                        type: object
+                    type: object
                 type: object
             type: object
         type: object

config/install.yaml

@@ -61,30 +61,57 @@ spec:
             description: TenantControlPlaneSpec defines the desired state of TenantControlPlane.
             properties:
               addons:
-                default:
-                  coreDNS:
-                    enabled: true
-                  kubeProxy:
-                    enabled: true
                 description: Addons contain which addons are enabled
                 properties:
                   coreDNS:
-                    default:
-                      enabled: true
                     description: AddonSpec defines the spec for every addon.
+                    type: object
+                  konnectivity:
+                    description: KonnectivitySpec defines the spec for Konnectivity.
                     properties:
-                      enabled:
-                        default: true
-                        type: boolean
+                      agentImage:
+                        default: us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-agent
+                        description: AgentImage defines the container image for Konnectivity's agent.
+                        type: string
+                      proxyPort:
+                        description: Port of Konnectivity proxy server.
+                        format: int32
+                        type: integer
+                      resources:
+                        description: Resources define the amount of CPU and memory to allocate to the Konnectivity server.
+                        properties:
+                          limits:
+                            additionalProperties:
+                              anyOf:
+                              - type: integer
+                              - type: string
+                              pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                              x-kubernetes-int-or-string: true
+                            description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                            type: object
+                          requests:
+                            additionalProperties:
+                              anyOf:
+                              - type: integer
+                              - type: string
+                              pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                              x-kubernetes-int-or-string: true
+                            description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                            type: object
+                        type: object
+                      serverImage:
+                        default: us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-server
+                        description: ServerImage defines the container image for Konnectivity's server.
+                        type: string
+                      version:
+                        default: v0.0.31
+                        description: Version for Konnectivity server and agent.
+                        type: string
+                    required:
+                    - proxyPort
                     type: object
                   kubeProxy:
-                    default:
-                      enabled: true
                     description: AddonSpec defines the spec for every addon.
-                    properties:
-                      enabled:
-                        default: true
-                        type: boolean
                     type: object
                 type: object
               controlPlane:
@@ -105,10 +132,101 @@ spec:
                               type: string
                             type: object
                         type: object
+                      extraArgs:
+                        description: ExtraArgs allows adding additional arguments to the Control Plane components, such as kube-apiserver, controller-manager, and scheduler.
+                        properties:
+                          apiServer:
+                            items:
+                              type: string
+                            type: array
+                          controllerManager:
+                            items:
+                              type: string
+                            type: array
+                          kine:
+                            description: Available only if Kamaji is running using Kine as backing storage.
+                            items:
+                              type: string
+                            type: array
+                          scheduler:
+                            items:
+                              type: string
+                            type: array
+                        type: object
                       replicas:
                         default: 2
                         format: int32
                         type: integer
+                      resources:
+                        description: Resources defines the amount of memory and CPU to allocate to each component of the Control Plane (kube-apiserver, controller-manager, and scheduler).
+                        properties:
+                          apiServer:
+                            description: ResourceRequirements describes the compute resource requirements.
+                            properties:
+                              limits:
+                                additionalProperties:
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
+                                description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                                type: object
+                              requests:
+                                additionalProperties:
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
+                                description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                                type: object
+                            type: object
+                          controllerManager:
+                            description: ResourceRequirements describes the compute resource requirements.
+                            properties:
+                              limits:
+                                additionalProperties:
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
+                                description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                                type: object
+                              requests:
+                                additionalProperties:
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
+                                description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                                type: object
+                            type: object
+                          scheduler:
+                            description: ResourceRequirements describes the compute resource requirements.
+                            properties:
+                              limits:
+                                additionalProperties:
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
+                                description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                                type: object
+                              requests:
+                                additionalProperties:
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
+                                description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                                type: object
+                            type: object
+                        type: object
                     type: object
                   ingress:
                     description: Defining the options for an Optional Ingress which will expose API Server of the Tenant Control Plane
@@ -252,14 +370,19 @@ spec:
                   allowAddressAsExternalIP:
                     description: AllowAddressAsExternalIP will include tenantControlPlane.Spec.NetworkProfile.Address in the section of ExternalIPs of the Kubernetes Service (only ClusterIP or NodePort)
                     type: boolean
-                  dnsServiceIPs:
+                  certSANs:
+                    description: CertSANs sets extra Subject Alternative Names (SANs) for the API Server signing certificate. Use this field to add additional hostnames when exposing the Tenant Control Plane with third solutions.
+                    items:
+                      type: string
+                    type: array
+                  dnsServiceIPs:
+                    default:
+                    - 10.96.0.10
                     items:
                       type: string
                     type: array
-                  domain:
-                    description: Domain of the tenant control plane
-                    type: string
                   podCidr:
+                    default: 10.244.0.0/16
                     description: CIDR for Kubernetes Pods
                     type: string
                   port:
@@ -268,14 +391,9 @@ spec:
                     format: int32
                     type: integer
                   serviceCidr:
+                    default: 10.96.0.0/16
                     description: Kubernetes Service
                     type: string
-                required:
-                - dnsServiceIPs
-                - domain
-                - podCidr
-                - port
-                - serviceCidr
                 type: object
             required:
             - controlPlane
@@ -290,23 +408,202 @@ spec:
                   coreDNS:
                     description: AddonStatus defines the observed state of an Addon.
                     properties:
+                      checksum:
+                        type: string
                       enabled:
                         type: boolean
-                      kubeadmConfigResourceVersion:
-                        type: string
                       lastUpdate:
                         format: date-time
                         type: string
                     required:
                     - enabled
                     type: object
+                  konnectivity:
+                    description: KonnectivityStatus defines the status of Konnectivity as Addon.
+                    properties:
+                      agent:
+                        properties:
+                          lastUpdate:
+                            description: Last time when k8s object was updated
+                            format: date-time
+                            type: string
+                          name:
+                            type: string
+                          namespace:
+                            type: string
+                          resourceVersion:
+                            description: Resource version of k8s object
+                            type: string
+                        type: object
+                      certificate:
+                        description: CertificatePrivateKeyPairStatus defines the status.
+                        properties:
+                          lastUpdate:
+                            format: date-time
+                            type: string
+                          resourceVersion:
+                            type: string
+                          secretName:
+                            type: string
+                        type: object
+                      clusterrolebinding:
+                        properties:
+                          lastUpdate:
+                            description: Last time when k8s object was updated
+                            format: date-time
+                            type: string
+                          name:
+                            type: string
+                          namespace:
+                            type: string
+                          resourceVersion:
+                            description: Resource version of k8s object
+                            type: string
+                        type: object
+                      egressSelectorConfiguration:
+                        type: string
+                      enabled:
+                        type: boolean
+                      kubeconfig:
+                        description: KubeconfigStatus contains information about the generated kubeconfig.
+                        properties:
+                          checksum:
+                            type: string
+                          lastUpdate:
+                            format: date-time
+                            type: string
+                          secretName:
+                            type: string
+                        type: object
+                      sa:
+                        properties:
+                          lastUpdate:
+                            description: Last time when k8s object was updated
+                            format: date-time
+                            type: string
+                          name:
+                            type: string
+                          namespace:
+                            type: string
+                          resourceVersion:
+                            description: Resource version of k8s object
+                            type: string
+                        type: object
+                      service:
+                        description: KubernetesServiceStatus defines the status for the Tenant Control Plane Service in the management cluster.
+                        properties:
+                          conditions:
+                            description: Current service state
+                            items:
+                              description: "Condition contains details for one aspect of the current state of this API Resource. --- This struct is intended for direct use as an array at the field path .status.conditions.  For example, type FooStatus struct{     // Represents the observations of a foo's current state.     // Known .status.conditions.type are: \"Available\", \"Progressing\", and \"Degraded\"     // +patchMergeKey=type     // +patchStrategy=merge     // +listType=map     // +listMapKey=type     Conditions []metav1.Condition `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"` \n     // other fields }"
+                              properties:
+                                lastTransitionTime:
+                                  description: lastTransitionTime is the last time the condition transitioned from one status to another. This should be when the underlying condition changed.  If that is not known, then using the time when the API field changed is acceptable.
+                                  format: date-time
+                                  type: string
+                                message:
+                                  description: message is a human readable message indicating details about the transition. This may be an empty string.
+                                  maxLength: 32768
+                                  type: string
+                                observedGeneration:
+                                  description: observedGeneration represents the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date with respect to the current state of the instance.
+                                  format: int64
+                                  minimum: 0
+                                  type: integer
+                                reason:
+                                  description: reason contains a programmatic identifier indicating the reason for the condition's last transition. Producers of specific condition types may define expected values and meanings for this field, and whether the values are considered a guaranteed API. The value should be a CamelCase string. This field may not be empty.
+                                  maxLength: 1024
+                                  minLength: 1
+                                  pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
+                                  type: string
+                                status:
+                                  description: status of the condition, one of True, False, Unknown.
+                                  enum:
+                                  - "True"
+                                  - "False"
+                                  - Unknown
+                                  type: string
+                                type:
+                                  description: type of condition in CamelCase or in foo.example.com/CamelCase. --- Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be useful (see .node.status.conditions), the ability to deconflict is important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)
+                                  maxLength: 316
+                                  pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
+                                  type: string
+                              required:
+                              - lastTransitionTime
+                              - message
+                              - reason
+                              - status
+                              - type
+                              type: object
+                            type: array
+                            x-kubernetes-list-map-keys:
+                            - type
+                            x-kubernetes-list-type: map
+                          loadBalancer:
+                            description: LoadBalancer contains the current status of the load-balancer, if one is present.
+                            properties:
+                              ingress:
+                                description: Ingress is a list containing ingress points for the load-balancer. Traffic intended for the service should be sent to these ingress points.
+                                items:
+                                  description: 'LoadBalancerIngress represents the status of a load-balancer ingress point: traffic intended for the service should be sent to an ingress point.'
+                                  properties:
+                                    hostname:
+                                      description: Hostname is set for load-balancer ingress points that are DNS based (typically AWS load-balancers)
+                                      type: string
+                                    ip:
+                                      description: IP is set for load-balancer ingress points that are IP based (typically GCE or OpenStack load-balancers)
+                                      type: string
+                                    ports:
+                                      description: Ports is a list of records of service ports If used, every port defined in the service should have an entry in it
+                                      items:
+                                        properties:
+                                          error:
+                                            description: 'Error is to record the problem with the service port The format of the error shall comply with the following rules: - built-in error values shall be specified in this file and those shall use   CamelCase names - cloud provider specific error values must have names that comply with the   format foo.example.com/CamelCase. --- The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)'
+                                            maxLength: 316
+                                            pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
+                                            type: string
+                                          port:
+                                            description: Port is the port number of the service port of which status is recorded here
+                                            format: int32
+                                            type: integer
+                                          protocol:
+                                            default: TCP
+                                            description: 'Protocol is the protocol of the service port of which status is recorded here The supported values are: "TCP", "UDP", "SCTP"'
+                                            type: string
+                                        required:
+                                        - port
+                                        - protocol
+                                        type: object
+                                      type: array
+                                      x-kubernetes-list-type: atomic
+                                  type: object
+                                type: array
+                            type: object
+                          name:
+                            description: The name of the Service for the given cluster.
+                            type: string
+                          namespace:
+                            description: The namespace which the Service for the given cluster is deployed.
+                            type: string
+                          port:
+                            description: The port where the service is running
+                            format: int32
+                            type: integer
+                        required:
+                        - name
+                        - namespace
+                        - port
+                        type: object
+                    required:
+                    - enabled
+                    type: object
                   kubeProxy:
                     description: AddonStatus defines the observed state of an Addon.
                     properties:
+                      checksum:
+                        type: string
                       enabled:
                         type: boolean
-                      kubeadmConfigResourceVersion:
-                        type: string
                       lastUpdate:
                         format: date-time
                         type: string
@@ -318,37 +615,43 @@ spec:
                 description: Certificates contains information about the different certificates that are necessary to run a kubernetes control plane
                 properties:
                   apiServer:
-                    description: CertificatePrivateKeyPair defines the status.
+                    description: CertificatePrivateKeyPairStatus defines the status.
                     properties:
                       lastUpdate:
                         format: date-time
                         type: string
+                      resourceVersion:
+                        type: string
                       secretName:
                         type: string
                     type: object
                   apiServerKubeletClient:
-                    description: CertificatePrivateKeyPair defines the status.
+                    description: CertificatePrivateKeyPairStatus defines the status.
                     properties:
                       lastUpdate:
                         format: date-time
                         type: string
+                      resourceVersion:
+                        type: string
                       secretName:
                         type: string
                     type: object
                   ca:
-                    description: CertificatePrivateKeyPair defines the status.
+                    description: CertificatePrivateKeyPairStatus defines the status.
                     properties:
                       lastUpdate:
                         format: date-time
                         type: string
+                      resourceVersion:
+                        type: string
                       secretName:
                         type: string
                     type: object
                   etcd:
-                    description: ETCDAPIServerCertificate defines the observed state of ETCD Certificate for API server.
+                    description: ETCDCertificatesStatus defines the observed state of ETCD Certificate for API server.
                     properties:
                       apiServer:
-                        description: ETCDAPIServerCertificate defines the observed state of ETCD Certificate for API server.
+                        description: ETCDCertificateStatus defines the observed state of ETCD Certificate for API server.
                         properties:
                           lastUpdate:
                             format: date-time
@@ -357,7 +660,7 @@ spec:
                             type: string
                         type: object
                       ca:
-                        description: ETCDAPIServerCertificate defines the observed state of ETCD Certificate for API server.
+                        description: ETCDCertificateStatus defines the observed state of ETCD Certificate for API server.
                         properties:
                           lastUpdate:
                             format: date-time
@@ -367,25 +670,29 @@ spec:
                         type: object
                     type: object
                   frontProxyCA:
-                    description: CertificatePrivateKeyPair defines the status.
+                    description: CertificatePrivateKeyPairStatus defines the status.
                     properties:
                       lastUpdate:
                         format: date-time
                         type: string
+                      resourceVersion:
+                        type: string
                       secretName:
                         type: string
                     type: object
                   frontProxyClient:
-                    description: CertificatePrivateKeyPair defines the status.
+                    description: CertificatePrivateKeyPairStatus defines the status.
                     properties:
                       lastUpdate:
                         format: date-time
                         type: string
+                      resourceVersion:
+                        type: string
                       secretName:
                         type: string
                     type: object
                   sa:
-                    description: CertificatePrivateKeyPair defines the status.
+                    description: PublicKeyPrivateKeyPairStatus defines the status.
                     properties:
                       lastUpdate:
                         format: date-time
@@ -401,27 +708,27 @@ spec:
                 description: KubeadmPhase contains the status of the kubeadm phases action
                 properties:
                   bootstrapToken:
-                    description: KubeadmPhasesStatus contains the status of of a kubeadm phase action.
+                    description: KubeadmPhaseStatus contains the status of a kubeadm phase action.
                     properties:
-                      kubeadmConfigResourceVersion:
+                      checksum:
                         type: string
                       lastUpdate:
                         format: date-time
                         type: string
                     type: object
                   uploadConfigKubeadm:
-                    description: KubeadmPhasesStatus contains the status of of a kubeadm phase action.
+                    description: KubeadmPhaseStatus contains the status of a kubeadm phase action.
                     properties:
-                      kubeadmConfigResourceVersion:
+                      checksum:
                         type: string
                       lastUpdate:
                         format: date-time
                         type: string
                     type: object
                   uploadConfigKubelet:
-                    description: KubeadmPhasesStatus contains the status of of a kubeadm phase action.
+                    description: KubeadmPhaseStatus contains the status of a kubeadm phase action.
                     properties:
-                      kubeadmConfigResourceVersion:
+                      checksum:
                         type: string
                       lastUpdate:
                         format: date-time
@@ -435,31 +742,34 @@ spec:
               kubeadmconfig:
                 description: KubeadmConfig contains the status of the configuration required by kubeadm
                 properties:
+                  checksum:
+                    description: Checksum of the kubeadm configuration to detect changes
+                    type: string
                   configmapName:
                     type: string
                   lastUpdate:
                     format: date-time
                     type: string
-                  resourceVersion:
-                    type: string
-                required:
-                - resourceVersion
                 type: object
               kubeconfig:
                 description: KubeConfig contains information about the kubenconfigs that control plane pieces need
                 properties:
                   admin:
-                    description: TenantControlPlaneKubeconfigsStatus contains information about a the generated kubeconfig.
+                    description: KubeconfigStatus contains information about the generated kubeconfig.
                     properties:
+                      checksum:
+                        type: string
                       lastUpdate:
                         format: date-time
                         type: string
                       secretName:
                         type: string
                     type: object
-                  controlerManager:
-                    description: TenantControlPlaneKubeconfigsStatus contains information about a the generated kubeconfig.
+                  controllerManager:
+                    description: KubeconfigStatus contains information about the generated kubeconfig.
                     properties:
+                      checksum:
+                        type: string
                       lastUpdate:
                         format: date-time
                         type: string
@@ -467,8 +777,10 @@ spec:
                         type: string
                     type: object
                   scheduler:
-                    description: TenantControlPlaneKubeconfigsStatus contains information about a the generated kubeconfig.
+                    description: KubeconfigStatus contains information about the generated kubeconfig.
                     properties:
+                      checksum:
+                        type: string
                       lastUpdate:
                         format: date-time
                         type: string
@@ -520,6 +832,10 @@ spec:
                           - type
                           type: object
                         type: array
+                      lastUpdate:
+                        description: Last time when deployment was updated
+                        format: date-time
+                        type: string
                       name:
                         description: The name of the Deployment for the given cluster.
                         type: string
@@ -769,6 +1085,38 @@ spec:
                         - name
                         type: object
                     type: object
+                  kineMySQL:
+                    properties:
+                      certificate:
+                        properties:
+                          lastUpdate:
+                            format: date-time
+                            type: string
+                          resourceVersion:
+                            type: string
+                          secretName:
+                            type: string
+                        type: object
+                      config:
+                        properties:
+                          resourceVersion:
+                            type: string
+                          secretName:
+                            type: string
+                        type: object
+                      setup:
+                        properties:
+                          lastUpdate:
+                            format: date-time
+                            type: string
+                          schema:
+                            type: string
+                          sqlConfigResourceVersion:
+                            type: string
+                          user:
+                            type: string
+                        type: object
+                    type: object
                 type: object
             type: object
         type: object
@@ -1059,7 +1407,7 @@ spec:
         command:
         - /manager
         image: clastix/kamaji:latest
-        imagePullPolicy: IfNotPresent
+        imagePullPolicy: Always
         livenessProbe:
           httpGet:
             path: /healthz

config/manager/manager.yaml

@@ -30,7 +30,7 @@ spec:
         args:
         - --leader-elect
         image: controller:latest
-        imagePullPolicy: IfNotPresent
+        imagePullPolicy: Always
         name: manager
         securityContext:
           allowPrivilegeEscalation: false

config/samples/kamaji_v1alpha1_tenantcontrolplane.yaml

@@ -30,7 +30,7 @@ spec:
         annotations:
           kubernetes.io/ingress.allow-http: "false"
           nginx.ingress.kubernetes.io/secure-backends: "true"
-          nginx.ingress.kubernetes.io/ssl-passthrough: "true"      
+          nginx.ingress.kubernetes.io/ssl-passthrough: "true"
   kubernetes:
     version: "v1.23.1"
     kubelet:
@@ -41,13 +41,12 @@ spec:
   networkProfile:
     address: "127.0.0.1"
     port: 6443
-    domain: "clastix.labs"
+    certSANs:
+      - "test.clastix.labs"
     serviceCidr: "10.96.0.0/16"
     podCidr: "10.244.0.0/16"
     dnsServiceIPs:
       - "10.96.0.10"
   addons:
-    coreDNS:
-      enabled: true
-    kubeProxy:
-      enabled: true
+    coreDNS: {}
+    kubeProxy: {}

controllers/resources.go

@@ -0,0 +1,360 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+package controllers
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/go-logr/logr"
+	"github.com/google/uuid"
+	k8stypes "k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+	"github.com/clastix/kamaji/internal/resources"
+	"github.com/clastix/kamaji/internal/resources/konnectivity"
+	"github.com/clastix/kamaji/internal/sql"
+	"github.com/clastix/kamaji/internal/types"
+)
+
+const (
+	separator = ","
+)
+
+type GroupResourceBuilderConfiguration struct {
+	client              client.Client
+	log                 logr.Logger
+	tcpReconcilerConfig TenantControlPlaneReconcilerConfig
+	tenantControlPlane  kamajiv1alpha1.TenantControlPlane
+	DBConnection        sql.DBConnection
+}
+
+type GroupDeleteableResourceBuilderConfiguration struct {
+	client              client.Client
+	log                 logr.Logger
+	tcpReconcilerConfig TenantControlPlaneReconcilerConfig
+	tenantControlPlane  kamajiv1alpha1.TenantControlPlane
+	DBConnection        sql.DBConnection
+}
+
+// GetResources returns a list of resources that will be used to provide tenant control planes
+// Currently there is only a default approach
+// TODO: the idea of this function is to become a factory to return the group of resources according to the given configuration.
+func GetResources(config GroupResourceBuilderConfiguration) []resources.Resource {
+	return getDefaultResources(config)
+}
+
+// GetDeletableResources returns a list of resources that have to be deleted when tenant control planes are deleted
+// Currently there is only a default approach
+// TODO: the idea of this function is to become a factory to return the group of deleteable resources according to the given configuration.
+func GetDeletableResources(config GroupDeleteableResourceBuilderConfiguration) []resources.DeleteableResource {
+	return getDefaultDeleteableResources(config)
+}
+
+func getDefaultResources(config GroupResourceBuilderConfiguration) []resources.Resource {
+	resources := append(getUpgradeResources(config.client, config.tenantControlPlane), getKubernetesServiceResources(config.client, config.tenantControlPlane)...)
+	resources = append(resources, getKubeadmConfigResources(config.client, config.tcpReconcilerConfig, config.tenantControlPlane)...)
+	resources = append(resources, getKubernetesCertificatesResources(config.client, config.log, config.tcpReconcilerConfig, config.tenantControlPlane)...)
+	resources = append(resources, getKubeconfigResources(config.client, config.log, config.tcpReconcilerConfig, config.tenantControlPlane)...)
+	resources = append(resources, getKubernetesStorageResources(config.client, config.log, config.tcpReconcilerConfig, config.DBConnection, config.tenantControlPlane)...)
+	resources = append(resources, getInternalKonnectivityResources(config.client, config.log, config.tcpReconcilerConfig, config.tenantControlPlane)...)
+	resources = append(resources, getKubernetesDeploymentResources(config.client, config.tcpReconcilerConfig, config.tenantControlPlane)...)
+	resources = append(resources, getKubernetesIngressResources(config.client, config.tenantControlPlane)...)
+	resources = append(resources, getKubeadmPhaseResources(config.client, config.log, config.tenantControlPlane)...)
+	resources = append(resources, getKubeadmAddonResources(config.client, config.log, config.tenantControlPlane)...)
+	resources = append(resources, getExternalKonnectivityResources(config.client, config.log, config.tcpReconcilerConfig, config.tenantControlPlane)...)
+
+	return resources
+}
+
+func getDefaultDeleteableResources(config GroupDeleteableResourceBuilderConfiguration) []resources.DeleteableResource {
+	switch config.tcpReconcilerConfig.ETCDStorageType {
+	case types.ETCD:
+		return []resources.DeleteableResource{
+			&resources.ETCDSetupResource{
+				Name:                  "etcd-setup",
+				Client:                config.client,
+				Log:                   config.log,
+				ETCDClientCertsSecret: getNamespacedName(config.tcpReconcilerConfig.ETCDClientSecretNamespace, config.tcpReconcilerConfig.ETCDClientSecretName),
+				ETCDCACertsSecret:     getNamespacedName(config.tcpReconcilerConfig.ETCDCASecretNamespace, config.tcpReconcilerConfig.ETCDCASecretName),
+				Endpoints:             getArrayFromString(config.tcpReconcilerConfig.ETCDEndpoints),
+			},
+		}
+	case types.KineMySQL:
+		return []resources.DeleteableResource{
+			&resources.SQLSetup{
+				Client:       config.client,
+				Name:         "sql-setup",
+				DBConnection: config.DBConnection,
+			},
+		}
+	default:
+		return []resources.DeleteableResource{}
+	}
+}
+
+func getUpgradeResources(c client.Client, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource {
+	return []resources.Resource{
+		&resources.KubernetesUpgrade{
+			Name:   "upgrade",
+			Client: c,
+		},
+	}
+}
+
+func getKubernetesServiceResources(c client.Client, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource {
+	return []resources.Resource{
+		&resources.KubernetesServiceResource{
+			Client: c,
+		},
+	}
+}
+
+func getKubeadmConfigResources(c client.Client, tcpReconcilerConfig TenantControlPlaneReconcilerConfig, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource {
+	return []resources.Resource{
+		&resources.KubeadmConfigResource{
+			Name:                   "kubeadmconfig",
+			ETCDs:                  getArrayFromString(tcpReconcilerConfig.ETCDEndpoints),
+			ETCDCompactionInterval: tcpReconcilerConfig.ETCDCompactionInterval,
+			Client:                 c,
+			TmpDirectory:           getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
+		},
+	}
+}
+
+func getKubernetesCertificatesResources(c client.Client, log logr.Logger, tcpReconcilerConfig TenantControlPlaneReconcilerConfig, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource {
+	return []resources.Resource{
+		&resources.CACertificate{
+			Name:         "ca",
+			Client:       c,
+			Log:          log,
+			TmpDirectory: getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
+		},
+		&resources.FrontProxyCACertificate{
+			Name:         "front-proxy-ca-certificate",
+			Client:       c,
+			Log:          log,
+			TmpDirectory: getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
+		},
+		&resources.SACertificate{
+			Name:         "sa-certificate",
+			Client:       c,
+			Log:          log,
+			TmpDirectory: getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
+		},
+		&resources.APIServerCertificate{
+			Name:         "api-server-certificate",
+			Client:       c,
+			Log:          log,
+			TmpDirectory: getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
+		},
+		&resources.APIServerKubeletClientCertificate{
+			Name:         "api-server-kubelet-client-certificate",
+			Client:       c,
+			Log:          log,
+			TmpDirectory: getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
+		},
+		&resources.FrontProxyClientCertificate{
+			Name:         "front-proxy-client-certificate",
+			Client:       c,
+			Log:          log,
+			TmpDirectory: getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
+		},
+	}
+}
+
+func getKubeconfigResources(c client.Client, log logr.Logger, tcpReconcilerConfig TenantControlPlaneReconcilerConfig, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource {
+	return []resources.Resource{
+		&resources.KubeconfigResource{
+			Name:               "admin-kubeconfig",
+			Client:             c,
+			Log:                log,
+			KubeConfigFileName: resources.AdminKubeConfigFileName,
+			TmpDirectory:       getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
+		},
+		&resources.KubeconfigResource{
+			Name:               "controller-manager-kubeconfig",
+			Client:             c,
+			Log:                log,
+			KubeConfigFileName: resources.ControllerManagerKubeConfigFileName,
+			TmpDirectory:       getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
+		},
+		&resources.KubeconfigResource{
+			Name:               "scheduler-kubeconfig",
+			Client:             c,
+			Log:                log,
+			KubeConfigFileName: resources.SchedulerKubeConfigFileName,
+			TmpDirectory:       getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
+		},
+	}
+}
+
+func getKubernetesStorageResources(c client.Client, log logr.Logger, tcpReconcilerConfig TenantControlPlaneReconcilerConfig, dbConnection sql.DBConnection, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource {
+	switch tcpReconcilerConfig.ETCDStorageType {
+	case types.ETCD:
+		return []resources.Resource{
+			&resources.ETCDCACertificatesResource{
+				Name:                  "etcd-ca-certificates",
+				Client:                c,
+				Log:                   log,
+				ETCDCASecretName:      tcpReconcilerConfig.ETCDCASecretName,
+				ETCDCASecretNamespace: tcpReconcilerConfig.ETCDCASecretNamespace,
+			},
+			&resources.ETCDCertificatesResource{
+				Name:   "etcd-certificates",
+				Client: c,
+				Log:    log,
+			},
+			&resources.ETCDSetupResource{
+				Name:                  "etcd-setup",
+				Client:                c,
+				Log:                   log,
+				ETCDClientCertsSecret: getNamespacedName(tcpReconcilerConfig.ETCDClientSecretNamespace, tcpReconcilerConfig.ETCDClientSecretName),
+				ETCDCACertsSecret:     getNamespacedName(tcpReconcilerConfig.ETCDCASecretNamespace, tcpReconcilerConfig.ETCDCASecretName),
+				Endpoints:             getArrayFromString(tcpReconcilerConfig.ETCDEndpoints),
+			},
+		}
+	case types.KineMySQL:
+		return []resources.Resource{
+			&resources.SQLStorageConfig{
+				Client: c,
+				Name:   "sql-config",
+				Host:   dbConnection.GetHost(),
+				Port:   dbConnection.GetPort(),
+			},
+			&resources.SQLSetup{
+				Client:       c,
+				Name:         "sql-setup",
+				DBConnection: dbConnection,
+			},
+			&resources.SQLCertificate{
+				Client:                   c,
+				Name:                     "sql-certificate",
+				StorageType:              tcpReconcilerConfig.ETCDStorageType,
+				SQLConfigSecretName:      tcpReconcilerConfig.KineMySQLSecretName,
+				SQLConfigSecretNamespace: tcpReconcilerConfig.KineMySQLSecretNamespace,
+			},
+		}
+	default:
+		return []resources.Resource{}
+	}
+}
+
+func getKubernetesDeploymentResources(c client.Client, tcpReconcilerConfig TenantControlPlaneReconcilerConfig, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource {
+	return []resources.Resource{
+		&resources.KubernetesDeploymentResource{
+			Client:                 c,
+			ETCDEndpoints:          getArrayFromString(tcpReconcilerConfig.ETCDEndpoints),
+			ETCDCompactionInterval: tcpReconcilerConfig.ETCDCompactionInterval,
+			ETCDStorageType:        tcpReconcilerConfig.ETCDStorageType,
+			KineContainerImage:     tcpReconcilerConfig.KineContainerImage,
+		},
+	}
+}
+
+func getKubernetesIngressResources(c client.Client, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource {
+	return []resources.Resource{
+		&resources.KubernetesIngressResource{
+			Client: c,
+		},
+	}
+}
+
+func getKubeadmPhaseResources(c client.Client, log logr.Logger, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource {
+	return []resources.Resource{
+		&resources.KubeadmPhase{
+			Name:   "upload-config-kubeadm",
+			Client: c,
+			Log:    log,
+			Phase:  resources.PhaseUploadConfigKubeadm,
+		},
+		&resources.KubeadmPhase{
+			Name:   "upload-config-kubelet",
+			Client: c,
+			Log:    log,
+			Phase:  resources.PhaseUploadConfigKubelet,
+		},
+		&resources.KubeadmPhase{
+			Name:   "bootstrap-token",
+			Client: c,
+			Log:    log,
+			Phase:  resources.PhaseBootstrapToken,
+		},
+	}
+}
+
+func getKubeadmAddonResources(c client.Client, log logr.Logger, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource {
+	return []resources.Resource{
+		&resources.KubeadmAddonResource{
+			Name:         "coredns",
+			Client:       c,
+			Log:          log,
+			KubeadmAddon: resources.AddonCoreDNS,
+		},
+		&resources.KubeadmAddonResource{
+			Name:         "kubeproxy",
+			Client:       c,
+			Log:          log,
+			KubeadmAddon: resources.AddonKubeProxy,
+		},
+	}
+}
+
+func getExternalKonnectivityResources(c client.Client, log logr.Logger, tcpReconcilerConfig TenantControlPlaneReconcilerConfig, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource {
+	return []resources.Resource{
+		&konnectivity.ServiceAccountResource{
+			Client: c,
+			Name:   "konnectivity-sa",
+		},
+		&konnectivity.ClusterRoleBindingResource{
+			Client: c,
+			Name:   "konnectivity-clusterrolebinding",
+		},
+		&konnectivity.KubernetesDeploymentResource{
+			Client: c,
+			Name:   "konnectivity-deployment",
+		},
+		&konnectivity.ServiceResource{
+			Client: c,
+			Name:   "konnectivity-service",
+		},
+		&konnectivity.Agent{
+			Client: c,
+			Name:   "konnectivity-agent",
+		},
+	}
+}
+
+func getInternalKonnectivityResources(c client.Client, log logr.Logger, tcpReconcilerConfig TenantControlPlaneReconcilerConfig, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource {
+	return []resources.Resource{
+		&konnectivity.EgressSelectorConfigurationResource{
+			Client: c,
+			Name:   "konnectivity-egress-selector-configuration",
+		},
+		&konnectivity.CertificateResource{
+			Client: c,
+			Log:    log,
+			Name:   "konnectivity-certificate",
+		},
+		&konnectivity.KubeconfigResource{
+			Client: c,
+			Name:   "konnectivity-kubeconfig",
+		},
+	}
+}
+
+func getArrayFromString(s string) []string {
+	var a []string
+	a = append(a, strings.Split(s, separator)...)
+
+	return a
+}
+
+func getNamespacedName(namespace string, name string) k8stypes.NamespacedName {
+	return k8stypes.NamespacedName{Namespace: namespace, Name: name}
+}
+
+func getTmpDirectory(base string, tenantControlPlane kamajiv1alpha1.TenantControlPlane) string {
+	return fmt.Sprintf("%s/%s/%s", base, tenantControlPlane.GetName(), uuid.New())
+}

controllers/storage.go

@@ -0,0 +1,57 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package controllers
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"fmt"
+
+	corev1 "k8s.io/api/core/v1"
+	k8stypes "k8s.io/apimachinery/pkg/types"
+
+	"github.com/clastix/kamaji/internal/sql"
+	"github.com/clastix/kamaji/internal/types"
+)
+
+func (r *TenantControlPlaneReconciler) getStorageConnection(ctx context.Context) (sql.DBConnection, error) {
+	// TODO: https://github.com/clastix/kamaji/issues/67
+	switch r.Config.ETCDStorageType {
+	case types.KineMySQL:
+		secret := &corev1.Secret{}
+		namespacedName := k8stypes.NamespacedName{Namespace: r.Config.KineMySQLSecretNamespace, Name: r.Config.KineMySQLSecretName}
+		if err := r.Client.Get(ctx, namespacedName, secret); err != nil {
+			return nil, err
+		}
+
+		rootCAs := x509.NewCertPool()
+		if ok := rootCAs.AppendCertsFromPEM(secret.Data["ca.crt"]); !ok {
+			return nil, fmt.Errorf("error creating root ca for mysql db connector")
+		}
+
+		certificate, err := tls.X509KeyPair(secret.Data["server.crt"], secret.Data["server.key"])
+		if err != nil {
+			return nil, err
+		}
+
+		return sql.GetDBConnection(
+			sql.ConnectionConfig{
+				SQLDriver: sql.MySQL,
+				User:      "root",
+				Password:  string(secret.Data["MYSQL_ROOT_PASSWORD"]),
+				Host:      r.Config.KineMySQLHost,
+				Port:      r.Config.KineMySQLPort,
+				DBName:    "mysql",
+				TLSConfig: &tls.Config{
+					ServerName:   r.Config.KineMySQLHost,
+					RootCAs:      rootCAs,
+					Certificates: []tls.Certificate{certificate},
+				},
+			},
+		)
+	default:
+		return nil, nil
+	}
+}

controllers/tenantcontrolplane_controller.go

@@ -6,9 +6,7 @@ package controllers
 import (
 	"context"
 	"fmt"
-	"strings"
 
-	"github.com/google/uuid"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	networkingv1 "k8s.io/api/networking/v1"
@@ -23,10 +21,11 @@ import (
 	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
 	kamajierrors "github.com/clastix/kamaji/internal/errors"
 	"github.com/clastix/kamaji/internal/resources"
+	"github.com/clastix/kamaji/internal/sql"
+	"github.com/clastix/kamaji/internal/types"
 )
 
 const (
-	separator = ","
 	finalizer = "finalizer.kamaji.clastix.io"
 )
 
@@ -39,6 +38,7 @@ type TenantControlPlaneReconciler struct {
 
 // TenantControlPlaneReconcilerConfig gives the necessary configuration for TenantControlPlaneReconciler.
 type TenantControlPlaneReconcilerConfig struct {
+	ETCDStorageType           types.ETCDStorageType
 	ETCDCASecretName          string
 	ETCDCASecretNamespace     string
 	ETCDClientSecretName      string
@@ -46,6 +46,12 @@ type TenantControlPlaneReconcilerConfig struct {
 	ETCDEndpoints             string
 	ETCDCompactionInterval    string
 	TmpBaseDirectory          string
+	DBConnection              sql.DBConnection
+	KineMySQLSecretName       string
+	KineMySQLSecretNamespace  string
+	KineMySQLHost             string
+	KineMySQLPort             int
+	KineContainerImage        string
 }
 
 //+kubebuilder:rbac:groups=kamaji.clastix.io,resources=tenantcontrolplanes,verbs=get;list;watch;create;update;patch;delete
@@ -76,186 +82,61 @@ func (r *TenantControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R
 		return ctrl.Result{}, nil
 	}
 
-	if markedToBeDeleted {
-		registeredDeleteableResources := []resources.DeleteableResource{
-			&resources.ETCDSetupResource{
-				Name:                  "etcd-setup",
-				Client:                r.Client,
-				Scheme:                r.Scheme,
-				Log:                   log,
-				ETCDClientCertsSecret: getNamespacedName(r.Config.ETCDClientSecretNamespace, r.Config.ETCDClientSecretName),
-				ETCDCACertsSecret:     getNamespacedName(r.Config.ETCDCASecretNamespace, r.Config.ETCDCASecretName),
-				Endpoints:             getArrayFromString(r.Config.ETCDEndpoints),
-			},
+	dbConnection, err := r.getStorageConnection(ctx)
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+	defer func() {
+		// TODO: Currently, etcd is not accessed using this dbConnection. For that reason we need this check
+		// Check: https://github.com/clastix/kamaji/issues/67
+		if dbConnection != nil {
+			dbConnection.Close()
 		}
+	}()
 
-		for _, resource := range registeredDeleteableResources {
-			if err := resource.Delete(ctx, tenantControlPlane); err != nil {
+	if markedToBeDeleted {
+		log.Info("marked for deletion, performing clean-up")
+
+		groupDeleteableResourceBuilderConfiguration := GroupDeleteableResourceBuilderConfiguration{
+			client:              r.Client,
+			log:                 log,
+			tcpReconcilerConfig: r.Config,
+			tenantControlPlane:  *tenantControlPlane,
+			DBConnection:        dbConnection,
+		}
+		registeredDeletableResources := GetDeletableResources(groupDeleteableResourceBuilderConfiguration)
+
+		for _, resource := range registeredDeletableResources {
+			if err := resources.HandleDeletion(ctx, resource, tenantControlPlane); err != nil {
 				return ctrl.Result{}, err
 			}
 		}
 
 		if hasFinalizer {
-			controllerutil.RemoveFinalizer(tenantControlPlane, finalizer)
-			if err := r.Update(ctx, tenantControlPlane); err != nil {
+			log.Info("removing finalizer")
+
+			if err := r.RemoveFinalizer(ctx, tenantControlPlane); err != nil {
 				return ctrl.Result{}, err
 			}
 		}
 
+		log.Info("resource deletion has been completed")
+
 		return ctrl.Result{}, nil
 	}
 
 	if !hasFinalizer {
-		controllerutil.AddFinalizer(tenantControlPlane, finalizer)
-		if err := r.Update(ctx, tenantControlPlane); err != nil {
-			return ctrl.Result{}, err
-		}
-
-		return ctrl.Result{}, nil
+		return ctrl.Result{}, r.AddFinalizer(ctx, tenantControlPlane)
 	}
 
-	registeredResources := []resources.Resource{
-		&resources.KubernetesUpgrade{
-			Name:   "upgrade",
-			Client: r.Client,
-		},
-		&resources.KubernetesServiceResource{
-			Client: r.Client,
-		},
-		&resources.KubeadmConfigResource{
-			Name:                   "kubeadmconfig",
-			Port:                   tenantControlPlane.Spec.NetworkProfile.Port,
-			KubernetesVersion:      tenantControlPlane.Spec.Kubernetes.Version,
-			PodCIDR:                tenantControlPlane.Spec.NetworkProfile.PodCIDR,
-			ServiceCIDR:            tenantControlPlane.Spec.NetworkProfile.ServiceCIDR,
-			Domain:                 tenantControlPlane.Spec.NetworkProfile.Domain,
-			ETCDs:                  getArrayFromString(r.Config.ETCDEndpoints),
-			ETCDCompactionInterval: r.Config.ETCDCompactionInterval,
-			Client:                 r.Client,
-			Scheme:                 r.Scheme,
-			Log:                    log,
-			TmpDirectory:           getTmpDirectory(r.Config.TmpBaseDirectory, *tenantControlPlane),
-		},
-		&resources.CACertificate{
-			Name:         "ca",
-			Client:       r.Client,
-			Log:          log,
-			TmpDirectory: getTmpDirectory(r.Config.TmpBaseDirectory, *tenantControlPlane),
-		},
-		&resources.FrontProxyCACertificate{
-			Name:         "front-proxy-ca-certificate",
-			Client:       r.Client,
-			Log:          log,
-			TmpDirectory: getTmpDirectory(r.Config.TmpBaseDirectory, *tenantControlPlane),
-		},
-		&resources.SACertificate{
-			Name:         "sa-certificate",
-			Client:       r.Client,
-			Log:          log,
-			TmpDirectory: getTmpDirectory(r.Config.TmpBaseDirectory, *tenantControlPlane),
-		},
-		&resources.APIServerCertificate{
-			Name:         "api-server-certificate",
-			Client:       r.Client,
-			Log:          log,
-			TmpDirectory: getTmpDirectory(r.Config.TmpBaseDirectory, *tenantControlPlane),
-		},
-		&resources.APIServerKubeletClientCertificate{
-			Name:         "api-server-kubelet-client-certificate",
-			Client:       r.Client,
-			Log:          log,
-			TmpDirectory: getTmpDirectory(r.Config.TmpBaseDirectory, *tenantControlPlane),
-		},
-		&resources.FrontProxyClientCertificate{
-			Name:         "front-proxy-client-certificate",
-			Client:       r.Client,
-			Log:          log,
-			TmpDirectory: getTmpDirectory(r.Config.TmpBaseDirectory, *tenantControlPlane),
-		},
-		&resources.KubeconfigResource{
-			Name:               "admin-kubeconfig",
-			Client:             r.Client,
-			Scheme:             r.Scheme,
-			Log:                log,
-			KubeConfigFileName: resources.AdminKubeConfigFileName,
-			TmpDirectory:       getTmpDirectory(r.Config.TmpBaseDirectory, *tenantControlPlane),
-		},
-		&resources.KubeconfigResource{
-			Name:               "controller-manager-kubeconfig",
-			Client:             r.Client,
-			Scheme:             r.Scheme,
-			Log:                log,
-			KubeConfigFileName: resources.ControllerManagerKubeConfigFileName,
-			TmpDirectory:       getTmpDirectory(r.Config.TmpBaseDirectory, *tenantControlPlane),
-		},
-		&resources.KubeconfigResource{
-			Name:               "scheduler-kubeconfig",
-			Client:             r.Client,
-			Scheme:             r.Scheme,
-			Log:                log,
-			KubeConfigFileName: resources.SchedulerKubeConfigFileName,
-			TmpDirectory:       getTmpDirectory(r.Config.TmpBaseDirectory, *tenantControlPlane),
-		},
-		&resources.ETCDCACertificatesResource{
-			Name:                  "etcd-ca-certificates",
-			Client:                r.Client,
-			Log:                   log,
-			ETCDCASecretName:      r.Config.ETCDCASecretName,
-			ETCDCASecretNamespace: r.Config.ETCDCASecretNamespace,
-		},
-		&resources.ETCDCertificatesResource{
-			Name:   "etcd-certificates",
-			Client: r.Client,
-			Log:    log,
-		},
-		&resources.ETCDSetupResource{
-			Name:                  "etcd-setup",
-			Client:                r.Client,
-			Scheme:                r.Scheme,
-			Log:                   log,
-			ETCDClientCertsSecret: getNamespacedName(r.Config.ETCDClientSecretNamespace, r.Config.ETCDClientSecretName),
-			ETCDCACertsSecret:     getNamespacedName(r.Config.ETCDCASecretNamespace, r.Config.ETCDCASecretName),
-			Endpoints:             getArrayFromString(r.Config.ETCDEndpoints),
-		},
-		&resources.KubernetesDeploymentResource{
-			Client:                 r.Client,
-			ETCDEndpoints:          getArrayFromString(r.Config.ETCDEndpoints),
-			ETCDCompactionInterval: r.Config.ETCDCompactionInterval,
-		},
-		&resources.KubernetesIngressResource{
-			Client: r.Client,
-		},
-		&resources.KubeadmPhase{
-			Name:   "upload-config-kubeadm",
-			Client: r.Client,
-			Log:    log,
-			Phase:  resources.PhaseUploadConfigKubeadm,
-		},
-		&resources.KubeadmPhase{
-			Name:   "upload-config-kubelet",
-			Client: r.Client,
-			Log:    log,
-			Phase:  resources.PhaseUploadConfigKubelet,
-		},
-		&resources.KubeadmPhase{
-			Name:   "bootstrap-token",
-			Client: r.Client,
-			Log:    log,
-			Phase:  resources.PhaseBootstrapToken,
-		},
-		&resources.KubeadmAddonResource{
-			Name:         "coredns",
-			Client:       r.Client,
-			Log:          log,
-			KubeadmAddon: resources.AddonCoreDNS,
-		},
-		&resources.KubeadmAddonResource{
-			Name:         "kubeproxy",
-			Client:       r.Client,
-			Log:          log,
-			KubeadmAddon: resources.AddonKubeProxy,
-		},
+	groupResourceBuilderConfiguration := GroupResourceBuilderConfiguration{
+		client:              r.Client,
+		log:                 log,
+		tcpReconcilerConfig: r.Config,
+		tenantControlPlane:  *tenantControlPlane,
+		DBConnection:        dbConnection,
 	}
+	registeredResources := GetResources(groupResourceBuilderConfiguration)
 
 	for _, resource := range registeredResources {
 		result, err := resources.Handle(ctx, resource, tenantControlPlane)
@@ -282,6 +163,8 @@ func (r *TenantControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R
 		return ctrl.Result{}, nil
 	}
 
+	log.Info(fmt.Sprintf("%s has been reconciled", tenantControlPlane.GetName()))
+
 	return ctrl.Result{}, nil
 }
 
@@ -331,21 +214,6 @@ func (r *TenantControlPlaneReconciler) updateStatus(ctx context.Context, namespa
 	return nil
 }
 
-func getArrayFromString(s string) []string {
-	var a []string
-	a = append(a, strings.Split(s, separator)...)
-
-	return a
-}
-
-func getNamespacedName(namespace string, name string) k8stypes.NamespacedName {
-	return k8stypes.NamespacedName{Namespace: namespace, Name: name}
-}
-
-func getTmpDirectory(base string, tenantControlPlane kamajiv1alpha1.TenantControlPlane) string {
-	return fmt.Sprintf("%s/%s/%s", base, tenantControlPlane.GetName(), uuid.New())
-}
-
 func hasFinalizer(tenantControlPlane kamajiv1alpha1.TenantControlPlane) bool {
 	for _, f := range tenantControlPlane.GetFinalizers() {
 		if f == finalizer {
@@ -355,3 +223,15 @@ func hasFinalizer(tenantControlPlane kamajiv1alpha1.TenantControlPlane) bool {
 
 	return false
 }
+
+func (r *TenantControlPlaneReconciler) AddFinalizer(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+	controllerutil.AddFinalizer(tenantControlPlane, finalizer)
+
+	return r.Update(ctx, tenantControlPlane)
+}
+
+func (r *TenantControlPlaneReconciler) RemoveFinalizer(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+	controllerutil.RemoveFinalizer(tenantControlPlane, finalizer)
+
+	return r.Update(ctx, tenantControlPlane)
+}

deploy/Makefile

@@ -0,0 +1,16 @@
+include etcd/Makefile
+
+deploy_path := $(patsubst %/,%,$(dir $(abspath $(lastword $(MAKEFILE_LIST)))))
+
+.DEFAULT_GOAL := kamaji
+
+.PHONY: etcd-cluster
+reqs: etcd-cluster
+
+.PHONY: kamaji
+kamaji: reqs
+	@kubectl apply -f $(deploy_path)/../../config/install.yaml
+
+.PHONY: destroy
+destroy: etcd-certificates/cleanup
+	@kubectl delete -f $(deploy_path)/../../config/install.yaml

deploy/README.md

@@ -0,0 +1,21 @@
+# Deploy Kamaji
+
+## Quickstart with KinD
+
+```sh
+make -C kind
+```
+
+## Multi-tenant etcd cluster
+
+> This assumes you already have a running Kubernetes cluster and kubeconfig.
+
+```sh
+make -C etcd
+```
+
+## Multi-tenant MySQL-MariaDB cluster
+
+> This assumes you already have a running Kubernetes cluster and kubeconfig.
+
+Read [this](./mysql/README.md) in order to know more about.

deploy/etcd/etcd-cluster.yaml

@@ -1,113 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: etcd
-  namespace:
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: etcd-server
-  namespace:
-spec:
-  type: ClusterIP
-  ports:
-    - name: client
-      port: 2379
-      protocol: TCP
-      targetPort: 2379
-  selector:
-    app: etcd
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: etcd
-  namespace:
-spec:
-  clusterIP: None
-  ports:
-    - port: 2379
-      name: client
-    - port: 2380
-      name: peer
-  selector:
-    app: etcd
----
-apiVersion: apps/v1
-kind: StatefulSet
-metadata:
-  name: etcd
-  labels:
-    app: etcd
-  namespace:
-spec:
-  serviceName: etcd
-  selector:
-    matchLabels:
-      app: etcd
-  replicas: 3
-  template:
-    metadata:
-      name: etcd
-      labels:
-        app: etcd
-    spec:
-      serviceAccountName: etcd
-      volumes:
-      - name: certs
-        secret:
-          secretName: etcd-certs
-      containers:
-        - name: etcd
-          image: quay.io/coreos/etcd:v3.5.1
-          ports:
-            - containerPort: 2379
-              name: client
-            - containerPort: 2380
-              name: peer
-          volumeMounts:
-          - name: data
-            mountPath: /var/run/etcd
-          - name: certs
-            mountPath: /etc/etcd/pki
-          command:
-          - etcd 
-          - --data-dir=/var/run/etcd
-          - --name=$(POD_NAME)
-          - --initial-cluster-state=new
-          - --initial-cluster=etcd-0=https://etcd-0.etcd.$(POD_NAMESPACE).svc.cluster.local:2380,etcd-1=https://etcd-1.etcd.$(POD_NAMESPACE).svc.cluster.local:2380,etcd-2=https://etcd-2.etcd.$(POD_NAMESPACE).svc.cluster.local:2380
-          - --initial-advertise-peer-urls=https://$(POD_NAME).etcd.$(POD_NAMESPACE).svc.cluster.local:2380
-          - --initial-cluster-token=kamaji
-          - --listen-client-urls=https://0.0.0.0:2379
-          - --advertise-client-urls=https://etcd-0.etcd.$(POD_NAMESPACE).svc.cluster.local:2379,https://etcd-1.etcd.$(POD_NAMESPACE).svc.cluster.local:2379,https://etcd-2.etcd.$(POD_NAMESPACE).svc.cluster.local:2379,https://etcd-server.$(POD_NAMESPACE).svc.cluster.local:2379
-          - --client-cert-auth=true
-          - --trusted-ca-file=/etc/etcd/pki/ca.crt
-          - --cert-file=/etc/etcd/pki/server.pem
-          - --key-file=/etc/etcd/pki/server-key.pem
-          - --listen-peer-urls=https://0.0.0.0:2380
-          - --peer-client-cert-auth=true
-          - --peer-trusted-ca-file=/etc/etcd/pki/ca.crt
-          - --peer-cert-file=/etc/etcd/pki/peer.pem 
-          - --peer-key-file=/etc/etcd/pki/peer-key.pem
-          - --auto-compaction-mode=periodic
-          - --auto-compaction-retention=5m
-          - --v=8
-          env:
-          - name: POD_NAME
-            valueFrom:
-              fieldRef:
-                fieldPath: metadata.name
-          - name: POD_NAMESPACE
-            valueFrom:
-              fieldRef:
-                fieldPath: metadata.namespace
-  volumeClaimTemplates:
-  - metadata:
-      name: data
-    spec:
-      accessModes: ["ReadWriteOnce"]
-      resources:
-        requests:
-          storage: 2Gi
-

deploy/kind/Makefile

@@ -11,7 +11,10 @@ prometheus-stack:
 	helm repo update
 	helm install prometheus-stack --create-namespace -n monitoring prometheus-community/kube-prometheus-stack
 
-kamaji: kind ingress-nginx etcd-cluster
+reqs: kind ingress-nginx etcd-cluster
+
+kamaji: reqs
+	@kubectl apply -f $(kind_path)/../../config/install.yaml
 
 destroy: kind/destroy etcd-certificates/cleanup
 
@@ -34,3 +37,6 @@ kamaji-kind-worker-push: kamaji-kind-worker-build
 
 kamaji-kind-worker-join:
 	$(kind_path)/join-node.bash
+
+kamaji-kind-worker-join-through-konnectivity:
+	$(kind_path)/join-node-konnectivity.bash

deploy/kind/etcd-cluster.yaml

@@ -1,177 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  creationTimestamp: "2021-12-08T17:49:38Z"
-  name: etcd
-  namespace: kamaji-system
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: etcd-server
-  namespace: kamaji-system
-spec:
-  type: ClusterIP
-  ports:
-    - name: client
-      port: 2379
-      protocol: TCP
-      targetPort: 2379
-  selector:
-    app: etcd
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: etcd
-  namespace: kamaji-system
-spec:
-  clusterIP: None
-  ports:
-    - port: 2379
-      name: client
-    - port: 2380
-      name: peer
-  selector:
-    app: etcd
----
-apiVersion: apps/v1
-kind: StatefulSet
-metadata:
-  name: etcd
-  labels:
-    app: etcd
-  namespace: kamaji-system
-spec:
-  serviceName: etcd
-  selector:
-    matchLabels:
-      app: etcd
-  replicas: 3
-  template:
-    metadata:
-      name: etcd
-      labels:
-        app: etcd
-    spec:
-      serviceAccountName: etcd
-      volumes:
-      - name: certs
-        secret:
-          secretName: etcd-certs
-      containers:
-        - name: etcd
-          image: quay.io/coreos/etcd:v3.5.1
-          ports:
-            - containerPort: 2379
-              name: client
-            - containerPort: 2380
-              name: peer
-          volumeMounts:
-            - name: data
-              mountPath: /var/run/etcd
-            - name: certs
-              mountPath: /etc/etcd/pki
-          command:
-          - etcd 
-          - --data-dir=/var/run/etcd
-          - --name=$(POD_NAME)
-          - --initial-cluster-state=new
-          - --initial-cluster=etcd-0=https://etcd-0.etcd.$(POD_NAMESPACE).svc.cluster.local:2380,etcd-1=https://etcd-1.etcd.$(POD_NAMESPACE).svc.cluster.local:2380,etcd-2=https://etcd-2.etcd.$(POD_NAMESPACE).svc.cluster.local:2380
-          - --initial-advertise-peer-urls=https://$(POD_NAME).etcd.$(POD_NAMESPACE).svc.cluster.local:2380
-          - --initial-cluster-token=kamaji
-          - --listen-client-urls=https://0.0.0.0:2379
-          - --advertise-client-urls=https://etcd-0.etcd.$(POD_NAMESPACE).svc.cluster.local:2379,https://etcd-1.etcd.$(POD_NAMESPACE).svc.cluster.local:2379,https://etcd-2.etcd.$(POD_NAMESPACE).svc.cluster.local:2379,https://etcd-server.$(POD_NAMESPACE).svc.cluster.local:2379
-          - --client-cert-auth=true
-          - --trusted-ca-file=/etc/etcd/pki/ca.crt
-          - --cert-file=/etc/etcd/pki/server.pem
-          - --key-file=/etc/etcd/pki/server-key.pem
-          - --listen-peer-urls=https://0.0.0.0:2380
-          - --peer-client-cert-auth=true
-          - --peer-trusted-ca-file=/etc/etcd/pki/ca.crt
-          - --peer-cert-file=/etc/etcd/pki/peer.pem 
-          - --peer-key-file=/etc/etcd/pki/peer-key.pem
-          - --auto-compaction-mode=periodic
-          - --auto-compaction-retention=5m
-          - --v=8
-          env:
-          - name: POD_NAME
-            valueFrom:
-              fieldRef:
-                fieldPath: metadata.name
-          - name: POD_NAMESPACE
-            valueFrom:
-              fieldRef:
-                fieldPath: metadata.namespace
-          livenessProbe:
-            failureThreshold: 8
-            httpGet:
-              host: 127.0.0.1
-              path: /health
-              port: 2381
-              scheme: HTTP
-            initialDelaySeconds: 10
-            periodSeconds: 10
-            timeoutSeconds: 15
-          startupProbe:
-            failureThreshold: 24
-            httpGet:
-              host: 127.0.0.1
-              path: /health
-              port: 2381
-              scheme: HTTP
-            initialDelaySeconds: 10
-            periodSeconds: 10
-            timeoutSeconds: 15
-  volumeClaimTemplates:
-    - metadata:
-        name: data
-      spec:
-        accessModes: ["ReadWriteOnce"]
-        resources:
-          requests:
-            storage: 8Gi
----
-apiVersion: v1
-kind: Pod
-metadata:
-  labels:
-    app: etcd
-  name: etcd-root-client
-  namespace: kamaji-system
-spec:
-  serviceAccountName: etcd
-  containers:
-  - command:
-    - sleep
-    - infinity
-    env:
-    - name: POD_NAMESPACE
-      valueFrom:
-        fieldRef:
-          fieldPath: metadata.namespace    
-    - name: ETCDCTL_ENDPOINTS
-      value: https://etcd-server.$(POD_NAMESPACE).svc.cluster.local:2379
-    - name: ETCDCTL_CACERT
-      value: /opt/certs/ca/ca.crt
-    - name: ETCDCTL_CERT
-      value: /opt/certs/root-client-certs/tls.crt
-    - name: ETCDCTL_KEY
-      value: /opt/certs/root-client-certs/tls.key
-    image: quay.io/coreos/etcd:v3.5.1
-    imagePullPolicy: IfNotPresent
-    name: etcd-client
-    resources: {}
-    volumeMounts:
-    - name: root-client-certs
-      mountPath: /opt/certs/root-client-certs
-    - name: ca
-      mountPath: /opt/certs/ca     
-  volumes:
-  - name: root-client-certs
-    secret:
-      secretName: root-client-certs
-  - name: ca
-    secret:
-      secretName: etcd-certs     
-

deploy/kind/join-node-konnectivity.bash

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+set -e
+
+# Constants
+export DOCKER_IMAGE_NAME="clastix/kamaji-kind-worker"
+
+# Variables
+export KUBERNETES_VERSION=${1:-latest}
+export KUBECONFIG="${KUBECONFIG:-/tmp/kubeconfig}"
+
+if [ -z $2 ]
+then
+    MAPPING_PORT=""
+else
+    MAPPING_PORT="-p ${2}:80"
+fi
+
+export KONNECTIVITY_PROXY_HOST=${3:-konnectiviy.local}
+
+clear
+echo "Welcome to join a new node through Konnectivity"
+
+echo -ne "\nChecking right kubeconfig\n"
+kubectl cluster-info
+echo "Are you pointing to the right tenant control plane? (Type return to continue)"
+read
+
+JOIN_CMD="$(kubeadm --kubeconfig=${KUBECONFIG} token create --print-join-command) --ignore-preflight-errors=SystemVerification"
+echo "Deploying new node..."
+KIND_IP=$(docker inspect kamaji-control-plane --format='{{.NetworkSettings.Networks.kind.IPAddress}}')
+NODE=$(docker run -d --add-host $KONNECTIVITY_PROXY_HOST:$KIND_IP --privileged -v /lib/modules:/lib/modules:ro -v /var --net host $MAPPING_PORT $DOCKER_IMAGE_NAME:$KUBERNETES_VERSION)
+sleep 10
+echo "Joining new node..."
+docker exec -e JOIN_CMD="$JOIN_CMD" $NODE /bin/bash -c "$JOIN_CMD"

deploy/kind/join-node.bash

@@ -8,6 +8,7 @@ export DOCKER_NETWORK="kind"
 
 # Variables
 export KUBERNETES_VERSION=${1:-latest}
+export KUBECONFIG="${KUBECONFIG:-/tmp/kubeconfig}"
 
 if [ -z $2 ]
 then
@@ -24,7 +25,7 @@ kubectl cluster-info
 echo "Are you pointing to the right tenant control plane? (Type return to continue)"
 read
 
-JOIN_CMD="$(kubeadm --kubeconfig=/tmp/kubeconfig token create --print-join-command) --ignore-preflight-errors=SystemVerification"
+JOIN_CMD="$(kubeadm --kubeconfig=${KUBECONFIG} token create --print-join-command) --ignore-preflight-errors=SystemVerification"
 echo "Deploying new node..."
 NODE=$(docker run -d --privileged -v /lib/modules:/lib/modules:ro -v /var --net $DOCKER_NETWORK $MAPPING_PORT $DOCKER_IMAGE_NAME:$KUBERNETES_VERSION)
 sleep 10

deploy/kind/kind-kamaji.yaml

@@ -23,10 +23,14 @@ nodes:
   - containerPort: 443
     hostPort: 443
     protocol: TCP
+    ## expose port 31132 of the node to port 31132 on the host for konnectivity
+  - containerPort: 31132
+    hostPort: 31132
+    protocol: TCP   
     ## expose port 31443 of the node to port 31443 on the host
   - containerPort: 31443
     hostPort: 31443
-    protocol: TCP    
+    protocol: TCP
     ## expose port 6443 of the node to port 8443 on the host
   - containerPort: 6443
     hostPort: 8443

deploy/mysql/Makefile

@@ -0,0 +1,31 @@
+mariadb_path := $(patsubst %/,%,$(dir $(abspath $(lastword $(MAKEFILE_LIST)))))
+
+.PHONY: mariadb mariadb-certificates mariadb-secrets
+
+mariadb: mariadb-certificates mariadb-secrets mariadb-deployment
+
+mariadb-certificates:
+	rm -rf $(mariadb_path)/certs && mkdir $(mariadb_path)/certs
+	cfssl gencert -initca $(mariadb_path)/ca-csr.json | cfssljson -bare $(mariadb_path)/certs/ca
+	@mv $(mariadb_path)/certs/ca.pem $(mariadb_path)/certs/ca.crt
+	@mv $(mariadb_path)/certs/ca-key.pem $(mariadb_path)/certs/ca.key
+	cfssl gencert -ca=$(mariadb_path)/certs/ca.crt -ca-key=$(mariadb_path)/certs/ca.key \
+		-config=$(mariadb_path)/config.json -profile=server \
+		$(mariadb_path)/server-csr.json | cfssljson -bare $(mariadb_path)/certs/server
+	@mv $(mariadb_path)/certs/server.pem $(mariadb_path)/certs/server.crt
+	@mv $(mariadb_path)/certs/server-key.pem $(mariadb_path)/certs/server.key
+	chmod 644 $(mariadb_path)/certs/*
+
+mariadb-secrets:
+	@kubectl -n kamaji-system create secret generic mysql-config \
+		--from-file=$(mariadb_path)/certs/ca.crt --from-file=$(mariadb_path)/certs/ca.key \
+		--from-file=$(mariadb_path)/certs/server.key --from-file=$(mariadb_path)/certs/server.crt \
+		--from-file=$(mariadb_path)/mysql-ssl.cnf \
+		--from-literal=MYSQL_ROOT_PASSWORD=root
+
+mariadb-deployment:
+	@kubectl -n kamaji-system apply -f $(mariadb_path)/mariadb.yaml
+
+destroy:
+	@kubectl delete -n kamaji-system -f $(mariadb_path)/mariadb.yaml
+	@kubectl delete -n kamaji-system secret mysql-config

deploy/mysql/README.md

@@ -0,0 +1,43 @@
+# MySQL as Kubernetes Storage
+
+Kamaji offers the possibility of having a different storage system than `ETCD` thanks to [kine](https://github.com/k3s-io/kine). One of the implementations is [MySQL](https://www.mysql.com/).
+
+Kamaji project is developed using [kind](https://kind.sigs.k8s.io), therefore, MySQL (or [MariaDB](https://mariadb.org/) in this case) will be deployed into the local kubernetes cluster in order to be used as storage for the tenants.
+
+There is a Makefile to help with the process:
+
+* **Full Installation**
+
+```bash
+$ make mariadb
+```
+
+This action will perform all the necessary stuffs to have MariaDB as kubernetes storage backend using kine.
+
+* **Certificate creation**
+
+```bash
+$ make mariadb-certificates
+```
+
+Communication between kine and the backend is encrypted, therefore, some certificates must be created.
+
+* **Secret Deployment**
+
+```bash
+$ make mariadb-secrets
+```
+
+Previous certificates and MySQL configuration have to be available in order to be used. They will be under the secret `kamaji-system:mysql-config`.
+
+* **Deployment**
+
+```bash
+$ make mariadb-deployment
+```
+
+* **Uninstall Everything**
+
+```bash
+$ make destroy
+```

deploy/mysql/ca-csr.json

@@ -0,0 +1,18 @@
+{
+  "CN": "Clastix CA",
+  "key": {
+    "algo": "rsa",
+    "size": 2048
+  },
+  "names": [
+    {
+      "C": "IT",
+      "ST": "Italy",
+      "L": "Milan"
+    }
+  ],
+  "hosts": [
+    "127.0.0.1",
+    "localhost"
+  ]
+}

deploy/mysql/config.json

@@ -0,0 +1,18 @@
+{
+  "signing": {
+      "default": {
+          "expiry": "8760h"
+      },
+      "profiles": {
+          "server": {
+              "expiry": "8760h",
+              "usages": [
+                  "signing",
+                  "key encipherment",
+                  "server auth",
+                  "client auth"
+              ]
+          }
+      }
+  }
+}

deploy/mysql/kine.yaml

@@ -0,0 +1,61 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kine-tenant
+  namespace:
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: kine-tenant
+  namespace:
+spec:
+  type: ClusterIP
+  ports:
+    - name: server
+      port: 2379
+      protocol: TCP
+      targetPort: 2379
+  selector:
+    app: kine-tenant
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: kine-tenant
+  labels:
+    app: kine-tenant
+  namespace:
+spec:
+  selector:
+    matchLabels:
+      app: kine-tenant
+  replicas: 1
+  template:
+    metadata:
+      name: kine-tenant
+      labels:
+        app: kine-tenant
+    spec:
+      serviceAccountName: kine-tenant
+      volumes:
+      - name: certs
+        secret:
+          secretName: mysql-certs
+      containers:
+        - name: kine-tenant
+          image: rancher/kine:v0.9.2-amd64
+          ports:
+            - containerPort: 2379
+              name: server
+          volumeMounts:
+          - name: certs
+            mountPath: /kine
+          env:    
+          - name: GODEBUG
+            value: "x509ignoreCN=0"
+          args:
+          - --endpoint=mysql://tenant1:tenant1@tcp(mysql:3306)/tenant1
+          - --ca-file=/kine/ca.crt
+          - --cert-file=/kine/server.crt
+          - --key-file=/kine/server.key

deploy/mysql/mariadb.yaml

@@ -0,0 +1,77 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: mariadb
+  namespace:
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: mariadb
+  namespace:
+spec:
+  type: ClusterIP
+  ports:
+    - name: server
+      port: 3306
+      protocol: TCP
+      targetPort: 3306
+  selector:
+    app: mariadb
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: mariadb
+  labels:
+    app: mariadb
+  namespace:
+spec:
+  selector:
+    matchLabels:
+      app: mariadb
+  replicas: 1
+  template:
+    metadata:
+      name: mariadb
+      labels:
+        app: mariadb
+    spec:
+      serviceAccountName: mariadb
+      volumes:
+      - name: certs
+        secret:
+          secretName: mysql-config
+      - name: data
+        persistentVolumeClaim:
+          claimName: pvc-mariadb
+      containers:
+        - name: mariadb
+          image: mariadb:10.7.4
+          ports:
+            - containerPort: 3306
+              name: server
+          volumeMounts:
+          - name: data
+            mountPath: /var/lib/mariadb
+          - name: certs
+            mountPath: /etc/mysql/conf.d/
+          env:
+          - name: MYSQL_ROOT_PASSWORD
+            valueFrom:
+              secretKeyRef:
+                name: mysql-config
+                key: MYSQL_ROOT_PASSWORD                
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: pvc-mariadb
+  namespace:
+spec:
+  accessModes:
+  - ReadWriteOnce
+  resources:
+    requests:
+      storage: 2Gi
+  storageClassName: standard

deploy/mysql/mysql-ssl.cnf

@@ -0,0 +1,5 @@
+[mysqld]
+ssl-ca=/etc/mysql/conf.d/ca.crt
+ssl-cert=/etc/mysql/conf.d/server.crt
+ssl-key=/etc/mysql/conf.d/server.key
+require_secure_transport=ON

deploy/mysql/server-csr.json

@@ -0,0 +1,19 @@
+{
+  "CN": "mariadb.kamaji-system.svc.cluster.local",
+  "key": {
+    "algo": "rsa",
+    "size": 2048
+  },
+  "hosts": [
+    "127.0.0.1",
+    "localhost",
+    "mariadb",
+    "mariadb.kamaji-system",
+    "mariadb.kamaji-system.svc",
+    "mariadb.kamaji-system.svc.cluster.local",
+    "mysql",
+    "mysql.kamaji-system",
+    "mysql.kamaji-system.svc",
+    "mysql.kamaji-system.svc.cluster.local"
+  ]
+}

docs/README.md

@@ -0,0 +1,10 @@
+# Kamaji documentation
+
+- [Architecture](./architecture.md)
+- [Concepts](./concepts.md)
+- [Getting started](./getting-started-with-kamaji.md)
+- [Kamaji Deployment](./kamaji-deployment-guide.md)
+- [Tenant deployment](./kamaji-tenant-deployment-guide.md)
+- Deployment on cloud providers:
+  - [Azure](./kamaji-azure-deployment-guide.md)
+- [Reference](./reference.md)

docs/getting-started-with-kamaji.md

@@ -2,7 +2,7 @@
 
 This document explains how to deploy a minimal Kamaji setup on [KinD](https://kind.sigs.k8s.io/) for development scopes. Please refer to the [Kamaji documentation](../README.md) for understanding all the terms used in this guide, as for example: `admin cluster` and `tenant control plane`.
 
-## Tools
+## Pre-requisites
 
 We assume you have installed on your workstation:
 
@@ -23,21 +23,85 @@ The instance of Kamaji is made of a single node hosting:
 - admin worker
 - multi-tenant etcd cluster
 
-The multi-tenant etcd cluster is deployed as statefulset into the Kamaji node.
+### Standard
 
-Run `make kamaji` to setup Kamaji on KinD.
+You can install your KinD cluster, ETCD multi-tenant cluster and Kamaji operator with a **single command**:
 
 ```bash
-cd ./deploy/kind
-make kamaji
+$ make -C deploy/kind
 ```
 
-At this moment you will have your KinD up and running and ETCD cluster in multitenant mode. 
+Now you can [create your first `TenantControlPlane`](#deploy-tenant-control-plane).
+
+### Data store-specific
+
+#### ETCD
+
+The multi-tenant etcd cluster is deployed as statefulset into the Kamaji node.
+
+Run `make reqs` to setup Kamaji's requisites on KinD:
+
+```bash
+$ make -C deploy/kind reqs
+```
+
+At this moment you will have your KinD up and running and ETCD cluster in multitenant mode.
+
+Now you're ready to [install Kamaji operator](#install-kamaji).
+
+#### Kine MySQL
+
+> The MySQL-compatible cluster provisioning is omitted here.
+
+Kamaji offers the possibility of using a different storage system than `ETCD` for the tenants, like MySQL compatible databases.
+
+Once a compatible-mysql database is running, we need to provide information about it to kamaji:
+
+```
+--etcd-storage-type=kine-mysql
+--kine-mysql-host=<database host>
+--kine-mysql-port=<database port>
+--kine-mysql-secret-name=<secret name>
+--kine-mysql-secret-namespace=<secret namespace>
+```
+
+The secret with the configuration and certificates for mysql should look like:
+```yaml
+apiVersion: v1
+data:
+  MYSQL_ROOT_PASSWORD: ...
+  ca.crt: ...
+  ca.key: ...
+  mysql-ssl.cnf: ...
+  server.crt:  ...
+  server.key:  ...
+kind: Secret
+metadata:
+  creationTimestamp: "2022-06-30T08:03:15Z"
+  name: mysql-config
+  namespace: kamaji-system
+  resourceVersion: "32228"
+  uid: 51b155a1-426c-42d2-8147-be680bf458a6
+type: Opaque
+```
+
+and `mysql-ssl.cnf`:
+```
+[mysqld]
+ssl-ca=/etc/mysql/conf.d/ca.crt
+ssl-cert=/etc/mysql/conf.d/server.crt
+ssl-key=/etc/mysql/conf.d/server.key
+require_secure_transport=ON
+```
+
+You can read more about it [here](../deploy/mysql/README.md).
+
+Assuming you adjusted the [Kamaji manifest](./config/install.yaml) to connect to the MySQL-compatible database, you can now install it.
 
 ### Install Kamaji
 
 ```bash
-$ kubectl apply -f ../../config/install.yaml
+$ kubectl apply -f config/install.yaml
 ```
 
 ### Deploy Tenant Control Plane
@@ -82,16 +146,15 @@ spec:
   networkProfile:
     address: "172.18.0.2"
     port: 31443
-    domain: "clastix.labs"
+    certSANs:
+    - "test.clastixlabs.io"
     serviceCidr: "10.96.0.0/16"
     podCidr: "10.244.0.0/16"
     dnsServiceIPs: 
     - "10.96.0.10"
   addons:
-    coreDNS:
-      enabled: true
-    kubeProxy:
-      enabled: true
+    coreDNS: {}
+    kubeProxy: {}
 EOF
 ```
 
@@ -141,3 +204,9 @@ d2d4b468c9de   Ready    <none>   44s   v1.23.4
 > For more complex scenarios (exposing port, different version and so on), run `join-node.bash`
 
 Tenant control plane provision has been finished in a minimal Kamaji setup based on KinD. Therefore, you could develop, test and make your own experiments with Kamaji.
+
+## Cleanup
+
+```bash
+$ make destroy
+```

docs/kamaji-azure-deployment-guide.md

@@ -67,10 +67,10 @@ kubectl cluster-info
 ```
 
 ## Setup internal multi-tenant etcd
-Follow the instructions [here](./getting-started-with-kamaji.md#setup-internal-multi-tenant-etcd).
+Follow the instructions [here](./kamaji-deployment-guide.md#setup-internal-multi-tenant-etcd).
 
 ## Install Kamaji controller
-Follow the instructions [here](./getting-started-with-kamaji.md#install-kamaji-controller).
+Follow the instructions [here](./kamaji-deployment-guide.md#install-kamaji-controller).
 
 ## Create Tenant Clusters
 To create a Tenant Cluster in Kamaji on AKS, we have to work on both the Kamaji and Azure infrastructure sides.
@@ -129,16 +129,15 @@ spec:
       - LimitRanger
   networkProfile:
     port: 6443
-    domain: ${KAMAJI_REGION}.cloudapp.azure.com
+    certSANs:
+    - ${TENANT_NAME}.${KAMAJI_REGION}.cloudapp.azure.com
     serviceCidr: ${TENANT_SVC_CIDR}
     podCidr: ${TENANT_POD_CIDR}
     dnsServiceIPs:
     - ${TENANT_DNS_SERVICE}
   addons:
-    coreDNS:
-      enabled: true
-    kubeProxy:
-      enabled: true
+    coreDNS: {}
+    kubeProxy: {}
 ---
 apiVersion: v1
 kind: Service

docs/kamaji-deployment-guide.md

@@ -496,7 +496,7 @@ kubectl -n ${ETCD_NAMESPACE} apply -f etcd/etcd-cluster.yaml
 Install an `etcd` client to interact with the `etcd` server
 
 ```bash
-kubectl -n ${ETCD_NAMESPACE} create secret tls root-certs \
+kubectl -n ${ETCD_NAMESPACE} create secret tls root-client-certs \
   --key=kamaji/etcd/root-key.pem \
   --cert=kamaji/etcd/root.pem
 

docs/kamaji-tenant-deployment-guide.md

@@ -62,19 +62,35 @@ spec:
   networkProfile:
     address: ${TENANT_ADDR}
     port: ${TENANT_PORT}
-    domain: ${TENANT_DOMAIN}
+    certSANs:
+    - ${TENANT_NAME}.${TENANT_DOMAIN}
     serviceCidr: ${TENANT_SVC_CIDR}
     podCidr: ${TENANT_POD_CIDR}
     dnsServiceIPs:
     - ${TENANT_DNS_SERVICE}
   addons:
-    coreDNS:
-      enabled: true
-    kubeProxy:
-      enabled: true
+    coreDNS: {}
+    kubeProxy: {}
 EOF
 ```
 
+If workers are not reachable from tenant control plane, konnectivity can be enabled (it is by default):
+
+```yaml
+...
+  addons:
+    konnectivity:
+      enabled: true
+      proxyHost: "172.18.0.2"
+      proxyPort: 31132
+...   
+```
+
+`proxyHost` is the address where konnectivity proxy server will be running. Konnectivity works as sidecar container into the tenant control plane pod. If no value is specified, it will take tenant IP.
+
+`proxyPort` is the port where konnectivity proxy server will be running. (default `8132`)
+
+
 ```bash
 kubectl create namespace ${TENANT_NAMESPACE}
 kubectl apply -f ${TENANT_NAMESPACE}-${TENANT_NAME}-tcp.yaml

docs/reference.md

@@ -21,8 +21,13 @@ Available flags are the following:
 --etcd-client-secret-name            Name of the secret which contains ETCD client certificates. (default: "root-client-certs")
 --etcd-client-secret-namespace       Name of the namespace where the secret which contains ETCD client certificates is. (default: "kamaji")
 --etcd-compaction-interval           ETCD Compaction interval (i.e. "5m0s"). (default: "0" (disabled))
---etcd-endpoints                     Comma-separated list with ETCD endpoints (i.e. etcd-0.etcd.kamaji.svc.cluster.local,etcd-1.etcd.kamaji.svc.cluster.local,etcd-2.etcd.kamaji.svc.cluster.local)
+--etcd-endpoints                     Comma-separated list with ETCD endpoints (i.e. https://etcd-0.etcd.kamaji.svc.cluster.local,https://etcd-1.etcd.kamaji.svc.cluster.local,https://etcd-2.etcd.kamaji.svc.cluster.local)
+--etcd-storage-type                  ETCD Storage type (i.e. "etcd", "kine-mysql"). (default: "etcd")
 --health-probe-bind-address string   The address the probe endpoint binds to. (default ":8081")
+--kine-mysql-host                    Host where MySQL is running (default: "localhost")
+--kine-mysql-port int                Port where MySQL is running (default: 3306)
+--kine-mysql-secret-name             Name of the secret where the necessary configuration and certificates are. (default: "mysql-config")
+--kine-mysql-secret-name             Name of the namespace of the secret where the necessary configuration and certificates are. (default: "kamaji-system")
 --kubeconfig string                  Paths to a kubeconfig. Only required if out-of-cluster.
 --leader-elect                       Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.
 --metrics-bind-address string        The address the metric endpoint binds to. (default ":8080")
@@ -43,9 +48,14 @@ Available environment variables are:
 | `KAMAJI_ETCD_CLIENT_SECRET_NAMESPACE`      | Name of the namespace where the secret which contains ETCD client certificates is. (default: "kamaji")  |
 | `KAMAJI_ETCD_COMPACTION_INTERVAL`      | ETCD Compaction interval (i.e. "5m0s"). (default: "0" (disabled))  |
 | `KAMAJI_ETCD_ENDPOINTS`      | Comma-separated list with ETCD endpoints (i.e. etcd-server-1:2379,etcd-server-2:2379). (default: "etcd-server:2379")  |
+| `KAMAJI_ETCD_STORAGE_TYPE`      | ETCD Storage type (i.e. "etcd", "kine-mysql"). (default: "etcd")  |
 | `KAMAJI_ETCD_SERVERS`      | Comma-separated list with ETCD servers (i.e. etcd-0.etcd.kamaji.svc.cluster.local,etcd-1.etcd.kamaji.svc.cluster.local,etcd-2.etcd.kamaji.svc.cluster.local)  |
 | `KAMAJI_METRICS_BIND_ADDRESS`      | The address the metric endpoint binds to. (default ":8080")  |
 | `KAMAJI_HEALTH_PROBE_BIND_ADDRESS` | The address the probe endpoint binds to. (default ":8081")   |
+| `KAMAJI_KINE_MYSQL_HOST` | Host where MySQL is running(default "localhost")   |
+| `KAMAJI_KINE_MYSQL_PORT` | Port where MySQL is running (default: 3306)   |
+| `KAMAJI_KINE_MYSQL_SECRET_NAME` | Name of the secret where the necessary configuration and certificates are. (default: "mysql-config")   |
+| `KAMAJI_KINE_MYSQL_SECRET_NAMESPACE` | Name of the namespace of the secret where the necessary configuration and certificates are. (default: "kamaji-system")   |
 | `KAMAJI_LEADER_ELECTION`           | Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager. |
 | `KAMAJI_TMP_DIRECTORY`           | Directory which will be used to work with temporary files. (default "/tmp/kamaji") |
 
@@ -80,7 +90,7 @@ $ make yaml-installation-file
 
 ```
 
-It will generate a yaml installation file at `config/installation.yaml`. It should be customize accordingly.
+It will generate a yaml installation file at `config/install.yaml`. It should be customize accordingly.
 
 
 ## Tenant Control Planes
@@ -89,20 +99,29 @@ It will generate a yaml installation file at `config/installation.yaml`. It shou
 
 Kamaji provides optional installations into the deployed tenant control plane through add-ons. Is it possible to enable/disable them through the `tcp` definition.
 
-By default, add-ons are installed if nothing is specified in the `tcp` definition.
-
 ### Core DNS
 
 ```yaml
 addons:
-    coreDNS:
-        enabled: true
+    coreDNS: {}
 ```
 
 ### Kube-Proxy
 
 ```yaml
 addons:
-    kubeProxy:
-        enabled: true
+    kubeProxy: {}
 ```
+
+### Konnectivity
+
+```yaml
+addons:
+    konnectivity:
+        proxyPort: 31132 # mandatory
+        proxyHost: "172.18.0.2"  
+        allowAddressAsExternalIP: false
+        serviceType: NodePort # mandatory
+        version: v0.0.31
+        serverImage: us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-server
+        agentImage: us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-agent

e2e/suite_test.go

@@ -23,9 +23,11 @@ import (
 // These tests use Ginkgo (BDD-style Go testing framework). Refer to
 // http://onsi.github.io/ginkgo/ to learn more about Ginkgo.
 
-var cfg *rest.Config // nolint
-var k8sClient client.Client
-var testEnv *envtest.Environment
+var (
+	cfg       *rest.Config
+	k8sClient client.Client
+	testEnv   *envtest.Environment
+)
 
 func TestAPIs(t *testing.T) {
 	RegisterFailHandler(Fail)
@@ -43,7 +45,9 @@ var _ = BeforeSuite(func() {
 		UseExistingCluster: pointer.Bool(true),
 	}
 
-	cfg, err := testEnv.Start()
+	var err error
+
+	cfg, err = testEnv.Start()
 	Expect(err).NotTo(HaveOccurred())
 	Expect(cfg).NotTo(BeNil())
 

e2e/tenant_control_plane_ready_test.go

@@ -1,3 +1,6 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
 package e2e
 
 import (
@@ -17,8 +20,8 @@ var _ = Describe("Deploy a TenantControlPlane resource", func() {
 	tcp := kamajiv1alpha1.TenantControlPlane{
 		TypeMeta: metav1.TypeMeta{},
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      "tenant-test",
-			Namespace: "kamaji-system",
+			Name:      "tcp-clusterip",
+			Namespace: "default",
 		},
 		Spec: kamajiv1alpha1.TenantControlPlaneSpec{
 			ControlPlane: kamajiv1alpha1.ControlPlane{
@@ -26,19 +29,14 @@ var _ = Describe("Deploy a TenantControlPlane resource", func() {
 					Replicas: 1,
 				},
 				Ingress: kamajiv1alpha1.IngressSpec{
-					Enabled: true,
+					Enabled: false,
 				},
 				Service: kamajiv1alpha1.ServiceSpec{
-					ServiceType: "NodePort",
+					ServiceType: "ClusterIP",
 				},
 			},
 			NetworkProfile: kamajiv1alpha1.NetworkProfileSpec{
-				Address:       "172.18.0.2",
-				DNSServiceIPs: []string{"10.96.0.10"},
-				Domain:        "clastix.labs",
-				PodCIDR:       "10.244.0.0/16",
-				Port:          31443,
-				ServiceCIDR:   "10.96.0.0/16",
+				Address: "172.18.0.2",
 			},
 			Kubernetes: kamajiv1alpha1.KubernetesSpec{
 				Version: "v1.23.6",
@@ -50,6 +48,7 @@ var _ = Describe("Deploy a TenantControlPlane resource", func() {
 					"ResourceQuota",
 				},
 			},
+			Addons: kamajiv1alpha1.AddonsSpec{},
 		},
 	}
 
@@ -60,6 +59,8 @@ var _ = Describe("Deploy a TenantControlPlane resource", func() {
 
 	// Delete the TenantControlPlane resource after test is finished
 	JustAfterEach(func() {
+		PrintTenantControlPlaneInfo(&tcp)
+		PrintKamajiLogs()
 		Expect(k8sClient.Delete(context.Background(), &tcp)).Should(Succeed())
 	})
 
@@ -75,7 +76,7 @@ var _ = Describe("Deploy a TenantControlPlane resource", func() {
 			}
 
 			// Check if Status field has been created on TenantControlPlane struct
-			if *&tcp.Status.Kubernetes.Version.Status == nil {
+			if tcp.Status.Kubernetes.Version.Status == nil {
 				return ""
 			}
 

e2e/utils_test.go

@@ -0,0 +1,110 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package e2e
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io/ioutil"
+	"os/exec"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/kubernetes"
+
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+)
+
+func GetKindIPAddress() string {
+	ep := &corev1.Endpoints{}
+	Expect(k8sClient.Get(context.Background(), types.NamespacedName{Name: "kubernetes", Namespace: "default"}, ep)).ToNot(HaveOccurred())
+
+	return ep.Subsets[0].Addresses[0].IP
+}
+
+func PrintTenantControlPlaneInfo(tcp *kamajiv1alpha1.TenantControlPlane) {
+	kubectlExec := func(args ...string) {
+		cmd := exec.Command("kubectl")
+
+		var out bytes.Buffer
+		cmd.Stdout = &out
+		cmd.Args = args
+
+		Expect(cmd.Run()).ToNot(HaveOccurred())
+
+		for {
+			line, err := out.ReadString('\n')
+			if err != nil {
+				return
+			}
+
+			_, _ = fmt.Fprint(GinkgoWriter, ">>> ", line)
+		}
+	}
+
+	if CurrentGinkgoTestDescription().Failed {
+		_, _ = fmt.Fprintln(GinkgoWriter, "DEBUG: Tenant Control Plane definition")
+		kubectlExec(
+			fmt.Sprintf("--namespace=%s", tcp.GetNamespace()),
+			"get",
+			"tcp",
+			tcp.GetName(),
+		)
+		_, _ = fmt.Fprintln(GinkgoWriter, "DEBUG: Tenant Control Plane resources")
+		kubectlExec(
+			fmt.Sprintf("--namespace=%s", tcp.GetNamespace()),
+			"get",
+			"svc,deployment,pods,ep,configmap,secrets",
+		)
+		_, _ = fmt.Fprintln(GinkgoWriter, "DEBUG: Tenant Control Plane pods")
+		kubectlExec(
+			fmt.Sprintf("--namespace=%s", tcp.GetNamespace()),
+			"describe",
+			"pods",
+		)
+	}
+}
+
+func PrintKamajiLogs() {
+	if CurrentGinkgoTestDescription().Failed {
+		clientset, err := kubernetes.NewForConfig(cfg)
+		Expect(err).ToNot(HaveOccurred())
+
+		list, err := clientset.CoreV1().Pods("kamaji-system").List(context.Background(), metav1.ListOptions{
+			LabelSelector: "app.kubernetes.io/component=controller-manager",
+		})
+		Expect(err).ToNot(HaveOccurred())
+		Expect(list.Items).To(HaveLen(1))
+
+		request := clientset.CoreV1().Pods("kamaji-system").GetLogs(list.Items[0].GetName(), &corev1.PodLogOptions{
+			Container: "manager",
+			SinceSeconds: func() *int64 {
+				seconds := int64(CurrentGinkgoTestDescription().Duration.Seconds())
+
+				return &seconds
+			}(),
+			Timestamps: true,
+		})
+
+		podLogs, err := request.Stream(context.Background())
+		Expect(err).ToNot(HaveOccurred())
+
+		defer podLogs.Close()
+
+		podBytes, err := ioutil.ReadAll(podLogs)
+		Expect(err).ToNot(HaveOccurred())
+
+		_, _ = fmt.Fprintln(GinkgoWriter, "DEBUG: retrieving Kamaji Pod logs")
+
+		for _, line := range bytes.Split(podBytes, []byte("\n")) {
+			_, _ = fmt.Fprintln(GinkgoWriter, ">>> ", string(line))
+		}
+
+		_, _ = fmt.Fprintln(GinkgoWriter, "DEBUG: end of Kamaji Pod logs")
+	}
+}

e2e/worker_kubeadm_join_test.go

@@ -0,0 +1,170 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package e2e
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"strings"
+	"time"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/testcontainers/testcontainers-go"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/kubernetes/cmd/kubeadm/app/cmd"
+	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/util"
+
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+)
+
+var _ = Describe("starting a kind worker with kubeadm", func() {
+	ctx := context.Background()
+
+	var tcp kamajiv1alpha1.TenantControlPlane
+
+	var workerContainer testcontainers.Container
+
+	var kubeconfigFile *os.File
+
+	JustBeforeEach(func() {
+		tcp = kamajiv1alpha1.TenantControlPlane{
+			TypeMeta: metav1.TypeMeta{},
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "worker-nodes-join",
+				Namespace: "default",
+			},
+			Spec: kamajiv1alpha1.TenantControlPlaneSpec{
+				ControlPlane: kamajiv1alpha1.ControlPlane{
+					Deployment: kamajiv1alpha1.DeploymentSpec{
+						Replicas: 1,
+					},
+					Ingress: kamajiv1alpha1.IngressSpec{
+						Enabled: false,
+					},
+					Service: kamajiv1alpha1.ServiceSpec{
+						ServiceType: "NodePort",
+					},
+				},
+				NetworkProfile: kamajiv1alpha1.NetworkProfileSpec{
+					Address: GetKindIPAddress(),
+					Port:    31443,
+				},
+				Kubernetes: kamajiv1alpha1.KubernetesSpec{
+					Version: "v1.23.6",
+					Kubelet: kamajiv1alpha1.KubeletSpec{
+						CGroupFS: "cgroupfs",
+					},
+					AdmissionControllers: kamajiv1alpha1.AdmissionControllers{
+						"LimitRanger",
+						"ResourceQuota",
+					},
+				},
+				Addons: kamajiv1alpha1.AddonsSpec{},
+			},
+		}
+		Expect(k8sClient.Create(ctx, &tcp)).NotTo(HaveOccurred())
+
+		var err error
+
+		workerContainer, err = testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
+			ContainerRequest: testcontainers.ContainerRequest{
+				Name:       fmt.Sprintf("%s-worker-node", tcp.GetName()),
+				Image:      fmt.Sprintf("kindest/node:%s", tcp.Spec.Kubernetes.Version),
+				Mounts:     testcontainers.ContainerMounts{testcontainers.BindMount("/lib/modules", "/lib/modules")},
+				Networks:   []string{"kind"},
+				Privileged: true,
+			},
+			Started: true,
+		})
+		Expect(err).ToNot(HaveOccurred())
+
+		kubeconfigFile, err = ioutil.TempFile("", "kamaji")
+		Expect(err).ToNot(HaveOccurred())
+	})
+
+	JustAfterEach(func() {
+		PrintTenantControlPlaneInfo(&tcp)
+		PrintKamajiLogs()
+		Expect(workerContainer.Terminate(ctx)).ToNot(HaveOccurred())
+		Expect(k8sClient.Delete(ctx, &tcp)).Should(Succeed())
+		Expect(os.Remove(kubeconfigFile.Name())).ToNot(HaveOccurred())
+	})
+
+	It("should join the Tenant Control Plane cluster", func() {
+		By("waiting for the Tenant Control Plane being ready", func() {
+			Eventually(func() kamajiv1alpha1.KubernetesVersionStatus {
+				err := k8sClient.Get(ctx, types.NamespacedName{Name: tcp.GetName(), Namespace: tcp.GetNamespace()}, &tcp)
+				if err != nil {
+					return ""
+				}
+
+				if tcp.Status.Kubernetes.Version.Status == nil {
+					return ""
+				}
+
+				return *tcp.Status.Kubernetes.Version.Status
+			}, 5*time.Minute, time.Second).Should(Equal(kamajiv1alpha1.VersionReady))
+		})
+
+		By("downloading Tenant Control Plane kubeconfig", func() {
+			secret := &corev1.Secret{}
+
+			Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: tcp.GetNamespace(), Name: tcp.Status.KubeConfig.Admin.SecretName}, secret)).NotTo(HaveOccurred())
+
+			_, err := kubeconfigFile.Write(secret.Data["admin.conf"])
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		var joinCommandBuffer *bytes.Buffer
+
+		By("generating kubeadm join command", func() {
+			joinCommandBuffer = bytes.NewBuffer([]byte(""))
+
+			config, err := clientcmd.BuildConfigFromFlags("", kubeconfigFile.Name())
+			Expect(err).ToNot(HaveOccurred())
+
+			clientset, err := kubernetes.NewForConfig(config)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(cmd.RunCreateToken(joinCommandBuffer, clientset, "", util.DefaultInitConfiguration(), true, "", kubeconfigFile.Name())).ToNot(HaveOccurred())
+		})
+
+		By("executing the command in the worker node", func() {
+			cmds := append(strings.Split(strings.TrimSpace(joinCommandBuffer.String()), " "), "--ignore-preflight-errors", "SystemVerification")
+
+			exitCode, err := workerContainer.Exec(ctx, cmds)
+			Expect(exitCode).To(Equal(0))
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		By("waiting for nodes", func() {
+			config, err := clientcmd.BuildConfigFromFlags("", kubeconfigFile.Name())
+			Expect(err).ToNot(HaveOccurred())
+
+			clientset, err := kubernetes.NewForConfig(config)
+			Expect(err).ToNot(HaveOccurred())
+
+			Eventually(func() string {
+				nodes, err := clientset.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
+				if err != nil {
+					return ""
+				}
+
+				if len(nodes.Items) == 0 {
+					return ""
+				}
+
+				return nodes.Items[0].GetName()
+			}, time.Minute, time.Second).Should(Equal(workerContainer.GetContainerID()[:12]))
+		})
+	})
+})

e2e/worker_tcp_change_port_test.go

@@ -0,0 +1,165 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package e2e
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"time"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/version"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
+
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+)
+
+var _ = Describe("validating kubeconfig", func() {
+	ctx := context.Background()
+
+	var tcp *kamajiv1alpha1.TenantControlPlane
+
+	var kubeconfigFile *os.File
+
+	JustBeforeEach(func() {
+		tcp = &kamajiv1alpha1.TenantControlPlane{
+			TypeMeta: metav1.TypeMeta{},
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "kubeconfig",
+				Namespace: "default",
+			},
+			Spec: kamajiv1alpha1.TenantControlPlaneSpec{
+				ControlPlane: kamajiv1alpha1.ControlPlane{
+					Deployment: kamajiv1alpha1.DeploymentSpec{
+						Replicas: 1,
+					},
+					Ingress: kamajiv1alpha1.IngressSpec{
+						Enabled: false,
+					},
+					Service: kamajiv1alpha1.ServiceSpec{
+						ServiceType: "NodePort",
+					},
+				},
+				NetworkProfile: kamajiv1alpha1.NetworkProfileSpec{
+					Address: GetKindIPAddress(),
+					Port:    30001,
+				},
+				Kubernetes: kamajiv1alpha1.KubernetesSpec{
+					Version: "v1.23.6",
+					Kubelet: kamajiv1alpha1.KubeletSpec{
+						CGroupFS: "cgroupfs",
+					},
+					AdmissionControllers: kamajiv1alpha1.AdmissionControllers{
+						"LimitRanger",
+						"ResourceQuota",
+					},
+				},
+				Addons: kamajiv1alpha1.AddonsSpec{},
+			},
+		}
+		Expect(k8sClient.Create(ctx, tcp)).NotTo(HaveOccurred())
+
+		var err error
+
+		kubeconfigFile, err = ioutil.TempFile("", "kamaji")
+		Expect(err).ToNot(HaveOccurred())
+	})
+
+	JustAfterEach(func() {
+		PrintKamajiLogs()
+		PrintTenantControlPlaneInfo(tcp)
+		Expect(k8sClient.Delete(ctx, tcp)).Should(Succeed())
+		Expect(os.Remove(kubeconfigFile.Name())).ToNot(HaveOccurred())
+	})
+
+	It("return kubernetes version", func() {
+		for _, port := range []int32{30002, 30003, 30004} {
+			Eventually(func() string {
+				By(fmt.Sprintf("ensuring TCP port is set to %d", port), func() {
+					Eventually(func() (err error) {
+						if err = k8sClient.Get(ctx, types.NamespacedName{Namespace: tcp.GetNamespace(), Name: tcp.GetName()}, tcp); err != nil {
+							_, _ = fmt.Fprintln(GinkgoWriter, "DEBUG: cannot retrieve TCP:", err.Error())
+
+							return err
+						}
+
+						tcp.Spec.NetworkProfile.Port = port
+
+						return k8sClient.Update(ctx, tcp)
+					}, time.Minute, 5*time.Second).ShouldNot(HaveOccurred())
+				})
+
+				By("ensuring port change is defined in the TCP status", func() {
+					Eventually(func() int32 {
+						if err := k8sClient.Get(ctx, types.NamespacedName{Namespace: tcp.GetNamespace(), Name: tcp.GetName()}, tcp); err != nil {
+							_, _ = fmt.Fprintln(GinkgoWriter, "DEBUG: cannot retrieve TCP:", err.Error())
+
+							return 0
+						}
+
+						return tcp.Status.Kubernetes.Service.Port
+					}, time.Minute, 5*time.Second).Should(Equal(port))
+				})
+
+				By("ensuring downloading the updated kubeconfig", func() {
+					Eventually(func() (err error) {
+						if err := k8sClient.Get(ctx, types.NamespacedName{Namespace: tcp.GetNamespace(), Name: tcp.GetName()}, tcp); err != nil {
+							_, _ = fmt.Fprintln(GinkgoWriter, "DEBUG: cannot retrieve TCP:", err.Error())
+
+							return err
+						}
+
+						secret := &corev1.Secret{}
+
+						if err = k8sClient.Get(ctx, types.NamespacedName{Namespace: tcp.GetNamespace(), Name: tcp.Status.KubeConfig.Admin.SecretName}, secret); err != nil {
+							_, _ = fmt.Fprintln(GinkgoWriter, "DEBUG: cannot retrieve kubeconfig secret name:", err.Error())
+
+							return err
+						}
+
+						_, err = kubeconfigFile.Write(secret.Data["admin.conf"])
+
+						return err
+					}, time.Minute, 5*time.Second).ShouldNot(HaveOccurred())
+				})
+
+				var version version.Info
+
+				By("retrieving TCP version using the kubeconfig", func() {
+					config, err := clientcmd.BuildConfigFromFlags("", kubeconfigFile.Name())
+					if err != nil {
+						_, _ = fmt.Fprintln(GinkgoWriter, "DEBUG: cannot generate REST configuration:", err.Error())
+
+						return
+					}
+
+					clientset, err := kubernetes.NewForConfig(config)
+					if err != nil {
+						_, _ = fmt.Fprintln(GinkgoWriter, "DEBUG: cannot generate clientset:", err.Error())
+
+						return
+					}
+
+					serverVersion, err := clientset.ServerVersion()
+					if err != nil {
+						_, _ = fmt.Fprintln(GinkgoWriter, "DEBUG: cannot retrieve server version:", err.Error())
+
+						return
+					}
+
+					version = *serverVersion
+				})
+
+				return version.GitVersion
+			}, 3*time.Minute, 5*time.Second).Should(Equal(tcp.Spec.Kubernetes.Version))
+		}
+	})
+})

go.mod

@@ -4,17 +4,20 @@ go 1.18
 
 require (
 	github.com/go-logr/logr v1.2.0
-	github.com/google/uuid v1.1.2
+	github.com/go-sql-driver/mysql v1.6.0
+	github.com/google/uuid v1.3.0
 	github.com/onsi/ginkgo v1.16.5
 	github.com/onsi/gomega v1.17.0
 	github.com/pkg/errors v0.9.1
 	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.10.1
+	github.com/testcontainers/testcontainers-go v0.13.0
 	go.etcd.io/etcd/api/v3 v3.5.1
 	go.etcd.io/etcd/client/v3 v3.5.1
 	google.golang.org/grpc v1.43.0
 	k8s.io/api v0.23.5
 	k8s.io/apimachinery v0.23.5
+	k8s.io/apiserver v0.23.5
 	k8s.io/client-go v0.23.5
 	k8s.io/cluster-bootstrap v0.0.0
 	k8s.io/component-base v0.23.5
@@ -28,6 +31,7 @@ require (
 require (
 	cloud.google.com/go v0.99.0 // indirect
 	cloud.google.com/go/storage v1.18.2 // indirect
+	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
 	github.com/Azure/go-autorest v14.2.0+incompatible // indirect
 	github.com/Azure/go-autorest/autorest v0.11.18 // indirect
 	github.com/Azure/go-autorest/autorest/adal v0.9.13 // indirect
@@ -35,53 +39,88 @@ require (
 	github.com/Azure/go-autorest/logger v0.2.1 // indirect
 	github.com/Azure/go-autorest/tracing v0.6.0 // indirect
 	github.com/Microsoft/go-winio v0.4.17 // indirect
+	github.com/Microsoft/hcsshim v0.8.23 // indirect
+	github.com/PuerkitoBio/purell v1.1.1 // indirect
+	github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/blang/semver v3.5.1+incompatible // indirect
+	github.com/cenkalti/backoff/v4 v4.1.2 // indirect
 	github.com/census-instrumentation/opencensus-proto v0.3.0 // indirect
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
 	github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4 // indirect
 	github.com/cncf/xds/go v0.0.0-20211216145620-d92e9ce0af51 // indirect
+	github.com/containerd/cgroups v1.0.1 // indirect
+	github.com/containerd/containerd v1.5.9 // indirect
 	github.com/coredns/caddy v1.1.0 // indirect
 	github.com/coredns/corefile-migration v1.0.14 // indirect
 	github.com/coreos/go-semver v0.3.0 // indirect
 	github.com/coreos/go-systemd/v22 v22.3.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/docker/distribution v2.7.1+incompatible // indirect
+	github.com/docker/docker v20.10.11+incompatible // indirect
+	github.com/docker/go-connections v0.4.0 // indirect
+	github.com/docker/go-units v0.4.0 // indirect
 	github.com/envoyproxy/go-control-plane v0.10.1 // indirect
 	github.com/envoyproxy/protoc-gen-validate v0.6.2 // indirect
 	github.com/evanphx/json-patch v4.12.0+incompatible // indirect
 	github.com/form3tech-oss/jwt-go v3.2.3+incompatible // indirect
 	github.com/fsnotify/fsnotify v1.5.1 // indirect
+	github.com/go-errors/errors v1.0.1 // indirect
 	github.com/go-logr/zapr v1.2.0 // indirect
+	github.com/go-openapi/jsonpointer v0.19.5 // indirect
+	github.com/go-openapi/jsonreference v0.19.5 // indirect
+	github.com/go-openapi/swag v0.19.14 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/google/btree v1.0.1 // indirect
 	github.com/google/go-cmp v0.5.6 // indirect
 	github.com/google/gofuzz v1.1.0 // indirect
+	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
 	github.com/googleapis/gax-go/v2 v2.1.1 // indirect
 	github.com/googleapis/gnostic v0.5.5 // indirect
 	github.com/googleapis/google-cloud-go-testing v0.0.0-20210719221736-1c9a4c676720 // indirect
+	github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/imdario/mergo v0.3.12 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
+	github.com/lithammer/dedent v1.1.0 // indirect
 	github.com/magiconair/properties v1.8.5 // indirect
+	github.com/mailru/easyjson v0.7.6 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
 	github.com/mitchellh/mapstructure v1.4.3 // indirect
+	github.com/moby/sys/mount v0.2.0 // indirect
+	github.com/moby/sys/mountinfo v0.5.0 // indirect
+	github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
+	github.com/morikuni/aec v1.0.0 // indirect
 	github.com/nxadm/tail v1.4.8 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/image-spec v1.0.2 // indirect
+	github.com/opencontainers/runc v1.0.2 // indirect
 	github.com/pelletier/go-toml v1.9.4 // indirect
+	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_golang v1.12.1 // indirect
 	github.com/prometheus/client_model v0.2.0 // indirect
 	github.com/prometheus/common v0.32.1 // indirect
 	github.com/prometheus/procfs v0.7.3 // indirect
+	github.com/sirupsen/logrus v1.8.1 // indirect
 	github.com/spf13/afero v1.7.0 // indirect
 	github.com/spf13/cast v1.4.1 // indirect
 	github.com/spf13/cobra v1.4.0 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
+	github.com/stretchr/testify v1.7.0 // indirect
 	github.com/subosito/gotenv v1.2.0 // indirect
+	github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca // indirect
 	go.etcd.io/etcd/client/pkg/v3 v3.5.1 // indirect
 	go.opencensus.io v0.23.0 // indirect
+	go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect
 	go.uber.org/atomic v1.7.0 // indirect
 	go.uber.org/multierr v1.6.0 // indirect
 	go.uber.org/zap v1.19.1 // indirect
@@ -100,15 +139,18 @@ require (
 	google.golang.org/protobuf v1.27.1 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/ini.v1 v1.66.2 // indirect
+	gopkg.in/square/go-jose.v2 v2.5.1 // indirect
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
 	k8s.io/apiextensions-apiserver v0.23.5 // indirect
-	k8s.io/apiserver v0.23.5 // indirect
+	k8s.io/cli-runtime v0.23.5 // indirect
 	k8s.io/klog/v2 v2.60.1 // indirect
 	k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 // indirect
 	k8s.io/system-validators v1.6.0 // indirect
 	sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 // indirect
+	sigs.k8s.io/kustomize/api v0.10.1 // indirect
+	sigs.k8s.io/kustomize/kyaml v0.13.0 // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect
 	sigs.k8s.io/yaml v1.3.0 // indirect
 )

helm/kamaji/Chart.yaml

@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.0
+version: 0.1.1
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

helm/kamaji/README.md

@@ -1,36 +1,24 @@
 # kamaji
 
-![Version: 0.1.0](https://img.shields.io/badge/Version-0.1.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.1.0](https://img.shields.io/badge/AppVersion-0.1.0-informational?style=flat-square)
+![Version: 0.1.1](https://img.shields.io/badge/Version-0.1.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.1.0](https://img.shields.io/badge/AppVersion-0.1.0-informational?style=flat-square)
 
 A Kubernetes distribution aimed to build and operate a Managed Kubernetes service with a fraction of operational burde.
 
 **Homepage:** <https://github.com/clastix/kamaji-internal/tree/master/helm/kamaji>
 
-## Installing the Chart
-
-To install the chart with the release name `kamaji`:
-
 ### Pre-requisites
 
-1. Deploy a [multi-tenant Etcd cluster](https://github.com/clastix/kamaji-internal/blob/master/deploy/getting-started-with-kamaji.md#setup-internal-multi-tenant-etcd)
-2. Create the `Secret` containing the Etcd CA cert keypair:
+Kamaji requires a [multi-tenant etcd cluster](https://github.com/clastix/kamaji-internal/blob/master/deploy/getting-started-with-kamaji.md#setup-internal-multi-tenant-etcd) cluster.
+The installation and provisioning processes are already put in place by the Helm Chart starting from v0.1.1 in order to streamline the local test.
 
-```
-kubectl -n kamaji-system create secret generic etcd-certs \
-  --from-file=/path/to/etcd/ca.crt \
-  --from-file=/path/to/etcd/ca.key
-```
+> For production use an externally managed etcd is highly recommended, the etcd addon offered by this chart is not considered production-grade.
 
-3. Create a `Secret` containing the Etcd root user client cert keypair:
-
-```
-kubectl -n kamaji-system create secret tls root-client-certs \
-  --cert=/path/to/etcd/root.pem \
-  --key=/path/to/etcd/root-key.pem
-```
+If you'd like to use an externally managed etcd instance, you can specify the overrides and by setting the value `etcd.deploy=false`.
 
 ### Install Kamaji
 
+To install the chart with the release name `kamaji`:
+
 ```console
 helm upgrade --install --namespace kamaji-system --create-namespace kamaji .
 ```
@@ -55,30 +43,24 @@ Kubernetes: `>=1.18`
 
 | Key | Type | Default | Description |
 |-----|------|---------|-------------|
-| addons.coreDNS.enabled | boolean | `true` | Enabling CoreDNS installation. If the value is not specified, the installation is enabled |
-| addons.kubeProxy.enabled | boolean | `true` | Enabling KubeProxy installation. If the value is not specified, the installation is enabled |
 | affinity | object | `{}` | Kubernetes affinity rules to apply to Kamaji controller pods |
 | configPath | string | `"./kamaji.yaml"` | Configuration file path alternative. (default "./kamaji.yaml") |
-| etcd.caSecret.name | string | `"etcd-certs"` | Name of the secret which contains CA's certificate and private key. (default: "etcd-certs") |
-| etcd.caSecret.namespace | string | `"kamaji-system"` | Namespace of the secret which contains CA's certificate and private key. (default: "kamaji") |
-| etcd.clientSecret.name | string | `"root-client-certs"` | Name of the secret which contains ETCD client certificates. (default: "root-client-certs") |
-| etcd.clientSecret.namespace | string | `"kamaji-system"` | Name of the namespace where the secret which contains ETCD client certificates is. (default: "kamaji") |
 | etcd.compactionInterval | int | `0` | ETCD Compaction interval (e.g. "5m0s"). (default: "0" (disabled)) |
-| etcd.endpoints | string | `"etcd-0.etcd.kamaji-system.svc.cluster.local:2379,etcd-1.etcd.kamaji-system.svc.cluster.local:2379,etcd-2.etcd.kamaji-system.svc.cluster.local:2379"` | (string) Comma-separated list of the endpoints of the etcd cluster's members. |
+| etcd.deploy | bool | `true` | Install an etcd 3.5 with enabled multi-tenancy along with Kamaji |
+| etcd.overrides.caSecret.name | string | `"etcd-certs"` | Name of the secret which contains CA's certificate and private key. (default: "etcd-certs") |
+| etcd.overrides.caSecret.namespace | string | `"kamaji-system"` | Namespace of the secret which contains CA's certificate and private key. (default: "kamaji-system") |
+| etcd.overrides.clientSecret.name | string | `"root-client-certs"` | Name of the secret which contains ETCD client certificates. (default: "root-client-certs") |
+| etcd.overrides.clientSecret.namespace | string | `"kamaji-system"` | Name of the namespace where the secret which contains ETCD client certificates is. (default: "kamaji-system") |
+| etcd.overrides.endpoints | string | `"https://etcd-0.etcd.kamaji-system.svc.cluster.local:2379,https://etcd-1.etcd.kamaji-system.svc.cluster.local:2379,https://etcd-2.etcd.kamaji-system.svc.cluster.local:2379"` | (string) Comma-separated list of the endpoints of the etcd cluster's members. |
+| etcd.serviceAccount.create | bool | `true` | Create a ServiceAccount, required to install and provision the etcd backing storage (default: true) |
+| etcd.serviceAccount.name | string | `""` | Define the ServiceAccount name to use during the setup and provision of the etcd backing storage (default: "") |
 | extraArgs | list | `[]` | A list of extra arguments to add to the kamaji controller default ones |
 | fullnameOverride | string | `""` |  |
 | healthProbeBindAddress | string | `":8081"` | The address the probe endpoint binds to. (default ":8081") |
-| image.pullPolicy | string | `"IfNotPresent"` |  |
+| image.pullPolicy | string | `"Always"` |  |
 | image.repository | string | `"clastix/kamaji"` | The container image of the Kamaji controller. |
 | image.tag | string | `"latest"` |  |
 | imagePullSecrets | list | `[]` |  |
-| ingress.annotations | object | `{}` |  |
-| ingress.className | string | `""` | Name of the ingress class to route through this controller. |
-| ingress.enabled | bool | `false` | Whether to expose the Kamaji controller through an Ingress. |
-| ingress.hosts[0].host | string | `"chart-example.local"` |  |
-| ingress.hosts[0].paths[0].path | string | `"/"` |  |
-| ingress.hosts[0].paths[0].pathType | string | `"ImplementationSpecific"` |  |
-| ingress.tls | list | `[]` |  |
 | livenessProbe | object | `{"httpGet":{"path":"/healthz","port":"healthcheck"},"initialDelaySeconds":15,"periodSeconds":20}` | The livenessProbe for the controller container |
 | loggingDevel.enable | bool | `false` | (string) Development Mode defaults(encoder=consoleEncoder,logLevel=Debug,stackTraceLevel=Warn). Production Mode defaults(encoder=jsonEncoder,logLevel=Info,stackTraceLevel=Error) (default false) |
 | metricsBindAddress | string | `":8080"` | (string) The address the metric endpoint binds to. (default ":8080") |

helm/kamaji/README.md.gotmpl

@@ -7,31 +7,19 @@
 
 {{ template "chart.homepageLine" . }}
 
-## Installing the Chart
-
-To install the chart with the release name `kamaji`:
-
 ### Pre-requisites
 
-1. Deploy a [multi-tenant Etcd cluster](https://github.com/clastix/kamaji-internal/blob/master/deploy/getting-started-with-kamaji.md#setup-internal-multi-tenant-etcd)
-2. Create the `Secret` containing the Etcd CA cert keypair:
+Kamaji requires a [multi-tenant etcd cluster](https://github.com/clastix/kamaji-internal/blob/master/deploy/getting-started-with-kamaji.md#setup-internal-multi-tenant-etcd) cluster.
+The installation and provisioning processes are already put in place by the Helm Chart starting from v0.1.1 in order to streamline the local test.
 
-```
-kubectl -n kamaji-system create secret generic etcd-certs \
-  --from-file=/path/to/etcd/ca.crt \
-  --from-file=/path/to/etcd/ca.key
-```
+> For production use an externally managed etcd is highly recommended, the etcd addon offered by this chart is not considered production-grade.
 
-3. Create a `Secret` containing the Etcd root user client cert keypair:
-
-```
-kubectl -n kamaji-system create secret tls root-client-certs \
-  --cert=/path/to/etcd/root.pem \
-  --key=/path/to/etcd/root-key.pem
-```
+If you'd like to use an externally managed etcd instance, you can specify the overrides and by setting the value `etcd.deploy=false`.
 
 ### Install Kamaji
 
+To install the chart with the release name `kamaji`:
+
 ```console
 helm upgrade --install --namespace kamaji-system --create-namespace kamaji .
 ```

helm/kamaji/templates/NOTES.txt

@@ -1,22 +1,5 @@
-1. Get the application URL by running these commands:
-{{- if .Values.ingress.enabled }}
-{{- range $host := .Values.ingress.hosts }}
-  {{- range .paths }}
-  http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }}
-  {{- end }}
-{{- end }}
-{{- else if contains "NodePort" .Values.service.type }}
-  export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "kamaji.fullname" . }})
-  export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
-  echo http://$NODE_IP:$NODE_PORT
-{{- else if contains "LoadBalancer" .Values.service.type }}
-     NOTE: It may take a few minutes for the LoadBalancer IP to be available.
-           You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "kamaji.fullname" . }}'
-  export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "kamaji.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}")
-  echo http://$SERVICE_IP:{{ .Values.service.port }}
-{{- else if contains "ClusterIP" .Values.service.type }}
-  export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "kamaji.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
-  export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}")
-  echo "Visit http://127.0.0.1:8080 to use your application"
-  kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT
-{{- end }}
+List the available CRDs installed by Kamaji:
+  kubectl get customresourcedefinitions.apiextensions.k8s.io
+
+List all the Tenant Control Plane resources deployed in your cluster:
+  kubectl get tenantcontrolplanes.kamaji.clastix.io --all-namespaces

helm/kamaji/templates/_helpers_etcd.tpl

@@ -0,0 +1,120 @@
+{{/*
+Create a default fully qualified etcd name.
+*/}}
+{{- define "etcd.fullname" -}}
+{{- printf "etcd" }}
+{{- end }}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "etcd.serviceAccountName" -}}
+{{- if .Values.etcd.serviceAccount.create }}
+{{- default (include "etcd.fullname" .) .Values.etcd.serviceAccount.name }}
+{{- else }}
+{{- default "default" .Values.etcd.serviceAccount.name }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create the name of the Service to use
+*/}}
+{{- define "etcd.serviceName" -}}
+{{- printf "%s" (include "etcd.fullname" .) | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "etcd.labels" -}}
+app.kubernetes.io/name: {{ include "kamaji.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+app.kubernetes.io/components: etcd
+{{- end }}
+
+{{/*
+Selector labels.
+*/}}
+{{- define "etcd.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "kamaji.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+app.kubernetes.io/component: etcd
+{{- end }}
+
+{{/*
+Name of the etcd CA secret.
+*/}}
+{{- define "etcd.caSecretName" }}
+{{- if .Values.etcd.deploy }}
+{{- printf "%s-%s" (include "etcd.fullname" .) "certs" | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- required "A valid .Values.etcd.overrides.caSecret.name required!" .Values.etcd.overrides.caSecret.name }}
+{{- end }}
+{{- end }}
+
+{{/*
+Namespace of the etcd CA secret.
+*/}}
+{{- define "etcd.caSecretNamespace" }}
+{{- if .Values.etcd.deploy }}
+{{- .Release.Namespace }}
+{{- else }}
+{{- required "A valid .Values.etcd.overrides.caSecret.namespace required!" .Values.etcd.overrides.caSecret.namespace }}
+{{- end }}
+{{- end }}
+
+{{/*
+Name of the certificate signing requests for the certificates required by etcd.
+*/}}
+{{- define "etcd.csrConfigMapName" }}
+{{- printf "%s-csr" (include "etcd.fullname" .) }}
+{{- end }}
+
+{{/*
+Name of the etcd root-client secret.
+*/}}
+{{- define "etcd.clientSecretName" }}
+{{- if .Values.etcd.deploy }}
+{{- printf "root-client-certs" }}
+{{- else }}
+{{- required "A valid .Values.etcd.overrides.clientSecret.name required!" .Values.etcd.overrides.clientSecret.name }}
+{{- end }}
+{{- end }}
+
+{{/*
+Namespace of the etcd root-client secret.
+*/}}
+{{- define "etcd.clientSecretNamespace" }}
+{{- if .Values.etcd.deploy }}
+{{- .Release.Namespace }}
+{{- else }}
+{{- required "A valid .Values.etcd.overrides.clientSecret.namespace required!" .Values.etcd.overrides.clientSecret.namespace }}
+{{- end }}
+{{- end }}
+
+{{/*
+List the declared etcd endpoints, using the overrides in case of unmanaged etcd.
+*/}}
+{{- define "etcd.endpoints" }}
+{{- if .Values.etcd.deploy }}
+{{- range $count := until 3 -}}
+    {{- printf "https://%s-%d.%s.%s.svc.cluster.local:2379" "etcd" $count ( include "etcd.serviceName" . ) $.Release.Namespace -}}
+    {{- if lt $count  ( sub 3 1 ) -}}
+      {{- printf "," -}}
+    {{- end -}}
+{{- end }}
+{{- else }}
+{{- required "A valid .Values.etcd.overrides.endpoints required!" .Values.etcd.overrides.endpoints }}
+{{- end }}
+{{- end }}
+
+{{/*
+Retrieve the current Kubernetes version to launch a kubectl container with the minimum version skew possible.
+*/}}
+{{- define "etcd.jobsTagKubeVersion" -}}
+{{- if contains "-eks-" .Capabilities.KubeVersion.GitVersion }}
+{{- print "v" .Capabilities.KubeVersion.Major "." (.Capabilities.KubeVersion.Minor | replace "+" "") -}}
+{{- else }}
+{{- print "v" .Capabilities.KubeVersion.Major "." .Capabilities.KubeVersion.Minor -}}
+{{- end }}
+{{- end }}

helm/kamaji/templates/controller.yaml

@@ -40,12 +40,12 @@ spec:
           protocol: TCP
       - args:
         - --config-file={{ .Values.configPath }}
-        - --etcd-ca-secret-name={{ .Values.etcd.caSecret.name }}
-        - --etcd-ca-secret-namespace={{ .Values.etcd.caSecret.namespace }}
-        - --etcd-client-secret-name={{ .Values.etcd.clientSecret.name }}
-        - --etcd-client-secret-namespace={{ .Values.etcd.clientSecret.namespace }}
+        - --etcd-ca-secret-name={{ include "etcd.caSecretName" . }}
+        - --etcd-ca-secret-namespace={{ include "etcd.caSecretNamespace" . }}
+        - --etcd-client-secret-name={{ include "etcd.clientSecretName" . }}
+        - --etcd-client-secret-namespace={{ include "etcd.clientSecretNamespace" . }}
         - --etcd-compaction-interval={{ .Values.etcd.compactionInterval }}
-        - --etcd-endpoints={{ .Values.etcd.endpoints }}
+        - --etcd-endpoints={{ include "etcd.endpoints" . }}
         - --health-probe-bind-address={{ .Values.healthProbeBindAddress }}
         - --leader-elect
         - --metrics-bind-address={{ .Values.metricsBindAddress }}

helm/kamaji/templates/etcd_cm.yaml

@@ -0,0 +1,94 @@
+{{- if .Values.etcd.deploy }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  labels:
+    {{- include "etcd.labels" . | nindent 4 }}
+  name: {{ include "etcd.csrConfigMapName" . }}
+  namespace: {{ .Release.Namespace }}
+data:
+  ca-csr.json: |-
+    {
+      "CN": "Clastix CA",
+      "key": {
+        "algo": "rsa",
+        "size": 2048
+      },
+      "names": [
+        {
+          "C": "IT",
+          "ST": "Italy",
+          "L": "Milan"
+        }
+      ]
+    }
+  config.json: |-
+    {
+      "signing": {
+        "default": {
+          "expiry": "8760h"
+        },
+        "profiles": {
+          "server-authentication": {
+            "usages": ["signing", "key encipherment", "server auth"],
+            "expiry": "8760h"
+          },
+          "client-authentication": {
+            "usages": ["signing", "key encipherment", "client auth"],
+            "expiry": "8760h"
+          },
+          "peer-authentication": {
+            "usages": ["signing", "key encipherment", "server auth", "client auth"],
+            "expiry": "8760h"
+          }
+        }
+      }
+    }
+  server-csr.json: |-
+    {
+      "CN": "etcd",
+      "key": {
+        "algo": "rsa",
+        "size": 2048
+      },
+      "hosts": [
+{{- range $count := until 3 -}}
+        {{ printf "\"etcd-%d.%s.%s.svc.cluster.local\"," $count (include "etcd.serviceName" .) $.Release.Namespace }}
+{{- end }}
+        "etcd-server.{{ .Release.Namespace }}.svc.cluster.local",
+        "etcd-server.{{ .Release.Namespace }}.svc",
+        "etcd-server",
+        "127.0.0.1"
+      ]
+    }
+  peer-csr.json: |-
+    {
+      "CN": "etcd",
+      "key": {
+        "algo": "rsa",
+        "size": 2048
+      },
+      "hosts": [
+{{- range $count := until 3 -}}
+        {{ printf "\"etcd-%d\"," $count }}
+        {{ printf "\"etcd-%d.%s\"," $count (include "etcd.serviceName" .) }}
+        {{ printf "\"etcd-%d.%s.%s.svc\"," $count (include "etcd.serviceName" .) $.Release.Namespace }}
+        {{ printf "\"etcd-%d.%s.%s.svc.cluster.local\"," $count (include "etcd.serviceName" .) $.Release.Namespace }}
+{{- end }}
+        "127.0.0.1"
+      ]
+    }
+  root-client-csr.json: |-
+    {
+      "CN": "root",
+      "key": {
+        "algo": "rsa",
+        "size": 2048
+      },
+      "names": [
+        {
+          "O": "system:masters"
+        }
+      ]
+    }
+{{- end }}

helm/kamaji/templates/etcd_job_postdelete.yaml

@@ -0,0 +1,31 @@
+{{- if .Values.etcd.deploy }}
+apiVersion: batch/v1
+kind: Job
+metadata:
+  labels:
+    {{- include "etcd.labels" . | nindent 4 }}
+  annotations:
+    "helm.sh/hook": pre-delete
+    "helm.sh/hook-weight": "-5"
+    "helm.sh/hook-delete-policy": hook-succeeded
+  name: "{{ .Release.Name }}-etcd-teardown"
+  namespace: {{ .Release.Namespace }}
+spec:
+  template:
+    metadata:
+      name: "{{ .Release.Name }}"
+    spec:
+      serviceAccountName: {{ include "etcd.serviceAccountName" . }}
+      restartPolicy: Never
+      containers:
+        - name: kubectl
+          image: {{ printf "clastix/kubectl:%s" (include "etcd.jobsTagKubeVersion" .) }}
+          command:
+            - kubectl
+            - --namespace={{ .Release.Namespace }}
+            - delete
+            - secret
+            - --ignore-not-found=true
+            - {{ include "etcd.caSecretName" . }}
+            - {{ include "etcd.clientSecretName" . }}
+{{- end }}

helm/kamaji/templates/etcd_job_postinstall.yaml

@@ -0,0 +1,91 @@
+{{- if .Values.etcd.deploy }}
+apiVersion: batch/v1
+kind: Job
+metadata:
+  labels:
+    {{- include "etcd.labels" . | nindent 4 }}
+  annotations:
+    "helm.sh/hook": post-install
+    "helm.sh/hook-weight": "-5"
+    "helm.sh/hook-delete-policy": hook-succeeded
+  name: "{{ .Release.Name }}-etcd-setup"
+  namespace: {{ .Release.Namespace }}
+spec:
+  template:
+    metadata:
+      name: "{{ .Release.Name }}"
+    spec:
+      serviceAccountName: {{ include "etcd.serviceAccountName" . }}
+      restartPolicy: Never
+      initContainers:
+        - name: cfssl
+          image: cfssl/cfssl:latest
+          command:
+            - bash
+            - -c
+            - |-
+              cfssl gencert -initca /csr/ca-csr.json | cfssljson -bare /certs/ca &&
+              mv /certs/ca.pem /certs/ca.crt && mv /certs/ca-key.pem /certs/ca.key &&
+              cfssl gencert -ca=/certs/ca.crt -ca-key=/certs/ca.key -config=/csr/config.json -profile=peer-authentication /csr/peer-csr.json | cfssljson -bare /certs/peer &&
+              cfssl gencert -ca=/certs/ca.crt -ca-key=/certs/ca.key -config=/csr/config.json -profile=peer-authentication /csr/server-csr.json | cfssljson -bare /certs/server &&
+              cfssl gencert -ca=/certs/ca.crt -ca-key=/certs/ca.key -config=/csr/config.json -profile=client-authentication /csr/root-client-csr.json | cfssljson -bare /certs/root-client
+          volumeMounts:
+            - mountPath: /certs
+              name: certs
+            - mountPath: /csr
+              name: csr
+        - name: kubectl
+          image: {{ printf "clastix/kubectl:%s" (include "etcd.jobsTagKubeVersion" .) }}
+          command:
+            - sh
+            - -c
+            - |-
+              kubectl --namespace={{ .Release.Namespace }} delete secret --ignore-not-found=true {{ include "etcd.caSecretName" . }} {{ include "etcd.clientSecretName" . }} &&
+              kubectl --namespace={{ .Release.Namespace }} create secret generic {{ include "etcd.caSecretName" . }} --from-file=/certs/ca.crt --from-file=/certs/ca.key --from-file=/certs/peer-key.pem --from-file=/certs/peer.pem --from-file=/certs/server-key.pem --from-file=/certs/server.pem &&
+              kubectl --namespace={{ .Release.Namespace }} create secret tls {{ include "etcd.clientSecretName" . }} --key=/certs/root-client-key.pem --cert=/certs/root-client.pem &&
+              kubectl --namespace={{ .Release.Namespace }} rollout status sts/etcd --timeout=120s
+          volumeMounts:
+            - mountPath: /certs
+              name: certs
+      containers:
+        - command:
+            - bash
+            - -c
+            - |-
+              etcdctl member list -w table &&
+              etcdctl user add --no-password=true root &&
+              etcdctl role add root &&
+              etcdctl user grant-role root root &&
+              etcdctl auth enable
+          env:
+            - name: ETCDCTL_ENDPOINTS
+              value: https://etcd-0.{{ include "etcd.serviceName" . }}.{{ .Release.Namespace }}.svc.cluster.local:2379
+            - name: ETCDCTL_CACERT
+              value: /opt/certs/ca/ca.crt
+            - name: ETCDCTL_CERT
+              value: /opt/certs/root-certs/tls.crt
+            - name: ETCDCTL_KEY
+              value: /opt/certs/root-certs/tls.key
+          image: quay.io/coreos/etcd:v3.5.1
+          imagePullPolicy: Always
+          name: etcd-client
+          volumeMounts:
+            - name: root-certs
+              mountPath: /opt/certs/root-certs
+            - name: certs
+              mountPath: /opt/certs/ca
+      securityContext:
+        runAsUser: 1000
+        runAsGroup: 1000
+        fsGroup: 1000
+      volumes:
+        - name: root-certs
+          secret:
+            secretName: {{ include "etcd.clientSecretName" . }}
+            optional: true
+        - name: csr
+          configMap:
+            name: {{ include "etcd.csrConfigMapName" . }}
+        - name: certs
+          emptyDir: {}
+{{- end }}

helm/kamaji/templates/etcd_rbac.yaml

@@ -0,0 +1,49 @@
+{{- if .Values.etcd.deploy }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  labels:
+    {{- include "etcd.labels" . | nindent 4 }}
+  name: etcd-gen-certs-role
+  namespace: {{ .Release.Namespace }}
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - secrets
+    verbs:
+      - delete
+    resourceNames:
+      - {{ include "etcd.caSecretName" . }}
+      - {{ include "etcd.clientSecretName" . }}
+  - apiGroups:
+      - ""
+    resources:
+      - secrets
+    verbs:
+      - create
+  - apiGroups:
+      - apps
+    resources:
+      - statefulsets
+    verbs:
+      - get
+      - list
+      - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  labels:
+    {{- include "etcd.labels" . | nindent 4 }}
+  name: etcd-gen-certs-rolebiding
+  namespace: {{ .Release.Namespace }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: etcd-gen-certs-role
+subjects:
+  - kind: ServiceAccount
+    name: {{ include "etcd.serviceAccountName" . }}
+    namespace: {{ .Release.Namespace }}
+{{- end }}

helm/kamaji/templates/etcd_sa.yaml

@@ -0,0 +1,9 @@
+{{- if .Values.etcd.deploy }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    {{- include "etcd.labels" . | nindent 4 }}
+  name: {{ include "etcd.serviceAccountName" . }}
+  namespace: {{ .Release.Namespace }}
+{{- end }}

helm/kamaji/templates/etcd_service.yaml

@@ -0,0 +1,18 @@
+{{- if .Values.etcd.deploy }}
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    {{- include "etcd.labels" . | nindent 4 }}
+  name: {{ include "etcd.serviceName" . }}
+  namespace: {{ .Release.Namespace }}
+spec:
+  clusterIP: None
+  ports:
+    - port: 2379
+      name: client
+    - port: 2380
+      name: peer
+  selector:
+    {{- include "etcd.selectorLabels" . | nindent 4 }}
+{{- end }}

helm/kamaji/templates/etcd_sts.yaml

@@ -0,0 +1,97 @@
+{{- if .Values.etcd.deploy }}
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  labels:
+    {{- include "etcd.labels" . | nindent 4 }}
+  name: {{ include "etcd.fullname" . }}
+  namespace: {{ .Release.Namespace }}
+spec:
+  serviceName: {{ include "etcd.serviceName" . }}
+  selector:
+    matchLabels:
+      {{- include "etcd.selectorLabels" . | nindent 6 }}
+  replicas: 3
+  template:
+    metadata:
+      name: etcd
+      labels:
+        {{- include "etcd.selectorLabels" . | nindent 8 }}
+    spec:
+      volumes:
+        - name: certs
+          secret:
+            secretName: {{ include "etcd.caSecretName" . }}
+      containers:
+        - name: etcd
+          image: quay.io/coreos/etcd:v3.5.1
+          ports:
+            - containerPort: 2379
+              name: client
+            - containerPort: 2380
+              name: peer
+          volumeMounts:
+            - name: data
+              mountPath: /var/run/etcd
+            - name: certs
+              mountPath: /etc/etcd/pki
+          command:
+            - etcd
+            - --data-dir=/var/run/etcd
+            - --name=$(POD_NAME)
+            - --initial-cluster-state=new
+            - --initial-cluster=etcd-0=https://etcd-0.etcd.$(POD_NAMESPACE).svc.cluster.local:2380,etcd-1=https://etcd-1.etcd.$(POD_NAMESPACE).svc.cluster.local:2380,etcd-2=https://etcd-2.etcd.$(POD_NAMESPACE).svc.cluster.local:2380
+            - --initial-advertise-peer-urls=https://$(POD_NAME).etcd.$(POD_NAMESPACE).svc.cluster.local:2380
+            - --initial-cluster-token=kamaji
+            - --listen-client-urls=https://0.0.0.0:2379
+            - --advertise-client-urls={{ include "etcd.endpoints" . }}
+            - --client-cert-auth=true
+            - --trusted-ca-file=/etc/etcd/pki/ca.crt
+            - --cert-file=/etc/etcd/pki/server.pem
+            - --key-file=/etc/etcd/pki/server-key.pem
+            - --listen-peer-urls=https://0.0.0.0:2380
+            - --peer-client-cert-auth=true
+            - --peer-trusted-ca-file=/etc/etcd/pki/ca.crt
+            - --peer-cert-file=/etc/etcd/pki/peer.pem
+            - --peer-key-file=/etc/etcd/pki/peer-key.pem
+            - --auto-compaction-mode=periodic
+            - --auto-compaction-retention=5m
+            - --v=8
+          env:
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+          livenessProbe:
+            failureThreshold: 8
+            httpGet:
+              host: 127.0.0.1
+              path: /health
+              port: 2381
+              scheme: HTTP
+            initialDelaySeconds: 10
+            periodSeconds: 10
+            timeoutSeconds: 15
+          startupProbe:
+            failureThreshold: 24
+            httpGet:
+              host: 127.0.0.1
+              path: /health
+              port: 2381
+              scheme: HTTP
+            initialDelaySeconds: 10
+            periodSeconds: 10
+            timeoutSeconds: 15
+  volumeClaimTemplates:
+    - metadata:
+        name: data
+      spec:
+        accessModes: ["ReadWriteOnce"]
+        resources:
+          requests:
+            storage: 8Gi
+{{- end }}

helm/kamaji/templates/ingress.yaml

@@ -1,61 +0,0 @@
-{{- if .Values.ingress.enabled -}}
-{{- $fullName := include "kamaji.fullname" . -}}
-{{- $svcPort := .Values.service.port -}}
-{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }}
-  {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }}
-  {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}}
-  {{- end }}
-{{- end }}
-{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}}
-apiVersion: networking.k8s.io/v1
-{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}}
-apiVersion: networking.k8s.io/v1beta1
-{{- else -}}
-apiVersion: extensions/v1beta1
-{{- end }}
-kind: Ingress
-metadata:
-  name: {{ $fullName }}
-  labels:
-    {{- include "kamaji.labels" . | nindent 4 }}
-  {{- with .Values.ingress.annotations }}
-  annotations:
-    {{- toYaml . | nindent 4 }}
-  {{- end }}
-spec:
-  {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }}
-  ingressClassName: {{ .Values.ingress.className }}
-  {{- end }}
-  {{- if .Values.ingress.tls }}
-  tls:
-    {{- range .Values.ingress.tls }}
-    - hosts:
-        {{- range .hosts }}
-        - {{ . | quote }}
-        {{- end }}
-      secretName: {{ .secretName }}
-    {{- end }}
-  {{- end }}
-  rules:
-    {{- range .Values.ingress.hosts }}
-    - host: {{ .host | quote }}
-      http:
-        paths:
-          {{- range .paths }}
-          - path: {{ .path }}
-            {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }}
-            pathType: {{ .pathType }}
-            {{- end }}
-            backend:
-              {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }}
-              service:
-                name: {{ $fullName }}
-                port:
-                  number: {{ $svcPort }}
-              {{- else }}
-              serviceName: {{ $fullName }}
-              servicePort: {{ $svcPort }}
-              {{- end }}
-          {{- end }}
-    {{- end }}
-{{- end }}

helm/kamaji/values.sample.yaml

@@ -1,2 +1,2 @@
 etcd:
-  endpoints: "etcd-0.etcd.kamaji-system.svc.cluster.local:2379,etcd-1.etcd.kamaji-system.svc.cluster.local:2379,etcd-2.etcd.kamaji-system.svc.cluster.local:2379"
+  endpoints: "https://etcd-0.etcd.kamaji-system.svc.cluster.local:2379,https://etcd-1.etcd.kamaji-system.svc.cluster.local:2379,https://etcd-2.etcd.kamaji-system.svc.cluster.local:2379"

helm/kamaji/values.yaml

@@ -8,7 +8,7 @@ replicaCount: 1
 image:
   # -- The container image of the Kamaji controller.
   repository: clastix/kamaji
-  pullPolicy: IfNotPresent
+  pullPolicy: Always
   # Overrides the image tag whose default is the chart appVersion.
   tag: latest
 
@@ -19,24 +19,29 @@ extraArgs: []
 configPath: "./kamaji.yaml"
 
 etcd:
-  caSecret:
-    # -- Name of the secret which contains CA's certificate and private key. (default: "etcd-certs")
-    name: etcd-certs
-    # -- Namespace of the secret which contains CA's certificate and private key. (default: "kamaji")
-    namespace: kamaji-system
-
-  clientSecret:
-    # -- Name of the secret which contains ETCD client certificates. (default: "root-client-certs")
-    name: root-client-certs
-    # -- Name of the namespace where the secret which contains ETCD client certificates is. (default: "kamaji")
-    namespace: kamaji-system
-
+  # -- Install an etcd 3.5 with enabled multi-tenancy along with Kamaji
+  deploy: true
+  serviceAccount:
+    # -- Create a ServiceAccount, required to install and provision the etcd backing storage (default: true)
+    create: true
+    # -- Define the ServiceAccount name to use during the setup and provision of the etcd backing storage (default: "")
+    name: ""
+  overrides:
+    caSecret:
+      # -- Name of the secret which contains CA's certificate and private key. (default: "etcd-certs")
+      name: etcd-certs
+      # -- Namespace of the secret which contains CA's certificate and private key. (default: "kamaji-system")
+      namespace: kamaji-system
+    clientSecret:
+      # -- Name of the secret which contains ETCD client certificates. (default: "root-client-certs")
+      name: root-client-certs
+      # -- Name of the namespace where the secret which contains ETCD client certificates is. (default: "kamaji-system")
+      namespace: kamaji-system
+    # -- (string) Comma-separated list of the endpoints of the etcd cluster's members.
+    endpoints: "https://etcd-0.etcd.kamaji-system.svc.cluster.local:2379,https://etcd-1.etcd.kamaji-system.svc.cluster.local:2379,https://etcd-2.etcd.kamaji-system.svc.cluster.local:2379"
   # -- ETCD Compaction interval (e.g. "5m0s"). (default: "0" (disabled))
   compactionInterval: 0
 
-  # -- (string) Comma-separated list of the endpoints of the etcd cluster's members.
-  endpoints: "etcd-0.etcd.kamaji-system.svc.cluster.local:2379,etcd-1.etcd.kamaji-system.svc.cluster.local:2379,etcd-2.etcd.kamaji-system.svc.cluster.local:2379"
-
 # -- The address the probe endpoint binds to. (default ":8081")
 healthProbeBindAddress: ":8081"
 
@@ -93,24 +98,6 @@ service:
   type: ClusterIP
   port: 8443
 
-ingress:
-  # -- Whether to expose the Kamaji controller through an Ingress.
-  enabled: false
-  # -- Name of the ingress class to route through this controller.
-  className: ""
-  annotations: {}
-    # kubernetes.io/ingress.class: nginx
-    # kubernetes.io/tls-acme: "true"
-  hosts:
-    - host: chart-example.local
-      paths:
-        - path: /
-          pathType: ImplementationSpecific
-  tls: []
-  #  - secretName: chart-example-tls
-  #    hosts:
-  #      - chart-example.local
-
 resources:
   limits:
     cpu: 200m
@@ -134,10 +121,3 @@ temporaryDirectoryPath: "/tmp/kamaji"
 loggingDevel:
   # -- (string) Development Mode defaults(encoder=consoleEncoder,logLevel=Debug,stackTraceLevel=Warn). Production Mode defaults(encoder=jsonEncoder,logLevel=Info,stackTraceLevel=Error) (default false)
   enable: false
-
-# -- Kubernetes Addons 
-addons:
-  coreDNS:
-    enabled: true
-  kubeProxy:
-    enabled: true

internal/builders/controlplane/deployment.go

@@ -0,0 +1,731 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package controlplane
+
+import (
+	"fmt"
+	"path"
+	"strconv"
+	"strings"
+
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3"
+	"k8s.io/kubernetes/cmd/kubeadm/app/constants"
+	"k8s.io/utils/pointer"
+
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+	"github.com/clastix/kamaji/internal/types"
+	"github.com/clastix/kamaji/internal/utilities"
+)
+
+type orderedIndex int
+
+const (
+	apiServerIndex orderedIndex = iota
+	schedulerIndex
+	controllerManagerIndex
+	kineIndex
+)
+
+const (
+	etcKubernetesPKIVolume orderedIndex = iota
+	etcCACertificates
+	etcSSLCerts
+	usrShareCACertificates
+	usrLocalShareCACertificates
+	schedulerKubeconfig
+	controllerManagerKubeconfig
+	kineConfig
+)
+
+const (
+	apiServerFlagsAnnotation = "kube-apiserver.kamaji.clastix.io/args"
+	kineVolumeName           = "kine-config"
+)
+
+type Deployment struct {
+	Address                string
+	ETCDEndpoints          []string
+	ETCDCompactionInterval string
+	ETCDStorageType        types.ETCDStorageType
+	KineContainerImage     string
+}
+
+func (d *Deployment) SetContainers(podSpec *corev1.PodSpec, tcp *kamajiv1alpha1.TenantControlPlane, address string) {
+	d.buildKubeAPIServer(podSpec, tcp, address)
+	d.BuildScheduler(podSpec, tcp)
+	d.buildControllerManager(podSpec, tcp)
+	d.buildKine(podSpec, tcp)
+}
+
+func (d *Deployment) SetStrategy(deployment *appsv1.DeploymentSpec) {
+	maxSurge := intstr.FromString("100%")
+
+	maxUnavailable := intstr.FromInt(0)
+
+	deployment.Strategy = appsv1.DeploymentStrategy{
+		Type: appsv1.RollingUpdateDeploymentStrategyType,
+		RollingUpdate: &appsv1.RollingUpdateDeployment{
+			MaxUnavailable: &maxUnavailable,
+			MaxSurge:       &maxSurge,
+		},
+	}
+}
+
+func (d *Deployment) SetVolumes(podSpec *corev1.PodSpec, tcp *kamajiv1alpha1.TenantControlPlane) {
+	for _, fn := range []func(*corev1.PodSpec, *kamajiv1alpha1.TenantControlPlane){
+		d.buildPKIVolume,
+		d.buildCAVolume,
+		d.buildSSLCertsVolume,
+		d.buildShareCAVolume,
+		d.buildLocalShareCAVolume,
+		d.buildSchedulerVolume,
+		d.buildControllerManagerVolume,
+		d.buildKineVolume,
+	} {
+		fn(podSpec, tcp)
+	}
+}
+
+func (d *Deployment) buildPKIVolume(podSpec *corev1.PodSpec, tcp *kamajiv1alpha1.TenantControlPlane) {
+	if index := int(etcKubernetesPKIVolume) + 1; len(podSpec.Volumes) < index {
+		podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{})
+	}
+
+	sources := []corev1.VolumeProjection{
+		{
+			Secret: d.secretProjection(tcp.Status.Certificates.APIServer.SecretName, constants.APIServerCertName, constants.APIServerKeyName),
+		},
+		{
+			Secret: d.secretProjection(tcp.Status.Certificates.CA.SecretName, constants.CACertName, constants.CAKeyName),
+		},
+		{
+			Secret: d.secretProjection(tcp.Status.Certificates.APIServerKubeletClient.SecretName, constants.APIServerKubeletClientCertName, constants.APIServerKubeletClientKeyName),
+		},
+		{
+			Secret: d.secretProjection(tcp.Status.Certificates.FrontProxyCA.SecretName, constants.FrontProxyCACertName, constants.FrontProxyCAKeyName),
+		},
+		{
+			Secret: d.secretProjection(tcp.Status.Certificates.FrontProxyClient.SecretName, constants.FrontProxyClientCertName, constants.FrontProxyClientKeyName),
+		},
+		{
+			Secret: d.secretProjection(tcp.Status.Certificates.SA.SecretName, constants.ServiceAccountPublicKeyName, constants.ServiceAccountPrivateKeyName),
+		},
+	}
+
+	if d.ETCDStorageType == types.ETCD {
+		sources = append(sources, corev1.VolumeProjection{
+			Secret: d.secretProjection(tcp.Status.Certificates.ETCD.APIServer.SecretName, constants.APIServerEtcdClientCertName, constants.APIServerEtcdClientKeyName),
+		})
+		sources = append(sources, corev1.VolumeProjection{
+			Secret: &corev1.SecretProjection{
+				LocalObjectReference: corev1.LocalObjectReference{
+					Name: tcp.Status.Certificates.ETCD.CA.SecretName,
+				},
+				Items: []corev1.KeyToPath{
+					{
+						Key:  constants.CACertName,
+						Path: constants.EtcdCACertName,
+					},
+				},
+			},
+		})
+	}
+
+	podSpec.Volumes[etcKubernetesPKIVolume] = corev1.Volume{
+		Name: "etc-kubernetes-pki",
+		VolumeSource: corev1.VolumeSource{
+			Projected: &corev1.ProjectedVolumeSource{
+				Sources:     sources,
+				DefaultMode: pointer.Int32Ptr(420),
+			},
+		},
+	}
+}
+
+func (d *Deployment) buildCAVolume(podSpec *corev1.PodSpec, tcp *kamajiv1alpha1.TenantControlPlane) {
+	if index := int(etcCACertificates) + 1; len(podSpec.Volumes) < index {
+		podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{})
+	}
+
+	podSpec.Volumes[etcCACertificates] = corev1.Volume{
+		Name: "etc-ca-certificates",
+		VolumeSource: corev1.VolumeSource{
+			Secret: &corev1.SecretVolumeSource{
+				SecretName:  tcp.Status.Certificates.CA.SecretName,
+				DefaultMode: pointer.Int32Ptr(420),
+			},
+		},
+	}
+}
+
+func (d *Deployment) buildSSLCertsVolume(podSpec *corev1.PodSpec, tcp *kamajiv1alpha1.TenantControlPlane) {
+	if index := int(etcSSLCerts) + 1; len(podSpec.Volumes) < index {
+		podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{})
+	}
+
+	podSpec.Volumes[etcSSLCerts] = corev1.Volume{
+		Name: "etc-ssl-certs",
+		VolumeSource: corev1.VolumeSource{
+			Secret: &corev1.SecretVolumeSource{
+				SecretName:  tcp.Status.Certificates.CA.SecretName,
+				DefaultMode: pointer.Int32Ptr(420),
+			},
+		},
+	}
+}
+
+func (d *Deployment) buildShareCAVolume(podSpec *corev1.PodSpec, tcp *kamajiv1alpha1.TenantControlPlane) {
+	if index := int(usrShareCACertificates) + 1; len(podSpec.Volumes) < index {
+		podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{})
+	}
+
+	podSpec.Volumes[usrShareCACertificates] = corev1.Volume{
+		Name: "usr-share-ca-certificates",
+		VolumeSource: corev1.VolumeSource{
+			Secret: &corev1.SecretVolumeSource{
+				SecretName:  tcp.Status.Certificates.CA.SecretName,
+				DefaultMode: pointer.Int32Ptr(420),
+			},
+		},
+	}
+}
+
+func (d *Deployment) buildLocalShareCAVolume(podSpec *corev1.PodSpec, tcp *kamajiv1alpha1.TenantControlPlane) {
+	if index := int(usrLocalShareCACertificates) + 1; len(podSpec.Volumes) < index {
+		podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{})
+	}
+
+	podSpec.Volumes[usrLocalShareCACertificates] = corev1.Volume{
+		Name: "usr-local-share-ca-certificates",
+		VolumeSource: corev1.VolumeSource{
+			Secret: &corev1.SecretVolumeSource{
+				SecretName:  tcp.Status.Certificates.CA.SecretName,
+				DefaultMode: pointer.Int32Ptr(420),
+			},
+		},
+	}
+}
+
+func (d *Deployment) buildSchedulerVolume(podSpec *corev1.PodSpec, tcp *kamajiv1alpha1.TenantControlPlane) {
+	if index := int(schedulerKubeconfig) + 1; len(podSpec.Volumes) < index {
+		podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{})
+	}
+
+	podSpec.Volumes[schedulerKubeconfig] = corev1.Volume{
+		Name: "scheduler-kubeconfig",
+		VolumeSource: corev1.VolumeSource{
+			Secret: &corev1.SecretVolumeSource{
+				SecretName:  tcp.Status.KubeConfig.Scheduler.SecretName,
+				DefaultMode: pointer.Int32Ptr(420),
+			},
+		},
+	}
+}
+
+func (d *Deployment) buildControllerManagerVolume(podSpec *corev1.PodSpec, tcp *kamajiv1alpha1.TenantControlPlane) {
+	if index := int(controllerManagerKubeconfig) + 1; len(podSpec.Volumes) < index {
+		podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{})
+	}
+
+	podSpec.Volumes[controllerManagerKubeconfig] = corev1.Volume{
+		Name: "controller-manager-kubeconfig",
+		VolumeSource: corev1.VolumeSource{
+			Secret: &corev1.SecretVolumeSource{
+				SecretName:  tcp.Status.KubeConfig.ControllerManager.SecretName,
+				DefaultMode: pointer.Int32Ptr(420),
+			},
+		},
+	}
+}
+
+func (d *Deployment) BuildScheduler(podSpec *corev1.PodSpec, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) {
+	if index := int(schedulerIndex) + 1; len(podSpec.Containers) < index {
+		podSpec.Containers = append(podSpec.Containers, corev1.Container{})
+	}
+
+	args := map[string]string{}
+
+	if tenantControlPlane.Spec.ControlPlane.Deployment.ExtraArgs != nil {
+		args = utilities.ArgsFromSliceToMap(tenantControlPlane.Spec.ControlPlane.Deployment.ExtraArgs.Scheduler)
+	}
+
+	kubeconfig := "/etc/kubernetes/scheduler.conf"
+
+	args["--authentication-kubeconfig"] = kubeconfig
+	args["--authorization-kubeconfig"] = kubeconfig
+	args["--bind-address"] = "0.0.0.0"
+	args["--kubeconfig"] = kubeconfig
+	args["--leader-elect"] = "true" // nolint:goconst
+
+	podSpec.Containers[schedulerIndex].Name = "kube-scheduler"
+	podSpec.Containers[schedulerIndex].Image = fmt.Sprintf("k8s.gcr.io/kube-scheduler:%s", tenantControlPlane.Spec.Kubernetes.Version)
+	podSpec.Containers[schedulerIndex].Command = []string{"kube-scheduler"}
+	podSpec.Containers[schedulerIndex].Args = utilities.ArgsFromMapToSlice(args)
+	podSpec.Containers[schedulerIndex].VolumeMounts = []corev1.VolumeMount{
+		{
+			Name:      "scheduler-kubeconfig",
+			ReadOnly:  true,
+			MountPath: "/etc/kubernetes",
+		},
+	}
+	podSpec.Containers[schedulerIndex].LivenessProbe = &corev1.Probe{
+		ProbeHandler: corev1.ProbeHandler{
+			HTTPGet: &corev1.HTTPGetAction{
+				Path:   "/healthz",
+				Port:   intstr.FromInt(10259),
+				Scheme: corev1.URISchemeHTTPS,
+			},
+		},
+		InitialDelaySeconds: 0,
+		TimeoutSeconds:      1,
+		PeriodSeconds:       10,
+		SuccessThreshold:    1,
+		FailureThreshold:    3,
+	}
+	podSpec.Containers[schedulerIndex].StartupProbe = &corev1.Probe{
+		ProbeHandler: corev1.ProbeHandler{
+			HTTPGet: &corev1.HTTPGetAction{
+				Path:   "/healthz",
+				Port:   intstr.FromInt(10259),
+				Scheme: corev1.URISchemeHTTPS,
+			},
+		},
+		InitialDelaySeconds: 0,
+		TimeoutSeconds:      1,
+		PeriodSeconds:       10,
+		SuccessThreshold:    1,
+		FailureThreshold:    3,
+	}
+	podSpec.Containers[schedulerIndex].ImagePullPolicy = corev1.PullAlways
+	podSpec.Containers[schedulerIndex].Resources = corev1.ResourceRequirements{
+		Limits:   nil,
+		Requests: nil,
+	}
+
+	if componentsResources := tenantControlPlane.Spec.ControlPlane.Deployment.Resources; componentsResources != nil {
+		if resource := componentsResources.Scheduler; resource != nil {
+			podSpec.Containers[schedulerIndex].Resources = *resource
+		}
+	}
+}
+
+func (d *Deployment) buildControllerManager(podSpec *corev1.PodSpec, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) {
+	if index := int(controllerManagerIndex) + 1; len(podSpec.Containers) < index {
+		podSpec.Containers = append(podSpec.Containers, corev1.Container{})
+	}
+
+	args := map[string]string{}
+
+	if tenantControlPlane.Spec.ControlPlane.Deployment.ExtraArgs != nil {
+		args = utilities.ArgsFromSliceToMap(tenantControlPlane.Spec.ControlPlane.Deployment.ExtraArgs.ControllerManager)
+	}
+
+	kubeconfig := "/etc/kubernetes/controller-manager.conf"
+
+	args["--allocate-node-cidrs"] = "true"
+	args["--authentication-kubeconfig"] = kubeconfig
+	args["--authorization-kubeconfig"] = kubeconfig
+	args["--bind-address"] = "0.0.0.0"
+	args["--client-ca-file"] = path.Join(v1beta3.DefaultCertificatesDir, constants.CACertName)
+	args["--cluster-name"] = tenantControlPlane.GetName()
+	args["--cluster-signing-cert-file"] = path.Join(v1beta3.DefaultCertificatesDir, constants.CACertName)
+	args["--cluster-signing-key-file"] = path.Join(v1beta3.DefaultCertificatesDir, constants.CAKeyName)
+	args["--controllers"] = "*,bootstrapsigner,tokencleaner"
+	args["--kubeconfig"] = kubeconfig
+	args["--leader-elect"] = "true"
+	args["--service-cluster-ip-range"] = tenantControlPlane.Spec.NetworkProfile.ServiceCIDR
+	args["--cluster-cidr"] = tenantControlPlane.Spec.NetworkProfile.PodCIDR
+	args["--requestheader-client-ca-file"] = path.Join(v1beta3.DefaultCertificatesDir, constants.FrontProxyCACertName)
+	args["--root-ca-file"] = path.Join(v1beta3.DefaultCertificatesDir, constants.CACertName)
+	args["--service-account-private-key-file"] = path.Join(v1beta3.DefaultCertificatesDir, constants.ServiceAccountPrivateKeyName)
+	args["--use-service-account-credentials"] = "true"
+
+	podSpec.Containers[controllerManagerIndex].Name = "kube-controller-manager"
+	podSpec.Containers[controllerManagerIndex].Image = fmt.Sprintf("k8s.gcr.io/kube-controller-manager:%s", tenantControlPlane.Spec.Kubernetes.Version)
+	podSpec.Containers[controllerManagerIndex].Command = []string{"kube-controller-manager"}
+	podSpec.Containers[controllerManagerIndex].Args = utilities.ArgsFromMapToSlice(args)
+	podSpec.Containers[controllerManagerIndex].VolumeMounts = []corev1.VolumeMount{
+		{
+			Name:      "controller-manager-kubeconfig",
+			ReadOnly:  true,
+			MountPath: "/etc/kubernetes",
+		},
+		{
+			Name:      "etc-kubernetes-pki",
+			ReadOnly:  true,
+			MountPath: v1beta3.DefaultCertificatesDir,
+		},
+		{
+			Name:      "etc-ca-certificates",
+			ReadOnly:  true,
+			MountPath: "/etc/ca-certificates",
+		},
+		{
+			Name:      "etc-ssl-certs",
+			ReadOnly:  true,
+			MountPath: "/etc/ssl/certs",
+		},
+		{
+			Name:      "usr-share-ca-certificates",
+			ReadOnly:  true,
+			MountPath: "/usr/share/ca-certificates",
+		},
+		{
+			Name:      "usr-local-share-ca-certificates",
+			ReadOnly:  true,
+			MountPath: "/usr/local/share/ca-certificates",
+		},
+	}
+	podSpec.Containers[controllerManagerIndex].LivenessProbe = &corev1.Probe{
+		ProbeHandler: corev1.ProbeHandler{
+			HTTPGet: &corev1.HTTPGetAction{
+				Path:   "/healthz",
+				Port:   intstr.FromInt(10257),
+				Scheme: corev1.URISchemeHTTPS,
+			},
+		},
+		InitialDelaySeconds: 0,
+		TimeoutSeconds:      1,
+		PeriodSeconds:       10,
+		SuccessThreshold:    1,
+		FailureThreshold:    3,
+	}
+	podSpec.Containers[controllerManagerIndex].StartupProbe = &corev1.Probe{
+		ProbeHandler: corev1.ProbeHandler{
+			HTTPGet: &corev1.HTTPGetAction{
+				Path:   "/healthz",
+				Port:   intstr.FromInt(10257),
+				Scheme: corev1.URISchemeHTTPS,
+			},
+		},
+		InitialDelaySeconds: 0,
+		TimeoutSeconds:      1,
+		PeriodSeconds:       10,
+		SuccessThreshold:    1,
+		FailureThreshold:    3,
+	}
+	podSpec.Containers[controllerManagerIndex].ImagePullPolicy = corev1.PullAlways
+	podSpec.Containers[controllerManagerIndex].Resources = corev1.ResourceRequirements{
+		Limits:   nil,
+		Requests: nil,
+	}
+
+	if componentsResources := tenantControlPlane.Spec.ControlPlane.Deployment.Resources; componentsResources != nil {
+		if resource := componentsResources.ControllerManager; resource != nil {
+			podSpec.Containers[controllerManagerIndex].Resources = *resource
+		}
+	}
+}
+
+func (d *Deployment) buildKubeAPIServer(podSpec *corev1.PodSpec, tenantControlPlane *kamajiv1alpha1.TenantControlPlane, address string) {
+	if index := int(apiServerIndex) + 1; len(podSpec.Containers) < index {
+		podSpec.Containers = append(podSpec.Containers, corev1.Container{})
+	}
+
+	args := d.buildKubeAPIServerCommand(tenantControlPlane, address, utilities.ArgsFromSliceToMap(podSpec.Containers[apiServerIndex].Args))
+
+	podSpec.Containers[apiServerIndex].Name = "kube-apiserver"
+	podSpec.Containers[apiServerIndex].Args = utilities.ArgsFromMapToSlice(args)
+	podSpec.Containers[apiServerIndex].Image = fmt.Sprintf("k8s.gcr.io/kube-apiserver:%s", tenantControlPlane.Spec.Kubernetes.Version)
+	podSpec.Containers[apiServerIndex].Command = []string{"kube-apiserver"}
+	podSpec.Containers[apiServerIndex].LivenessProbe = &corev1.Probe{
+		ProbeHandler: corev1.ProbeHandler{
+			HTTPGet: &corev1.HTTPGetAction{
+				Path:   "/livez",
+				Port:   intstr.FromInt(int(tenantControlPlane.Spec.NetworkProfile.Port)),
+				Scheme: corev1.URISchemeHTTPS,
+			},
+		},
+		InitialDelaySeconds: 0,
+		TimeoutSeconds:      1,
+		PeriodSeconds:       10,
+		SuccessThreshold:    1,
+		FailureThreshold:    3,
+	}
+	podSpec.Containers[apiServerIndex].ReadinessProbe = &corev1.Probe{
+		ProbeHandler: corev1.ProbeHandler{
+			HTTPGet: &corev1.HTTPGetAction{
+				Path:   "/readyz",
+				Port:   intstr.FromInt(int(tenantControlPlane.Spec.NetworkProfile.Port)),
+				Scheme: corev1.URISchemeHTTPS,
+			},
+		},
+		InitialDelaySeconds: 0,
+		TimeoutSeconds:      1,
+		PeriodSeconds:       10,
+		SuccessThreshold:    1,
+		FailureThreshold:    3,
+	}
+	podSpec.Containers[apiServerIndex].StartupProbe = &corev1.Probe{
+		ProbeHandler: corev1.ProbeHandler{
+			HTTPGet: &corev1.HTTPGetAction{
+				Path:   "/livez",
+				Port:   intstr.FromInt(int(tenantControlPlane.Spec.NetworkProfile.Port)),
+				Scheme: corev1.URISchemeHTTPS,
+			},
+		},
+		InitialDelaySeconds: 0,
+		TimeoutSeconds:      1,
+		PeriodSeconds:       10,
+		SuccessThreshold:    1,
+		FailureThreshold:    3,
+	}
+	podSpec.Containers[apiServerIndex].ImagePullPolicy = corev1.PullAlways
+
+	if len(podSpec.Containers[apiServerIndex].VolumeMounts) < 5 {
+		podSpec.Containers[apiServerIndex].VolumeMounts = make([]corev1.VolumeMount, 5)
+	}
+	podSpec.Containers[apiServerIndex].VolumeMounts[0] = corev1.VolumeMount{
+		Name:      "etc-kubernetes-pki",
+		ReadOnly:  true,
+		MountPath: v1beta3.DefaultCertificatesDir,
+	}
+	podSpec.Containers[apiServerIndex].VolumeMounts[1] = corev1.VolumeMount{
+		Name:      "etc-ca-certificates",
+		ReadOnly:  true,
+		MountPath: "/etc/ca-certificates",
+	}
+	podSpec.Containers[apiServerIndex].VolumeMounts[2] = corev1.VolumeMount{
+		Name:      "etc-ssl-certs",
+		ReadOnly:  true,
+		MountPath: "/etc/ssl/certs",
+	}
+	podSpec.Containers[apiServerIndex].VolumeMounts[3] = corev1.VolumeMount{
+		Name:      "usr-share-ca-certificates",
+		ReadOnly:  true,
+		MountPath: "/usr/share/ca-certificates",
+	}
+	podSpec.Containers[apiServerIndex].VolumeMounts[4] = corev1.VolumeMount{
+		Name:      "usr-local-share-ca-certificates",
+		ReadOnly:  true,
+		MountPath: "/usr/local/share/ca-certificates",
+	}
+	podSpec.Containers[apiServerIndex].Resources = corev1.ResourceRequirements{
+		Limits:   nil,
+		Requests: nil,
+	}
+
+	if componentsResources := tenantControlPlane.Spec.ControlPlane.Deployment.Resources; componentsResources != nil {
+		if resource := componentsResources.APIServer; resource != nil {
+			podSpec.Containers[apiServerIndex].Resources = *resource
+		}
+	}
+}
+
+func (d *Deployment) buildKubeAPIServerCommand(tenantControlPlane *kamajiv1alpha1.TenantControlPlane, address string, current map[string]string) map[string]string {
+	var extraArgs map[string]string
+
+	if tenantControlPlane.Spec.ControlPlane.Deployment.ExtraArgs != nil {
+		extraArgs = utilities.ArgsFromSliceToMap(tenantControlPlane.Spec.ControlPlane.Deployment.ExtraArgs.APIServer)
+	}
+
+	desiredArgs := map[string]string{
+		"--allow-privileged":                   "true",
+		"--authorization-mode":                 "Node,RBAC",
+		"--advertise-address":                  address,
+		"--client-ca-file":                     path.Join(v1beta3.DefaultCertificatesDir, constants.CACertName),
+		"--enable-admission-plugins":           strings.Join(tenantControlPlane.Spec.Kubernetes.AdmissionControllers.ToSlice(), ","),
+		"--enable-bootstrap-token-auth":        "true",
+		"--etcd-servers":                       strings.Join(d.ETCDEndpoints, ","),
+		"--service-cluster-ip-range":           tenantControlPlane.Spec.NetworkProfile.ServiceCIDR,
+		"--kubelet-client-certificate":         path.Join(v1beta3.DefaultCertificatesDir, constants.APIServerKubeletClientCertName),
+		"--kubelet-client-key":                 path.Join(v1beta3.DefaultCertificatesDir, constants.APIServerKubeletClientKeyName),
+		"--kubelet-preferred-address-types":    "Hostname,InternalIP,ExternalIP",
+		"--proxy-client-cert-file":             path.Join(v1beta3.DefaultCertificatesDir, constants.FrontProxyClientCertName),
+		"--proxy-client-key-file":              path.Join(v1beta3.DefaultCertificatesDir, constants.FrontProxyClientKeyName),
+		"--requestheader-allowed-names":        "front-proxy-client",
+		"--requestheader-extra-headers-prefix": "X-Remote-Extra-",
+		"--requestheader-group-headers":        "X-Remote-Group",
+		"--requestheader-username-headers":     "X-Remote-User",
+		"--secure-port":                        fmt.Sprintf("%d", tenantControlPlane.Spec.NetworkProfile.Port),
+		"--service-account-issuer":             fmt.Sprintf("https://localhost:%d", tenantControlPlane.Spec.NetworkProfile.Port),
+		"--service-account-key-file":           path.Join(v1beta3.DefaultCertificatesDir, constants.ServiceAccountPublicKeyName),
+		"--service-account-signing-key-file":   path.Join(v1beta3.DefaultCertificatesDir, constants.ServiceAccountPrivateKeyName),
+		"--tls-cert-file":                      path.Join(v1beta3.DefaultCertificatesDir, constants.APIServerCertName),
+		"--tls-private-key-file":               path.Join(v1beta3.DefaultCertificatesDir, constants.APIServerKeyName),
+	}
+
+	if d.ETCDStorageType == types.ETCD {
+		desiredArgs["--etcd-compaction-interval"] = d.ETCDCompactionInterval
+		desiredArgs["--etcd-cafile"] = path.Join(v1beta3.DefaultCertificatesDir, constants.EtcdCACertName)
+		desiredArgs["--etcd-certfile"] = path.Join(v1beta3.DefaultCertificatesDir, constants.APIServerEtcdClientCertName)
+		desiredArgs["--etcd-keyfile"] = path.Join(v1beta3.DefaultCertificatesDir, constants.APIServerEtcdClientKeyName)
+		desiredArgs["--etcd-prefix"] = fmt.Sprintf("/%s", tenantControlPlane.GetName())
+	}
+	// Order matters, here: extraArgs could try to overwrite some arguments managed by Kamaji and that would be crucial.
+	// Adding as first element of the array of maps, we're sure that these overrides will be sanitized by our configuration.
+	return utilities.MergeMaps(extraArgs, current, desiredArgs)
+}
+
+func (d *Deployment) secretProjection(secretName, certKeyName, keyName string) *corev1.SecretProjection {
+	return &corev1.SecretProjection{
+		LocalObjectReference: corev1.LocalObjectReference{
+			Name: secretName,
+		},
+		Items: []corev1.KeyToPath{
+			{
+				Key:  certKeyName,
+				Path: certKeyName,
+			},
+			{
+				Key:  keyName,
+				Path: keyName,
+			},
+		},
+	}
+}
+
+func (d *Deployment) buildKineVolume(podSpec *corev1.PodSpec, tcp *kamajiv1alpha1.TenantControlPlane) {
+	// Kine is expecting an additional volume for its configuration, and it must be removed before proceeding with the
+	// customized storage that is idempotent
+	if found, index := utilities.HasNamedVolume(podSpec.Volumes, kineVolumeName); found {
+		var volumes []corev1.Volume
+
+		volumes = append(volumes, podSpec.Volumes[:index]...)
+		volumes = append(volumes, podSpec.Volumes[index+1:]...)
+
+		podSpec.Volumes = volumes
+	}
+
+	if d.ETCDStorageType == types.KineMySQL {
+		if index := int(kineConfig) + 1; len(podSpec.Volumes) < index {
+			podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{})
+		}
+
+		podSpec.Volumes[kineConfig].Name = kineVolumeName
+		podSpec.Volumes[kineConfig].VolumeSource = corev1.VolumeSource{
+			Secret: &corev1.SecretVolumeSource{
+				SecretName:  tcp.Status.Storage.KineMySQL.Certificate.SecretName,
+				DefaultMode: pointer.Int32Ptr(420),
+			},
+		}
+	}
+}
+
+func (d *Deployment) buildKine(podSpec *corev1.PodSpec, tcp *kamajiv1alpha1.TenantControlPlane) {
+	const kineContainerName = "kine"
+	// Kine is expecting an additional container, and it must be removed before proceeding with the additional one
+	// in order to make this function idempotent.
+	if found, index := utilities.HasNamedContainer(podSpec.Containers, kineContainerName); found {
+		var containers []corev1.Container
+
+		containers = append(containers, podSpec.Containers[:index]...)
+		containers = append(containers, podSpec.Containers[index+1:]...)
+
+		podSpec.Containers = containers
+	}
+
+	if d.ETCDStorageType == types.KineMySQL {
+		if index := int(kineIndex) + 1; len(podSpec.Containers) < index {
+			podSpec.Containers = append(podSpec.Containers, corev1.Container{})
+		}
+
+		args := map[string]string{}
+
+		if tcp.Spec.ControlPlane.Deployment.ExtraArgs != nil {
+			args = utilities.ArgsFromSliceToMap(tcp.Spec.ControlPlane.Deployment.ExtraArgs.Kine)
+		}
+
+		args["--endpoint"] = "mysql://$(MYSQL_USER):$(MYSQL_PASSWORD)@tcp($(MYSQL_HOST):$(MYSQL_PORT))/$(MYSQL_SCHEMA)"
+		args["--ca-file"] = "/kine/ca.crt"
+		args["--cert-file"] = "/kine/server.crt"
+		args["--key-file"] = "/kine/server.key"
+
+		podSpec.Containers[kineIndex].Name = kineContainerName
+		podSpec.Containers[kineIndex].Image = d.KineContainerImage
+		podSpec.Containers[kineIndex].Command = []string{"/bin/kine"}
+		podSpec.Containers[kineIndex].Args = utilities.ArgsFromMapToSlice(args)
+		podSpec.Containers[kineIndex].VolumeMounts = []corev1.VolumeMount{
+			{
+				Name:      kineVolumeName,
+				MountPath: "/kine",
+				ReadOnly:  true,
+			},
+		}
+		podSpec.Containers[kineIndex].Env = []corev1.EnvVar{
+			{
+				Name:  "GODEBUG",
+				Value: "x509ignoreCN=0",
+			},
+		}
+		podSpec.Containers[kineIndex].EnvFrom = []corev1.EnvFromSource{
+			{
+				SecretRef: &corev1.SecretEnvSource{
+					LocalObjectReference: corev1.LocalObjectReference{
+						Name: tcp.Status.Storage.KineMySQL.Config.SecretName,
+					},
+				},
+			},
+		}
+		podSpec.Containers[kineIndex].Ports = []corev1.ContainerPort{
+			{
+				ContainerPort: 2379,
+				Name:          "server",
+				Protocol:      corev1.ProtocolTCP,
+			},
+		}
+		podSpec.Containers[kineIndex].ImagePullPolicy = corev1.PullAlways
+	}
+}
+
+func (d *Deployment) SetSelector(deploymentSpec *appsv1.DeploymentSpec, tcp *kamajiv1alpha1.TenantControlPlane) {
+	deploymentSpec.Selector = &metav1.LabelSelector{
+		MatchLabels: map[string]string{
+			"kamaji.clastix.io/soot": tcp.GetName(),
+		},
+	}
+}
+
+func (d *Deployment) SetReplicas(deploymentSpec *appsv1.DeploymentSpec, tcp *kamajiv1alpha1.TenantControlPlane) {
+	deploymentSpec.Replicas = pointer.Int32(tcp.Spec.ControlPlane.Deployment.Replicas)
+}
+
+func (d *Deployment) SetTemplateLabels(template *corev1.PodTemplateSpec, labels map[string]string) {
+	template.SetLabels(labels)
+}
+
+func (d *Deployment) SetLabels(resource *appsv1.Deployment, labels map[string]string) {
+	resource.SetLabels(labels)
+}
+
+func (d *Deployment) SetAnnotations(resource *appsv1.Deployment, annotations map[string]string) {
+	resource.SetAnnotations(annotations)
+}
+
+// ResetKubeAPIServerFlags ensures that upon a change of the kube-apiserver extra flags the desired ones are properly
+// applied, also considering that the container could be lately patched by the konnectivity addon resources.
+func (d *Deployment) ResetKubeAPIServerFlags(resource *appsv1.Deployment, tcp *kamajiv1alpha1.TenantControlPlane) {
+	if tcp.Spec.ControlPlane.Deployment.ExtraArgs == nil {
+		return
+	}
+	// kube-apiserver container is not still there, we can skip the hashing
+	if found, _ := utilities.HasNamedContainer(resource.Spec.Template.Spec.Containers, "kube-apiserver"); !found {
+		return
+	}
+	// setting up annotation to avoid assignment to a nil one
+	if resource.GetAnnotations() == nil {
+		resource.SetAnnotations(map[string]string{})
+	}
+	// retrieving the current amount of extra flags, used as a sort of hash:
+	// in case of non-matching values, removing all the args in order to perform a full reconciliation from a clean start.
+	var count int
+
+	if v, ok := resource.GetAnnotations()[apiServerFlagsAnnotation]; ok {
+		var err error
+
+		if count, err = strconv.Atoi(v); err != nil {
+			return
+		}
+	}
+	// there's a mismatch in the count from the previous hash: let's reset and store the desired extra args count.
+	if count != len(tcp.Spec.ControlPlane.Deployment.ExtraArgs.APIServer) {
+		resource.Spec.Template.Spec.Containers[apiServerIndex].Args = []string{}
+	}
+
+	resource.GetAnnotations()[apiServerFlagsAnnotation] = fmt.Sprintf("%d", len(tcp.Spec.ControlPlane.Deployment.ExtraArgs.APIServer))
+}

internal/config/config.go

@@ -22,6 +22,7 @@ var (
 
 const (
 	envPrefix                        = "KAMAJI"
+	defaultETCDStorageType           = "etcd"
 	defaultETCDCASecretName          = "etcd-certs"
 	defaultETCDCASecretNamespace     = "kamaji-system"
 	defaultETCDEndpoints             = "etcd-server:2379"
@@ -29,6 +30,11 @@ const (
 	defaultETCDClientSecretName      = "root-client-certs"
 	defaultETCDClientSecretNamespace = "kamaji-system"
 	defaultTmpDirectory              = "/tmp/kamaji"
+	defaultKineMySQLSecretName       = "mysql-config"
+	defaultKineMySQLSecretNamespace  = "kamaji-system"
+	defaultKineMySQLHost             = "localhost"
+	defaultKineMySQLPort             = 3306
+	defaultKineImage                 = "rancher/kine:v0.9.2-amd64"
 )
 
 func InitConfig() (*viper.Viper, error) {
@@ -40,13 +46,19 @@ func InitConfig() (*viper.Viper, error) {
 	flag.String("health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
 	flag.Bool("leader-elect", false, "Enable leader election for controller manager. "+
 		"Enabling this will ensure there is only one active controller manager.")
+	flag.String("etcd-storage-type", defaultETCDStorageType, "Type of storage for ETCD (i.e etcd, kine-mysql, kine-postgres)")
 	flag.String("etcd-ca-secret-name", defaultETCDCASecretName, "Name of the secret which contains CA's certificate and private key.")
 	flag.String("etcd-ca-secret-namespace", defaultETCDCASecretNamespace, "Namespace of the secret which contains CA's certificate and private key.")
 	flag.String("etcd-client-secret-name", defaultETCDClientSecretName, "Name of the secret which contains ETCD client certificates")
 	flag.String("etcd-client-secret-namespace", defaultETCDClientSecretNamespace, "Name of the namespace where the secret which contains ETCD client certificates is")
-	flag.String("etcd-endpoints", defaultETCDEndpoints, "Comma-separated list with ETCD endpoints (i.e. etcd-0.etcd.kamaji-system.svc.cluster.local,etcd-1.etcd.kamaji-system.svc.cluster.local,etcd-2.etcd.kamaji-system.svc.cluster.local)")
+	flag.String("etcd-endpoints", defaultETCDEndpoints, "Comma-separated list with ETCD endpoints (i.e. https://etcd-0.etcd.kamaji-system.svc.cluster.local,https://etcd-1.etcd.kamaji-system.svc.cluster.local,https://etcd-2.etcd.kamaji-system.svc.cluster.local)")
 	flag.String("etcd-compaction-interval", defaultETCDCompactionInterval, "ETCD Compaction interval (i.e. \"5m0s\"). (default: \"0\" (disabled))")
 	flag.String("tmp-directory", defaultTmpDirectory, "Directory which will be used to work with temporary files.")
+	flag.String("kine-mysql-secret-name", defaultKineMySQLSecretName, "Name of the secret which contains MySQL (Kine) configuration.")
+	flag.String("kine-mysql-secret-namespace", defaultKineMySQLSecretNamespace, "Name of the namespace where the secret which contains MySQL (Kine) configuration.")
+	flag.String("kine-mysql-host", defaultKineMySQLHost, "Host where MySQL (Kine) is working")
+	flag.Int("kine-mysql-port", defaultKineMySQLPort, "Port where MySQL (Kine) is working")
+	flag.String("kine-image", defaultKineImage, "Container image along with tag to use for the Kine sidecar container (used only if etcd-storage-type is set to one of kine strategies)")
 
 	// Setup zap configuration
 	opts := zap.Options{
@@ -74,6 +86,9 @@ func InitConfig() (*viper.Viper, error) {
 	if err := config.BindEnv("leader-elect", fmt.Sprintf("%s_LEADER_ELECTION", envPrefix)); err != nil {
 		return nil, err
 	}
+	if err := config.BindEnv("etcd-storage-type", fmt.Sprintf("%s_ETCD_STORAGE_TYPE", envPrefix)); err != nil {
+		return nil, err
+	}
 	if err := config.BindEnv("etcd-ca-secret-name", fmt.Sprintf("%s_ETCD_CA_SECRET_NAME", envPrefix)); err != nil {
 		return nil, err
 	}
@@ -95,6 +110,21 @@ func InitConfig() (*viper.Viper, error) {
 	if err := config.BindEnv("tmp-directory", fmt.Sprintf("%s_TMP_DIRECTORY", envPrefix)); err != nil {
 		return nil, err
 	}
+	if err := config.BindEnv("kine-mysql-secret-name", fmt.Sprintf("%s_KINE_MYSQL_SECRET_NAME", envPrefix)); err != nil {
+		return nil, err
+	}
+	if err := config.BindEnv("kine-mysql-secret-namespace", fmt.Sprintf("%s_KINE_MYSQL_SECRET_NAMESPACE", envPrefix)); err != nil {
+		return nil, err
+	}
+	if err := config.BindEnv("kine-mysql-host", fmt.Sprintf("%s_KINE_MYSQL_HOST", envPrefix)); err != nil {
+		return nil, err
+	}
+	if err := config.BindEnv("kine-mysql-port", fmt.Sprintf("%s_KINE_MYSQL_PORT", envPrefix)); err != nil {
+		return nil, err
+	}
+	if err := config.BindEnv("kine-image", fmt.Sprintf("%s_KINE_IMAGE", envPrefix)); err != nil {
+		return nil, err
+	}
 
 	// Setup config file
 	if cfgFile != "" {

internal/crypto/crypto.go

@@ -13,6 +13,24 @@ import (
 	"time"
 )
 
+const (
+	certBitSize = 2048
+)
+
+func GetCertificateAndKeyPair(template *x509.Certificate, caCert []byte, caPrivKey []byte) (*bytes.Buffer, *bytes.Buffer, error) {
+	caCertBytes, err := GetCertificate(caCert)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	caPrivKeyBytes, err := GetPrivateKey(caPrivKey)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return GenerateCertificateKeyPairBytes(template, certBitSize, caCertBytes, caPrivKeyBytes)
+}
+
 func GetCertificate(cert []byte) (*x509.Certificate, error) {
 	pemContent, _ := pem.Decode(cert)
 	if pemContent == nil {

internal/etcd/certs.go

@@ -17,17 +17,7 @@ import (
 func GetETCDCACertificateAndKeyPair(tenant string, caCert []byte, caPrivKey []byte) (*bytes.Buffer, *bytes.Buffer, error) {
 	template := getCertTemplate(tenant)
 
-	caCertBytes, err := crypto.GetCertificate(caCert)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	caPrivKeyBytes, err := crypto.GetPrivateKey(caPrivKey)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	return crypto.GenerateCertificateKeyPairBytes(template, certBitSize, caCertBytes, caPrivKeyBytes)
+	return crypto.GetCertificateAndKeyPair(template, caCert, caPrivKey)
 }
 
 func IsETCDCertificateAndKeyPairValid(cert []byte, privKey []byte) (bool, error) {

internal/etcd/constant.go

@@ -6,5 +6,4 @@ package etcd
 const (
 	certExpirationDelayYears = 10
 	certOrganization         = "system:masters"
-	certBitSize              = 2048
 )

internal/kubeadm/addon.go

@@ -22,6 +22,8 @@ import (
 	"k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/proxy"
 	"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
 	"k8s.io/utils/pointer"
+
+	"github.com/clastix/kamaji/internal/utilities"
 )
 
 const (
@@ -412,7 +414,7 @@ func getKubeproxyConfigmapContent(config *Configuration) ([]byte, error) {
 		},
 	}
 
-	return EncondeToYaml(&kubeProxyConfiguration)
+	return utilities.EncondeToYaml(&kubeProxyConfiguration)
 }
 
 func getKubeproxyKubeconfigContent(config *Configuration) ([]byte, error) {
@@ -447,5 +449,5 @@ func getKubeproxyKubeconfigContent(config *Configuration) ([]byte, error) {
 		},
 	}
 
-	return EncondeToYaml(&kubeconfig)
+	return utilities.EncondeToYaml(&kubeconfig)
 }

internal/kubeadm/configuration.go

@@ -6,6 +6,7 @@ package kubeadm
 import (
 	"encoding/json"
 	"fmt"
+	"strings"
 	"time"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -14,6 +15,12 @@ import (
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 )
 
+const (
+	defaultCAFile   = "/etc/kubernetes/pki/etcd/ca.crt"
+	defaultCertFile = "/etc/kubernetes/pki/apiserver-etcd-client.crt"
+	defaultKeyFile  = "/etc/kubernetes/pki/apiserver-etcd-client.key"
+)
+
 func CreateKubeadmInitConfiguration(params Parameters) Configuration {
 	config := kubeadmapi.InitConfiguration{
 		ClusterConfiguration: getKubeadmClusterConfiguration(params),
@@ -40,7 +47,16 @@ func CreateKubeadmInitConfiguration(params Parameters) Configuration {
 	return Configuration{InitConfiguration: config}
 }
 
+func isHTTPS(url string) bool {
+	return strings.HasPrefix(url, "https")
+}
+
 func getKubeadmClusterConfiguration(params Parameters) kubeadmapi.ClusterConfiguration {
+	caFile, certFile, keyFile := "", "", ""
+	if isHTTPS(params.ETCDs[0]) {
+		caFile, certFile, keyFile = defaultCAFile, defaultCertFile, defaultKeyFile
+	}
+
 	return kubeadmapi.ClusterConfiguration{
 		KubernetesVersion: params.TenantControlPlaneVersion,
 		ClusterName:       params.TenantControlPlaneName,
@@ -57,22 +73,21 @@ func getKubeadmClusterConfiguration(params Parameters) kubeadmapi.ClusterConfigu
 		ControlPlaneEndpoint: params.TenantControlPlaneEndpoint,
 		Etcd: kubeadmapi.Etcd{
 			External: &kubeadmapi.ExternalEtcd{
-				Endpoints: formatETCDEndpoints(params.ETCDs),
-				CAFile:    "/etc/kubernetes/pki/etcd/ca.crt",
-				CertFile:  "/etc/kubernetes/pki/apiserver-etcd-client.crt",
-				KeyFile:   "/etc/kubernetes/pki/apiserver-etcd-client.key",
+				Endpoints: params.ETCDs,
+				CAFile:    caFile,
+				CertFile:  certFile,
+				KeyFile:   keyFile,
 			},
 		},
 		APIServer: kubeadmapi.APIServer{
-			CertSANs: []string{
+			CertSANs: append([]string{
 				"127.0.0.1",
 				"localhost",
-				fmt.Sprintf("%s.%s", params.TenantControlPlaneName, params.TenantControlPlaneDomain),
 				params.TenantControlPlaneName,
 				fmt.Sprintf("%s.%s.svc", params.TenantControlPlaneName, params.TenantControlPlaneNamespace),
 				fmt.Sprintf("%s.%s.svc.cluster.local", params.TenantControlPlaneName, params.TenantControlPlaneNamespace),
 				params.TenantControlPlaneAddress,
-			},
+			}, params.TenantControlPlaneCertSANs...),
 			ControlPlaneComponent: kubeadmapi.ControlPlaneComponent{
 				ExtraArgs: map[string]string{
 					"etcd-compaction-interval": params.ETCDCompactionInterval,
@@ -131,12 +146,3 @@ func getJSONStringFromStruct(i interface{}) (string, error) {
 
 	return string(b), nil
 }
-
-func formatETCDEndpoints(etcds []string) []string {
-	formatedETCDs := make([]string, 0, len(etcds))
-	for _, etcd := range etcds {
-		formatedETCDs = append(formatedETCDs, fmt.Sprintf("https://%s/", etcd))
-	}
-
-	return formatedETCDs
-}

internal/kubeadm/types.go

@@ -4,9 +4,11 @@
 package kubeadm
 
 import (
+	json "github.com/json-iterator/go"
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
 
 	kubeconfigutil "github.com/clastix/kamaji/internal/kubeconfig"
+	"github.com/clastix/kamaji/internal/utilities"
 )
 
 type Configuration struct {
@@ -15,13 +17,27 @@ type Configuration struct {
 	Parameters        Parameters
 }
 
+func (c *Configuration) Checksum() string {
+	initConfiguration, _ := utilities.EncondeToYaml(&c.InitConfiguration)
+	kubeconfig, _ := json.Marshal(c.Kubeconfig)
+	parameters, _ := json.Marshal(c.Parameters)
+
+	data := map[string]string{
+		"InitConfiguration": string(initConfiguration),
+		"Kubeconfig":        string(kubeconfig),
+		"Parameters":        string(parameters),
+	}
+
+	return utilities.CalculateConfigMapChecksum(data)
+}
+
 type Parameters struct {
 	TenantControlPlaneName         string
 	TenantControlPlaneNamespace    string
 	TenantControlPlaneEndpoint     string
 	TenantControlPlaneAddress      string
+	TenantControlPlaneCertSANs     []string
 	TenantControlPlanePort         int32
-	TenantControlPlaneDomain       string
 	TenantControlPlanePodCIDR      string
 	TenantControlPlaneServiceCIDR  string
 	TenantDNSServiceIPs            []string

internal/kubeadm/uploadconfig.go

@@ -18,6 +18,8 @@ import (
 	"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
 	"k8s.io/kubernetes/pkg/apis/rbac"
 	"k8s.io/utils/pointer"
+
+	"github.com/clastix/kamaji/internal/utilities"
 )
 
 func UploadKubeadmConfig(client kubernetes.Interface, config *Configuration) error {
@@ -115,7 +117,7 @@ func getKubeletConfigmapContent(kubeletConfiguration KubeletConfiguration) ([]by
 		VolumeStatsAggPeriod:             zeroDuration,
 	}
 
-	return EncondeToYaml(&kc)
+	return utilities.EncondeToYaml(&kc)
 }
 
 func createConfigMapRBACRules(client kubernetes.Interface, kubernetesVersion string) error {

internal/kubeadm/utils.go

@@ -1,20 +0,0 @@
-// Copyright 2022 Clastix Labs
-// SPDX-License-Identifier: Apache-2.0
-
-package kubeadm
-
-import (
-	"bytes"
-
-	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/apimachinery/pkg/runtime/serializer/json"
-)
-
-func EncondeToYaml(o runtime.Object) ([]byte, error) {
-	scheme := runtime.NewScheme()
-	encoder := json.NewYAMLSerializer(json.SimpleMetaFactory{}, scheme, scheme)
-	buf := bytes.NewBuffer([]byte{})
-	err := encoder.Encode(o, buf)
-
-	return buf.Bytes(), err
-}

internal/resources/api_server_certificate.go

@@ -64,8 +64,8 @@ func (r *APIServerCertificate) GetTmpDirectory() string {
 	return r.TmpDirectory
 }
 
-func (r *APIServerCertificate) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) {
-	return controllerutil.CreateOrUpdate(ctx, r.Client, r.resource, r.mutate(ctx, tenantControlPlane))
+func (r *APIServerCertificate) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (res controllerutil.OperationResult, err error) {
+	return utilities.CreateOrUpdateWithConflict(ctx, r.Client, r.resource, r.mutate(ctx, tenantControlPlane))
 }
 
 func (r *APIServerCertificate) GetName() string {
@@ -96,7 +96,7 @@ func (r *APIServerCertificate) mutate(ctx context.Context, tenantControlPlane *k
 			}
 		}
 
-		config, _, err := getKubeadmConfiguration(ctx, r, tenantControlPlane)
+		config, err := getStoredKubeadmConfiguration(ctx, r, tenantControlPlane)
 		if err != nil {
 			return err
 		}

internal/resources/api_server_kubelet_client_certificate.go

@@ -64,8 +64,8 @@ func (r *APIServerKubeletClientCertificate) GetTmpDirectory() string {
 	return r.TmpDirectory
 }
 
-func (r *APIServerKubeletClientCertificate) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) {
-	return controllerutil.CreateOrUpdate(ctx, r.Client, r.resource, r.mutate(ctx, tenantControlPlane))
+func (r *APIServerKubeletClientCertificate) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (res controllerutil.OperationResult, err error) {
+	return utilities.CreateOrUpdateWithConflict(ctx, r.Client, r.resource, r.mutate(ctx, tenantControlPlane))
 }
 
 func (r *APIServerKubeletClientCertificate) GetName() string {
@@ -96,7 +96,7 @@ func (r *APIServerKubeletClientCertificate) mutate(ctx context.Context, tenantCo
 			}
 		}
 
-		config, _, err := getKubeadmConfiguration(ctx, r, tenantControlPlane)
+		config, err := getStoredKubeadmConfiguration(ctx, r, tenantControlPlane)
 		if err != nil {
 			return err
 		}

internal/resources/ca_certificate.go

@@ -29,7 +29,8 @@ type CACertificate struct {
 }
 
 func (r *CACertificate) ShouldStatusBeUpdated(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
-	return tenantControlPlane.Status.Certificates.CA.SecretName != r.resource.GetName()
+	return tenantControlPlane.Status.Certificates.CA.SecretName != r.resource.GetName() ||
+		tenantControlPlane.Status.Certificates.CA.ResourceVersion != r.resource.ResourceVersion
 }
 
 func (r *CACertificate) ShouldCleanup(plane *kamajiv1alpha1.TenantControlPlane) bool {
@@ -64,7 +65,7 @@ func (r *CACertificate) GetTmpDirectory() string {
 }
 
 func (r *CACertificate) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) {
-	return controllerutil.CreateOrUpdate(ctx, r.Client, r.resource, r.mutate(ctx, tenantControlPlane))
+	return utilities.CreateOrUpdateWithConflict(ctx, r.Client, r.resource, r.mutate(ctx, tenantControlPlane))
 }
 
 func (r *CACertificate) GetName() string {
@@ -74,6 +75,7 @@ func (r *CACertificate) GetName() string {
 func (r *CACertificate) UpdateTenantControlPlaneStatus(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
 	tenantControlPlane.Status.Certificates.CA.LastUpdate = metav1.Now()
 	tenantControlPlane.Status.Certificates.CA.SecretName = r.resource.GetName()
+	tenantControlPlane.Status.Certificates.CA.ResourceVersion = r.resource.ResourceVersion
 
 	return nil
 }
@@ -91,7 +93,7 @@ func (r *CACertificate) mutate(ctx context.Context, tenantControlPlane *kamajiv1
 			return nil
 		}
 
-		config, _, err := getKubeadmConfiguration(ctx, r, tenantControlPlane)
+		config, err := getStoredKubeadmConfiguration(ctx, r, tenantControlPlane)
 		if err != nil {
 			return err
 		}

internal/resources/constants.go

@@ -1,9 +0,0 @@
-// Copyright 2022 Clastix Labs
-// SPDX-License-Identifier: Apache-2.0
-
-package resources
-
-const (
-	kubeconfigAdminKeyName = "admin.conf"
-	defaultIngressPort     = 443
-)

internal/resources/etcd_ca_certificates.go

@@ -59,7 +59,7 @@ func (r *ETCDCACertificatesResource) Define(ctx context.Context, tenantControlPl
 }
 
 func (r *ETCDCACertificatesResource) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) {
-	return controllerutil.CreateOrUpdate(ctx, r.Client, r.resource, r.mutate(ctx, tenantControlPlane))
+	return utilities.CreateOrUpdateWithConflict(ctx, r.Client, r.resource, r.mutate(ctx, tenantControlPlane))
 }
 
 func (r *ETCDCACertificatesResource) GetName() string {

internal/resources/etcd_certificates.go

@@ -56,7 +56,7 @@ func (r *ETCDCertificatesResource) Define(ctx context.Context, tenantControlPlan
 }
 
 func (r *ETCDCertificatesResource) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) {
-	return controllerutil.CreateOrUpdate(ctx, r.Client, r.resource, r.mutate(ctx, tenantControlPlane))
+	return utilities.CreateOrUpdateWithConflict(ctx, r.Client, r.resource, r.mutate(ctx, tenantControlPlane))
 }
 
 func (r *ETCDCertificatesResource) GetName() string {

internal/resources/etcd_setup.go

@@ -9,7 +9,6 @@ import (
 	"github.com/go-logr/logr"
 	etcdclient "go.etcd.io/etcd/client/v3"
 	corev1 "k8s.io/api/core/v1"
-	"k8s.io/apimachinery/pkg/runtime"
 	k8stypes "k8s.io/apimachinery/pkg/types"
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -23,15 +22,14 @@ const (
 	caKeyName = kubeadmconstants.CACertName
 )
 
-type resource struct {
+type etcdSetupResource struct {
 	role etcd.Role
 	user etcd.User
 }
 
 type ETCDSetupResource struct {
-	resource              *resource
+	resource              *etcdSetupResource
 	Client                client.Client
-	Scheme                *runtime.Scheme
 	Log                   logr.Logger
 	Name                  string
 	Endpoints             []string
@@ -57,7 +55,7 @@ func (r *ETCDSetupResource) CleanUp(ctx context.Context, tenantControlPlane *kam
 }
 
 func (r *ETCDSetupResource) Define(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
-	r.resource = &resource{
+	r.resource = &etcdSetupResource{
 		role: etcd.Role{Name: tenantControlPlane.Name, Exists: false},
 		user: etcd.User{Name: tenantControlPlane.Name, Exists: false},
 	}
@@ -116,6 +114,7 @@ func (r *ETCDSetupResource) reconcile(ctx context.Context) (controllerutil.Opera
 	if err != nil {
 		return reconcilationResult, err
 	}
+	defer client.Close()
 
 	operationResult, err = r.reconcileUser(ctx, client)
 	if err != nil {

internal/resources/front-proxy-client-certificate.go

@@ -65,7 +65,7 @@ func (r *FrontProxyClientCertificate) GetTmpDirectory() string {
 }
 
 func (r *FrontProxyClientCertificate) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) {
-	return controllerutil.CreateOrUpdate(ctx, r.Client, r.resource, r.mutate(ctx, tenantControlPlane))
+	return utilities.CreateOrUpdateWithConflict(ctx, r.Client, r.resource, r.mutate(ctx, tenantControlPlane))
 }
 
 func (r *FrontProxyClientCertificate) GetName() string {
@@ -96,7 +96,7 @@ func (r *FrontProxyClientCertificate) mutate(ctx context.Context, tenantControlP
 			}
 		}
 
-		config, _, err := getKubeadmConfiguration(ctx, r, tenantControlPlane)
+		config, err := getStoredKubeadmConfiguration(ctx, r, tenantControlPlane)
 		if err != nil {
 			return err
 		}

internal/resources/front_proxy_ca_certificate.go

@@ -66,7 +66,7 @@ func (r *FrontProxyCACertificate) GetTmpDirectory() string {
 }
 
 func (r *FrontProxyCACertificate) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) {
-	return controllerutil.CreateOrUpdate(ctx, r.Client, r.resource, r.mutate(ctx, tenantControlPlane))
+	return utilities.CreateOrUpdateWithConflict(ctx, r.Client, r.resource, r.mutate(ctx, tenantControlPlane))
 }
 
 func (r *FrontProxyCACertificate) GetName() string {
@@ -93,7 +93,7 @@ func (r *FrontProxyCACertificate) mutate(ctx context.Context, tenantControlPlane
 			return nil
 		}
 
-		config, _, err := getKubeadmConfiguration(ctx, r, tenantControlPlane)
+		config, err := getStoredKubeadmConfiguration(ctx, r, tenantControlPlane)
 		if err != nil {
 			return err
 		}

internal/resources/k8s_deployment_resource.go

@@ -5,59 +5,50 @@ package resources
 
 import (
 	"context"
-	"crypto/md5"
-	"fmt"
-	"path"
-	"sort"
-	"strings"
 
 	"github.com/pkg/errors"
 	appsv1 "k8s.io/api/apps/v1"
-	corev1 "k8s.io/api/core/v1"
-	quantity "k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/types"
-	"k8s.io/apimachinery/pkg/util/intstr"
-	"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3"
-	"k8s.io/kubernetes/cmd/kubeadm/app/constants"
-	"k8s.io/utils/pointer"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 
 	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+	builder "github.com/clastix/kamaji/internal/builders/controlplane"
+	"github.com/clastix/kamaji/internal/types"
 	"github.com/clastix/kamaji/internal/utilities"
 )
 
 type KubernetesDeploymentResource struct {
 	resource               *appsv1.Deployment
 	Client                 client.Client
+	ETCDStorageType        types.ETCDStorageType
 	ETCDEndpoints          []string
 	ETCDCompactionInterval string
 	Name                   string
+	KineContainerImage     string
 }
 
 func (r *KubernetesDeploymentResource) isStatusEqual(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
 	return r.resource.Status.String() == tenantControlPlane.Status.Kubernetes.Deployment.DeploymentStatus.String()
 }
 
-func (r *KubernetesDeploymentResource) ShouldStatusBeUpdated(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
+func (r *KubernetesDeploymentResource) ShouldStatusBeUpdated(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
 	return !r.isStatusEqual(tenantControlPlane) || tenantControlPlane.Spec.Kubernetes.Version != tenantControlPlane.Status.Kubernetes.Version.Version
 }
 
-func (r *KubernetesDeploymentResource) ShouldCleanup(plane *kamajiv1alpha1.TenantControlPlane) bool {
+func (r *KubernetesDeploymentResource) ShouldCleanup(*kamajiv1alpha1.TenantControlPlane) bool {
 	return false
 }
 
-func (r *KubernetesDeploymentResource) CleanUp(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) {
+func (r *KubernetesDeploymentResource) CleanUp(context.Context, *kamajiv1alpha1.TenantControlPlane) (bool, error) {
 	return false, nil
 }
 
-func (r *KubernetesDeploymentResource) Define(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+func (r *KubernetesDeploymentResource) Define(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
 	r.resource = &appsv1.Deployment{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      tenantControlPlane.GetName(),
 			Namespace: tenantControlPlane.GetNamespace(),
-			Labels:    utilities.CommonLabels(tenantControlPlane.GetName()),
 		},
 	}
 
@@ -66,497 +57,43 @@ func (r *KubernetesDeploymentResource) Define(ctx context.Context, tenantControl
 	return nil
 }
 
-// secretHashValue function returns the md5 value for the given secret:
-// this will trigger a new rollout in case of value change.
-func (r *KubernetesDeploymentResource) secretHashValue(ctx context.Context, namespace, name string) (string, error) {
-	secret := &corev1.Secret{}
+func (r *KubernetesDeploymentResource) mutate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) controllerutil.MutateFn {
+	return func() error {
+		address, _, err := tenantControlPlane.AssignedControlPlaneAddress()
+		if err != nil {
+			return errors.Wrap(err, "cannot create TenantControlPlane Deployment")
+		}
 
-	if err := r.Client.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, secret); err != nil {
-		return "", errors.Wrap(err, "cannot retrieve *corev1.Secret for resource version retrieval")
+		d := builder.Deployment{
+			Address:                address,
+			ETCDEndpoints:          r.ETCDEndpoints,
+			ETCDCompactionInterval: r.ETCDCompactionInterval,
+			ETCDStorageType:        r.ETCDStorageType,
+			KineContainerImage:     r.KineContainerImage,
+		}
+		d.SetLabels(r.resource, utilities.MergeMaps(utilities.CommonLabels(tenantControlPlane.GetName()), tenantControlPlane.Spec.ControlPlane.Deployment.AdditionalMetadata.Labels))
+		d.SetAnnotations(r.resource, utilities.MergeMaps(r.resource.Annotations, tenantControlPlane.Spec.ControlPlane.Deployment.AdditionalMetadata.Annotations))
+		d.SetTemplateLabels(&r.resource.Spec.Template, r.deploymentTemplateLabels(ctx, tenantControlPlane))
+		d.SetStrategy(&r.resource.Spec)
+		d.SetSelector(&r.resource.Spec, tenantControlPlane)
+		d.SetReplicas(&r.resource.Spec, tenantControlPlane)
+		d.ResetKubeAPIServerFlags(r.resource, tenantControlPlane)
+		d.SetContainers(&r.resource.Spec.Template.Spec, tenantControlPlane, address)
+		d.SetVolumes(&r.resource.Spec.Template.Spec, tenantControlPlane)
+
+		return controllerutil.SetControllerReference(tenantControlPlane, r.resource, r.Client.Scheme())
 	}
-	// Go access map values in random way, it means we have to sort them
-	keys := make([]string, 0, len(secret.Data))
-
-	for k := range secret.Data {
-		keys = append(keys, k)
-	}
-
-	sort.Strings(keys)
-	// Generating MD5 of Secret values, sorted by key
-	h := md5.New()
-
-	for _, key := range keys {
-		h.Write(secret.Data[key])
-	}
-
-	return fmt.Sprintf("%x", h.Sum(nil)), nil
 }
 
 func (r *KubernetesDeploymentResource) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) {
-	maxSurge := intstr.FromString("100%")
-
-	maxUnavailable := intstr.FromInt(0)
-
-	etcdEndpoints := make([]string, len(r.ETCDEndpoints))
-	for i, v := range r.ETCDEndpoints {
-		etcdEndpoints[i] = fmt.Sprintf("https://%s", v)
-	}
-
-	address, err := tenantControlPlane.GetAddress(ctx, r.Client)
-	if err != nil {
-		return controllerutil.OperationResultNone, errors.Wrap(err, "cannot create TenantControlPlane Deployment")
-	}
-
-	return controllerutil.CreateOrUpdate(ctx, r.Client, r.resource, func() error {
-		labels := utilities.MergeMaps(r.resource.GetLabels(), tenantControlPlane.Spec.ControlPlane.Deployment.AdditionalMetadata.Labels)
-		r.resource.SetLabels(labels)
-
-		annotations := utilities.MergeMaps(r.resource.GetAnnotations(), tenantControlPlane.Spec.ControlPlane.Deployment.AdditionalMetadata.Annotations)
-		r.resource.SetAnnotations(annotations)
-
-		r.resource.Spec.Replicas = pointer.Int32(tenantControlPlane.Spec.ControlPlane.Deployment.Replicas)
-		r.resource.Spec.Selector = &metav1.LabelSelector{
-			MatchLabels: map[string]string{
-				"kamaji.clastix.io/soot": tenantControlPlane.GetName(),
-			},
-		}
-		r.resource.Spec.Template.ObjectMeta = metav1.ObjectMeta{
-			Labels: map[string]string{
-				"kamaji.clastix.io/soot": tenantControlPlane.GetName(),
-				"component.kamaji.clastix.io/api-server-certificate": func() (hash string) {
-					hash, _ = r.secretHashValue(ctx, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.Certificates.APIServer.SecretName)
-
-					return
-				}(),
-				"component.kamaji.clastix.io/api-server-kubelet-client-certificate": func() (hash string) {
-					hash, _ = r.secretHashValue(ctx, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.Certificates.APIServerKubeletClient.SecretName)
-
-					return
-				}(),
-				"component.kamaji.clastix.io/ca": func() (hash string) {
-					hash, _ = r.secretHashValue(ctx, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.Certificates.CA.SecretName)
-
-					return
-				}(),
-				"component.kamaji.clastix.io/controller-manager-kubeconfig": func() (hash string) {
-					hash, _ = r.secretHashValue(ctx, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.KubeConfig.ControllerManager.SecretName)
-
-					return
-				}(),
-				"component.kamaji.clastix.io/etcd-ca-certificates": func() (hash string) {
-					hash, _ = r.secretHashValue(ctx, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.Certificates.ETCD.CA.SecretName)
-
-					return
-				}(),
-				"component.kamaji.clastix.io/etcd-certificates": func() (hash string) {
-					hash, _ = r.secretHashValue(ctx, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.Certificates.ETCD.APIServer.SecretName)
-
-					return
-				}(),
-				"component.kamaji.clastix.io/front-proxy-ca-certificate": func() (hash string) {
-					hash, _ = r.secretHashValue(ctx, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.Certificates.FrontProxyCA.SecretName)
-
-					return
-				}(),
-				"component.kamaji.clastix.io/front-proxy-client-certificate": func() (hash string) {
-					hash, _ = r.secretHashValue(ctx, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.Certificates.FrontProxyClient.SecretName)
-
-					return
-				}(),
-				"component.kamaji.clastix.io/service-account": func() (hash string) {
-					hash, _ = r.secretHashValue(ctx, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.Certificates.SA.SecretName)
-
-					return
-				}(),
-				"component.kamaji.clastix.io/scheduler-kubeconfig": func() (hash string) {
-					hash, _ = r.secretHashValue(ctx, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.KubeConfig.Scheduler.SecretName)
-
-					return
-				}(),
-			},
-		}
-		r.resource.Spec.Template.Spec.Volumes = []corev1.Volume{
-			{
-				Name: "etc-kubernetes-pki",
-				VolumeSource: corev1.VolumeSource{
-					Projected: &corev1.ProjectedVolumeSource{
-						Sources: []corev1.VolumeProjection{
-							{
-								Secret: secretProjection(tenantControlPlane.Status.Certificates.APIServer.SecretName, constants.APIServerCertName, constants.APIServerKeyName),
-							},
-							{
-								Secret: secretProjection(tenantControlPlane.Status.Certificates.CA.SecretName, constants.CACertName, constants.CAKeyName),
-							},
-							{
-								Secret: secretProjection(tenantControlPlane.Status.Certificates.APIServerKubeletClient.SecretName, constants.APIServerKubeletClientCertName, constants.APIServerKubeletClientKeyName),
-							},
-							{
-								Secret: secretProjection(tenantControlPlane.Status.Certificates.FrontProxyCA.SecretName, constants.FrontProxyCACertName, constants.FrontProxyCAKeyName),
-							},
-							{
-								Secret: secretProjection(tenantControlPlane.Status.Certificates.FrontProxyClient.SecretName, constants.FrontProxyClientCertName, constants.FrontProxyClientKeyName),
-							},
-							{
-								Secret: secretProjection(tenantControlPlane.Status.Certificates.SA.SecretName, constants.ServiceAccountPublicKeyName, constants.ServiceAccountPrivateKeyName),
-							},
-							{
-								Secret: secretProjection(tenantControlPlane.Status.Certificates.ETCD.APIServer.SecretName, constants.APIServerEtcdClientCertName, constants.APIServerEtcdClientKeyName),
-							},
-							{
-								Secret: &corev1.SecretProjection{
-									LocalObjectReference: corev1.LocalObjectReference{
-										Name: tenantControlPlane.Status.Certificates.ETCD.CA.SecretName,
-									},
-									Items: []corev1.KeyToPath{
-										{
-											Key:  constants.CACertName,
-											Path: constants.EtcdCACertName,
-										},
-									},
-								},
-							},
-						},
-						DefaultMode: pointer.Int32Ptr(420),
-					},
-				},
-			},
-			{
-				Name: "etc-ca-certificates",
-				VolumeSource: corev1.VolumeSource{
-					Secret: &corev1.SecretVolumeSource{
-						SecretName:  tenantControlPlane.Status.Certificates.CA.SecretName,
-						DefaultMode: pointer.Int32Ptr(420),
-					},
-				},
-			},
-			{
-				Name: "etc-ssl-certs",
-				VolumeSource: corev1.VolumeSource{
-					Secret: &corev1.SecretVolumeSource{
-						SecretName:  tenantControlPlane.Status.Certificates.CA.SecretName,
-						DefaultMode: pointer.Int32Ptr(420),
-					},
-				},
-			},
-			{
-				Name: "usr-share-ca-certificates",
-				VolumeSource: corev1.VolumeSource{
-					Secret: &corev1.SecretVolumeSource{
-						SecretName:  tenantControlPlane.Status.Certificates.CA.SecretName,
-						DefaultMode: pointer.Int32Ptr(420),
-					},
-				},
-			},
-			{
-				Name: "usr-local-share-ca-certificates",
-				VolumeSource: corev1.VolumeSource{
-					Secret: &corev1.SecretVolumeSource{
-						SecretName:  tenantControlPlane.Status.Certificates.CA.SecretName,
-						DefaultMode: pointer.Int32Ptr(420),
-					},
-				},
-			},
-			{
-				Name: "scheduler-kubeconfig",
-				VolumeSource: corev1.VolumeSource{
-					Secret: &corev1.SecretVolumeSource{
-						SecretName:  tenantControlPlane.Status.KubeConfig.Scheduler.SecretName,
-						DefaultMode: pointer.Int32Ptr(420),
-					},
-				},
-			},
-			{
-				Name: "controller-manager-kubeconfig",
-				VolumeSource: corev1.VolumeSource{
-					Secret: &corev1.SecretVolumeSource{
-						SecretName:  tenantControlPlane.Status.KubeConfig.ControllerManager.SecretName,
-						DefaultMode: pointer.Int32Ptr(420),
-					},
-				},
-			},
-		}
-
-		r.resource.Spec.Template.Spec.Containers = []corev1.Container{
-			{
-				Name:  "kube-apiserver",
-				Image: fmt.Sprintf("k8s.gcr.io/kube-apiserver:%s", tenantControlPlane.Spec.Kubernetes.Version),
-				Command: []string{
-					"kube-apiserver",
-					"--allow-privileged=true",
-					"--authorization-mode=Node,RBAC",
-					fmt.Sprintf("--advertise-address=%s", address),
-					fmt.Sprintf("--client-ca-file=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.CACertName)),
-					fmt.Sprintf("--enable-admission-plugins=%s", strings.Join(tenantControlPlane.Spec.Kubernetes.AdmissionControllers.ToSlice(), ",")),
-					"--enable-bootstrap-token-auth=true",
-					fmt.Sprintf("--etcd-compaction-interval=%s", r.ETCDCompactionInterval),
-					fmt.Sprintf("--etcd-cafile=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.EtcdCACertName)),
-					fmt.Sprintf("--etcd-certfile=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.APIServerEtcdClientCertName)),
-					fmt.Sprintf("--etcd-keyfile=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.APIServerEtcdClientKeyName)),
-					fmt.Sprintf("--etcd-servers=%s", strings.Join(etcdEndpoints, ",")),
-					fmt.Sprintf("--etcd-prefix=/%s", tenantControlPlane.GetName()),
-					fmt.Sprintf("--service-cluster-ip-range=%s", tenantControlPlane.Spec.NetworkProfile.ServiceCIDR),
-					fmt.Sprintf("--kubelet-client-certificate=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.APIServerKubeletClientCertName)),
-					fmt.Sprintf("--kubelet-client-key=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.APIServerKubeletClientKeyName)),
-					"--kubelet-preferred-address-types=Hostname,InternalIP,ExternalIP",
-					fmt.Sprintf("--proxy-client-cert-file=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.FrontProxyClientCertName)),
-					fmt.Sprintf("--proxy-client-key-file=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.FrontProxyClientKeyName)),
-					"--requestheader-allowed-names=front-proxy-client",
-					"--requestheader-extra-headers-prefix=X-Remote-Extra-",
-					"--requestheader-group-headers=X-Remote-Group",
-					"--requestheader-username-headers=X-Remote-User",
-					fmt.Sprintf("--secure-port=%d", tenantControlPlane.Spec.NetworkProfile.Port),
-					fmt.Sprintf("--service-account-issuer=https://localhost:%d", tenantControlPlane.Spec.NetworkProfile.Port),
-					fmt.Sprintf("--service-account-key-file=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.ServiceAccountPublicKeyName)),
-					fmt.Sprintf("--service-account-signing-key-file=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.ServiceAccountPrivateKeyName)),
-					fmt.Sprintf("--tls-cert-file=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.APIServerCertName)),
-					fmt.Sprintf("--tls-private-key-file=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.APIServerKeyName)),
-				},
-				Resources: corev1.ResourceRequirements{
-					Limits: nil,
-					Requests: corev1.ResourceList{
-						corev1.ResourceCPU: quantity.MustParse("250m"),
-					},
-				},
-				LivenessProbe: &corev1.Probe{
-					ProbeHandler: corev1.ProbeHandler{
-						HTTPGet: &corev1.HTTPGetAction{
-							Path:   "/livez",
-							Port:   intstr.FromInt(int(tenantControlPlane.Spec.NetworkProfile.Port)),
-							Scheme: corev1.URISchemeHTTPS,
-						},
-					},
-					InitialDelaySeconds: 0,
-					TimeoutSeconds:      1,
-					PeriodSeconds:       10,
-					SuccessThreshold:    1,
-					FailureThreshold:    3,
-				},
-				ReadinessProbe: &corev1.Probe{
-					ProbeHandler: corev1.ProbeHandler{
-						HTTPGet: &corev1.HTTPGetAction{
-							Path:   "/readyz",
-							Port:   intstr.FromInt(int(tenantControlPlane.Spec.NetworkProfile.Port)),
-							Scheme: corev1.URISchemeHTTPS,
-						},
-					},
-					InitialDelaySeconds: 0,
-					TimeoutSeconds:      1,
-					PeriodSeconds:       10,
-					SuccessThreshold:    1,
-					FailureThreshold:    3,
-				},
-				StartupProbe: &corev1.Probe{
-					ProbeHandler: corev1.ProbeHandler{
-						HTTPGet: &corev1.HTTPGetAction{
-							Path:   "/livez",
-							Port:   intstr.FromInt(int(tenantControlPlane.Spec.NetworkProfile.Port)),
-							Scheme: corev1.URISchemeHTTPS,
-						},
-					},
-					InitialDelaySeconds: 0,
-					TimeoutSeconds:      1,
-					PeriodSeconds:       10,
-					SuccessThreshold:    1,
-					FailureThreshold:    3,
-				},
-				TerminationMessagePath:   "/dev/termination-log",
-				TerminationMessagePolicy: "File",
-				ImagePullPolicy:          corev1.PullAlways,
-				VolumeMounts: []corev1.VolumeMount{
-					{
-						Name:      "etc-kubernetes-pki",
-						ReadOnly:  true,
-						MountPath: v1beta3.DefaultCertificatesDir,
-					},
-					{
-						Name:      "etc-ca-certificates",
-						ReadOnly:  true,
-						MountPath: "/etc/ca-certificates",
-					},
-					{
-						Name:      "etc-ssl-certs",
-						ReadOnly:  true,
-						MountPath: "/etc/ssl/certs",
-					},
-					{
-						Name:      "usr-share-ca-certificates",
-						ReadOnly:  true,
-						MountPath: "/usr/share/ca-certificates",
-					},
-					{
-						Name:      "usr-local-share-ca-certificates",
-						ReadOnly:  true,
-						MountPath: "/usr/local/share/ca-certificates",
-					},
-				},
-			},
-			{
-				Name:  "kube-scheduler",
-				Image: fmt.Sprintf("k8s.gcr.io/kube-scheduler:%s", tenantControlPlane.Spec.Kubernetes.Version),
-				Command: []string{
-					"kube-scheduler",
-					"--authentication-kubeconfig=/etc/kubernetes/scheduler.conf",
-					"--authorization-kubeconfig=/etc/kubernetes/scheduler.conf",
-					"--bind-address=0.0.0.0",
-					"--kubeconfig=/etc/kubernetes/scheduler.conf",
-					"--leader-elect=true",
-				},
-				Resources: corev1.ResourceRequirements{
-					Limits: nil,
-					Requests: corev1.ResourceList{
-						corev1.ResourceCPU: quantity.MustParse("100m"),
-					},
-				},
-				VolumeMounts: []corev1.VolumeMount{
-					{
-						Name:      "scheduler-kubeconfig",
-						ReadOnly:  true,
-						MountPath: "/etc/kubernetes",
-					},
-				},
-				LivenessProbe: &corev1.Probe{
-					ProbeHandler: corev1.ProbeHandler{
-						HTTPGet: &corev1.HTTPGetAction{
-							Path:   "/healthz",
-							Port:   intstr.FromInt(10259),
-							Scheme: corev1.URISchemeHTTPS,
-						},
-					},
-					InitialDelaySeconds: 0,
-					TimeoutSeconds:      1,
-					PeriodSeconds:       10,
-					SuccessThreshold:    1,
-					FailureThreshold:    3,
-				},
-				StartupProbe: &corev1.Probe{
-					ProbeHandler: corev1.ProbeHandler{
-						HTTPGet: &corev1.HTTPGetAction{
-							Path:   "/healthz",
-							Port:   intstr.FromInt(10259),
-							Scheme: corev1.URISchemeHTTPS,
-						},
-					},
-					InitialDelaySeconds: 0,
-					TimeoutSeconds:      1,
-					PeriodSeconds:       10,
-					SuccessThreshold:    1,
-					FailureThreshold:    3,
-				},
-				TerminationMessagePath:   "/dev/termination-log",
-				TerminationMessagePolicy: "File",
-				ImagePullPolicy:          corev1.PullIfNotPresent,
-			},
-			{
-				Name:  "kube-controller-manager",
-				Image: fmt.Sprintf("k8s.gcr.io/kube-controller-manager:%s", tenantControlPlane.Spec.Kubernetes.Version),
-				Command: []string{
-					"kube-controller-manager",
-					"--allocate-node-cidrs=true",
-					"--authentication-kubeconfig=/etc/kubernetes/controller-manager.conf",
-					"--authorization-kubeconfig=/etc/kubernetes/controller-manager.conf",
-					"--bind-address=0.0.0.0",
-					fmt.Sprintf("--client-ca-file=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.CACertName)),
-					fmt.Sprintf("--cluster-name=%s", tenantControlPlane.GetName()),
-					fmt.Sprintf("--cluster-signing-cert-file=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.CACertName)),
-					fmt.Sprintf("--cluster-signing-key-file=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.CAKeyName)),
-					"--controllers=*,bootstrapsigner,tokencleaner",
-					"--kubeconfig=/etc/kubernetes/controller-manager.conf",
-					"--leader-elect=true",
-					fmt.Sprintf("--service-cluster-ip-range=%s", tenantControlPlane.Spec.NetworkProfile.ServiceCIDR),
-					fmt.Sprintf("--cluster-cidr=%s", tenantControlPlane.Spec.NetworkProfile.PodCIDR),
-					fmt.Sprintf("--requestheader-client-ca-file=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.FrontProxyCACertName)),
-					fmt.Sprintf("--root-ca-file=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.CACertName)),
-					fmt.Sprintf("--service-account-private-key-file=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.ServiceAccountPrivateKeyName)),
-					"--use-service-account-credentials=true",
-				},
-				Resources: corev1.ResourceRequirements{
-					Limits: nil,
-					Requests: corev1.ResourceList{
-						corev1.ResourceCPU: quantity.MustParse("200m"),
-					},
-				},
-				VolumeMounts: []corev1.VolumeMount{
-					{
-						Name:      "controller-manager-kubeconfig",
-						ReadOnly:  true,
-						MountPath: "/etc/kubernetes",
-					},
-					{
-						Name:      "etc-kubernetes-pki",
-						ReadOnly:  true,
-						MountPath: v1beta3.DefaultCertificatesDir,
-					},
-					{
-						Name:      "etc-ca-certificates",
-						ReadOnly:  true,
-						MountPath: "/etc/ca-certificates",
-					},
-					{
-						Name:      "etc-ssl-certs",
-						ReadOnly:  true,
-						MountPath: "/etc/ssl/certs",
-					},
-					{
-						Name:      "usr-share-ca-certificates",
-						ReadOnly:  true,
-						MountPath: "/usr/share/ca-certificates",
-					},
-					{
-						Name:      "usr-local-share-ca-certificates",
-						ReadOnly:  true,
-						MountPath: "/usr/local/share/ca-certificates",
-					},
-				},
-				LivenessProbe: &corev1.Probe{
-					ProbeHandler: corev1.ProbeHandler{
-						HTTPGet: &corev1.HTTPGetAction{
-							Path:   "/healthz",
-							Port:   intstr.FromInt(10257),
-							Scheme: corev1.URISchemeHTTPS,
-						},
-					},
-					InitialDelaySeconds: 0,
-					TimeoutSeconds:      1,
-					PeriodSeconds:       10,
-					SuccessThreshold:    1,
-					FailureThreshold:    3,
-				},
-				StartupProbe: &corev1.Probe{
-					ProbeHandler: corev1.ProbeHandler{
-						HTTPGet: &corev1.HTTPGetAction{
-							Path:   "/healthz",
-							Port:   intstr.FromInt(10257),
-							Scheme: corev1.URISchemeHTTPS,
-						},
-					},
-					InitialDelaySeconds: 0,
-					TimeoutSeconds:      1,
-					PeriodSeconds:       10,
-					SuccessThreshold:    1,
-					FailureThreshold:    3,
-				},
-				TerminationMessagePath:   "/dev/termination-log",
-				TerminationMessagePolicy: "File",
-				ImagePullPolicy:          corev1.PullIfNotPresent,
-			},
-		}
-		r.resource.Spec.Strategy = appsv1.DeploymentStrategy{
-			Type: appsv1.RollingUpdateDeploymentStrategyType,
-			RollingUpdate: &appsv1.RollingUpdateDeployment{
-				MaxUnavailable: &maxUnavailable,
-				MaxSurge:       &maxSurge,
-			},
-		}
-
-		return controllerutil.SetControllerReference(tenantControlPlane, r.resource, r.Client.Scheme())
-	})
+	return utilities.CreateOrUpdateWithConflict(ctx, r.Client, r.resource, r.mutate(ctx, tenantControlPlane))
 }
 
 func (r *KubernetesDeploymentResource) GetName() string {
 	return r.Name
 }
 
-func (r *KubernetesDeploymentResource) UpdateTenantControlPlaneStatus(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+func (r *KubernetesDeploymentResource) UpdateTenantControlPlaneStatus(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
 	switch {
 	case !r.isProgressingUpgrade():
 		tenantControlPlane.Status.Kubernetes.Version.Status = &kamajiv1alpha1.VersionReady
@@ -573,11 +110,39 @@ func (r *KubernetesDeploymentResource) UpdateTenantControlPlaneStatus(ctx contex
 		DeploymentStatus: r.resource.Status,
 		Name:             r.resource.GetName(),
 		Namespace:        r.resource.GetNamespace(),
+		LastUpdate:       metav1.Now(),
 	}
 
 	return nil
 }
 
+func (r *KubernetesDeploymentResource) deploymentTemplateLabels(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (labels map[string]string) {
+	hash := func(ctx context.Context, namespace, secretName string) (hash string) {
+		hash, _ = utilities.SecretHashValue(ctx, r.Client, namespace, secretName)
+
+		return
+	}
+
+	labels = map[string]string{
+		"kamaji.clastix.io/soot":                                            tenantControlPlane.GetName(),
+		"component.kamaji.clastix.io/api-server-certificate":                hash(ctx, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.Certificates.APIServer.SecretName),
+		"component.kamaji.clastix.io/api-server-kubelet-client-certificate": hash(ctx, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.Certificates.APIServerKubeletClient.SecretName),
+		"component.kamaji.clastix.io/ca":                                    hash(ctx, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.Certificates.CA.SecretName),
+		"component.kamaji.clastix.io/controller-manager-kubeconfig":         hash(ctx, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.KubeConfig.ControllerManager.SecretName),
+		"component.kamaji.clastix.io/front-proxy-ca-certificate":            hash(ctx, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.Certificates.FrontProxyCA.SecretName),
+		"component.kamaji.clastix.io/front-proxy-client-certificate":        hash(ctx, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.Certificates.FrontProxyClient.SecretName),
+		"component.kamaji.clastix.io/service-account":                       hash(ctx, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.Certificates.SA.SecretName),
+		"component.kamaji.clastix.io/scheduler-kubeconfig":                  hash(ctx, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.KubeConfig.Scheduler.SecretName),
+	}
+
+	if r.ETCDStorageType == types.ETCD {
+		labels["component.kamaji.clastix.io/etcd-ca-certificates"] = hash(ctx, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.Certificates.ETCD.CA.SecretName)
+		labels["component.kamaji.clastix.io/etcd-certificates"] = hash(ctx, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.Certificates.ETCD.APIServer.SecretName)
+	}
+
+	return labels
+}
+
 func (r *KubernetesDeploymentResource) isProgressingUpgrade() bool {
 	if r.resource.ObjectMeta.GetGeneration() != r.resource.Status.ObservedGeneration {
 		return true

internal/resources/k8s_ingress_resource.go

@@ -73,8 +73,8 @@ func (r *KubernetesIngressResource) Define(ctx context.Context, tenantControlPla
 	return nil
 }
 
-func (r *KubernetesIngressResource) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) {
-	return controllerutil.CreateOrUpdate(ctx, r.Client, r.resource, func() error {
+func (r *KubernetesIngressResource) mutate(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) controllerutil.MutateFn {
+	return func() error {
 		labels := utilities.MergeMaps(r.resource.GetLabels(), tenantControlPlane.Spec.ControlPlane.Ingress.AdditionalMetadata.Labels)
 		r.resource.SetLabels(labels)
 
@@ -119,17 +119,23 @@ func (r *KubernetesIngressResource) CreateOrUpdate(ctx context.Context, tenantCo
 		}
 
 		rule.HTTP.Paths[0] = path
-		rule.Host = tenantControlPlane.Spec.ControlPlane.Ingress.Hostname
-		if rule.Host == "" {
-			rule.Host = fmt.Sprintf("%s.%s.%s", tenantControlPlane.GetName(), tenantControlPlane.GetNamespace(), tenantControlPlane.Spec.NetworkProfile.Domain)
+
+		if len(tenantControlPlane.Spec.ControlPlane.Ingress.Hostname) == 0 {
+			return fmt.Errorf("missing hostname to expose the Tenant Control Plane using an Ingress resource")
 		}
 
+		rule.Host = tenantControlPlane.Spec.ControlPlane.Ingress.Hostname
+
 		r.resource.Spec.Rules = []networkingv1.IngressRule{
 			rule,
 		}
 
 		return controllerutil.SetControllerReference(tenantControlPlane, r.resource, r.Client.Scheme())
-	})
+	}
+}
+
+func (r *KubernetesIngressResource) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) {
+	return utilities.CreateOrUpdateWithConflict(ctx, r.Client, r.resource, r.mutate(tenantControlPlane))
 }
 
 func (r *KubernetesIngressResource) GetName() string {

internal/resources/k8s_service_resource.go

@@ -5,6 +5,7 @@ package resources
 
 import (
 	"context"
+	"fmt"
 
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -44,6 +45,13 @@ func (r *KubernetesServiceResource) UpdateTenantControlPlaneStatus(ctx context.C
 	tenantControlPlane.Status.Kubernetes.Service.Namespace = r.resource.GetNamespace()
 	tenantControlPlane.Status.Kubernetes.Service.Port = r.resource.Spec.Ports[0].Port
 
+	address, err := tenantControlPlane.DeclaredControlPlaneAddress(ctx, r.Client)
+	if err != nil {
+		return err
+	}
+
+	tenantControlPlane.Status.ControlPlaneEndpoint = fmt.Sprintf("%s:%d", address, tenantControlPlane.Spec.NetworkProfile.Port)
+
 	return nil
 }
 
@@ -52,7 +60,6 @@ func (r *KubernetesServiceResource) Define(ctx context.Context, tenantControlPla
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      tenantControlPlane.GetName(),
 			Namespace: tenantControlPlane.GetNamespace(),
-			Labels:    utilities.CommonLabels(tenantControlPlane.GetName()),
 		},
 	}
 
@@ -62,30 +69,34 @@ func (r *KubernetesServiceResource) Define(ctx context.Context, tenantControlPla
 }
 
 func (r *KubernetesServiceResource) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) {
+	return utilities.CreateOrUpdateWithConflict(ctx, r.Client, r.resource, r.mutate(ctx, tenantControlPlane))
+}
+
+func (r *KubernetesServiceResource) mutate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) controllerutil.MutateFn {
 	// We don't need to check error here: in case of dynamic external IP, the Service must be created in advance.
 	// After that, the specific cloud controller-manager will provide an IP that will be then used.
-	address, _ := tenantControlPlane.GetAddress(ctx, r.Client)
+	address, _ := tenantControlPlane.DeclaredControlPlaneAddress(ctx, r.Client)
 
-	return controllerutil.CreateOrUpdate(ctx, r.Client, r.resource, func() error {
-		var servicePort corev1.ServicePort
-		if len(r.resource.Spec.Ports) > 0 {
-			servicePort = r.resource.Spec.Ports[0]
-		}
-		servicePort.Protocol = corev1.ProtocolTCP
-		servicePort.Port = tenantControlPlane.Spec.NetworkProfile.Port
-		servicePort.TargetPort = intstr.FromInt(int(tenantControlPlane.Spec.NetworkProfile.Port))
-
-		r.resource.Spec.Ports = []corev1.ServicePort{servicePort}
-		r.resource.Spec.Selector = map[string]string{
-			"kamaji.clastix.io/soot": tenantControlPlane.GetName(),
-		}
-
-		labels := utilities.MergeMaps(r.resource.GetLabels(), tenantControlPlane.Spec.ControlPlane.Service.AdditionalMetadata.Labels)
+	return func() error {
+		labels := utilities.MergeMaps(utilities.CommonLabels(tenantControlPlane.GetName()), tenantControlPlane.Spec.ControlPlane.Service.AdditionalMetadata.Labels)
 		r.resource.SetLabels(labels)
 
 		annotations := utilities.MergeMaps(r.resource.GetAnnotations(), tenantControlPlane.Spec.ControlPlane.Service.AdditionalMetadata.Annotations)
 		r.resource.SetAnnotations(annotations)
 
+		r.resource.Spec.Selector = map[string]string{
+			"kamaji.clastix.io/soot": tenantControlPlane.GetName(),
+		}
+
+		if len(r.resource.Spec.Ports) == 0 {
+			r.resource.Spec.Ports = make([]corev1.ServicePort, 1)
+		}
+
+		r.resource.Spec.Ports[0].Name = "kube-apiserver"
+		r.resource.Spec.Ports[0].Protocol = corev1.ProtocolTCP
+		r.resource.Spec.Ports[0].Port = tenantControlPlane.Spec.NetworkProfile.Port
+		r.resource.Spec.Ports[0].TargetPort = intstr.FromInt(int(tenantControlPlane.Spec.NetworkProfile.Port))
+
 		switch tenantControlPlane.Spec.ControlPlane.Service.ServiceType {
 		case kamajiv1alpha1.ServiceTypeLoadBalancer:
 			r.resource.Spec.Type = corev1.ServiceTypeLoadBalancer
@@ -109,7 +120,7 @@ func (r *KubernetesServiceResource) CreateOrUpdate(ctx context.Context, tenantCo
 		}
 
 		return controllerutil.SetControllerReference(tenantControlPlane, r.resource, r.Client.Scheme())
-	})
+	}
 }
 
 func (r *KubernetesServiceResource) GetName() string {

internal/resources/konnectivity/agent.go

@@ -0,0 +1,201 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package konnectivity
+
+import (
+	"context"
+	"fmt"
+
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/utils/pointer"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+	"github.com/clastix/kamaji/internal/utilities"
+)
+
+const (
+	agentNamespace = "kube-system"
+)
+
+type Agent struct {
+	resource     *appsv1.DaemonSet
+	Client       client.Client
+	Name         string
+	tenantClient client.Client
+}
+
+func (r *Agent) ShouldStatusBeUpdated(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
+	return tenantControlPlane.Status.Addons.Konnectivity.Agent.Name != r.resource.GetName() ||
+		tenantControlPlane.Status.Addons.Konnectivity.Agent.Namespace != r.resource.GetNamespace() ||
+		tenantControlPlane.Status.Addons.Konnectivity.Agent.RV != r.resource.ObjectMeta.ResourceVersion
+}
+
+func (r *Agent) ShouldCleanup(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
+	return tenantControlPlane.Spec.Addons.Konnectivity == nil
+}
+
+func (r *Agent) CleanUp(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) {
+	if err := r.tenantClient.Delete(ctx, r.resource); err != nil {
+		if !k8serrors.IsNotFound(err) {
+			return false, err
+		}
+
+		return false, nil
+	}
+
+	return true, nil
+}
+
+func (r *Agent) Define(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+	r.resource = &appsv1.DaemonSet{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      AgentName,
+			Namespace: agentNamespace,
+		},
+	}
+
+	client, err := utilities.GetTenantClient(ctx, r.Client, tenantControlPlane)
+	if err != nil {
+		return err
+	}
+
+	r.tenantClient = client
+
+	return nil
+}
+
+func (r *Agent) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) {
+	return controllerutil.CreateOrUpdate(ctx, r.tenantClient, r.resource, r.mutate(ctx, tenantControlPlane))
+}
+
+func (r *Agent) GetName() string {
+	return r.Name
+}
+
+func (r *Agent) UpdateTenantControlPlaneStatus(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+	if tenantControlPlane.Spec.Addons.Konnectivity != nil {
+		tenantControlPlane.Status.Addons.Konnectivity.Agent = kamajiv1alpha1.ExternalKubernetesObjectStatus{
+			Name:       r.resource.GetName(),
+			Namespace:  r.resource.GetNamespace(),
+			RV:         r.resource.ObjectMeta.ResourceVersion,
+			LastUpdate: metav1.Now(),
+		}
+		tenantControlPlane.Status.Addons.Konnectivity.Enabled = true
+
+		return nil
+	}
+
+	tenantControlPlane.Status.Addons.Konnectivity.Enabled = false
+	tenantControlPlane.Status.Addons.Konnectivity.Agent = kamajiv1alpha1.ExternalKubernetesObjectStatus{}
+
+	return nil
+}
+
+func (r *Agent) mutate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) controllerutil.MutateFn {
+	return func() (err error) {
+		address, _, err := tenantControlPlane.AssignedControlPlaneAddress()
+		if err != nil {
+			return err
+		}
+
+		r.resource.SetLabels(utilities.MergeMaps(
+			utilities.KamajiLabels(),
+			map[string]string{
+				"k8s-app":                         AgentName,
+				"addonmanager.kubernetes.io/mode": "Reconcile",
+			},
+		))
+
+		if r.resource.Spec.Selector == nil {
+			r.resource.Spec.Selector = &metav1.LabelSelector{}
+		}
+		r.resource.Spec.Selector.MatchLabels = map[string]string{
+			"k8s-app": AgentName,
+		}
+
+		r.resource.Spec.Template.SetLabels(utilities.MergeMaps(
+			r.resource.Spec.Template.GetLabels(),
+			map[string]string{
+				"k8s-app": AgentName,
+			},
+		))
+
+		r.resource.Spec.Template.Spec.PriorityClassName = "system-cluster-critical"
+		r.resource.Spec.Template.Spec.Tolerations = []corev1.Toleration{
+			{
+				Key:      "CriticalAddonsOnly",
+				Operator: "Exists",
+			},
+		}
+		r.resource.Spec.Template.Spec.NodeSelector = map[string]string{
+			"kubernetes.io/os": "linux",
+		}
+		r.resource.Spec.Template.Spec.ServiceAccountName = AgentName
+		r.resource.Spec.Template.Spec.Volumes = []corev1.Volume{
+			{
+				Name: agentTokenName,
+				VolumeSource: corev1.VolumeSource{
+					Projected: &corev1.ProjectedVolumeSource{
+						Sources: []corev1.VolumeProjection{
+							{
+								ServiceAccountToken: &corev1.ServiceAccountTokenProjection{
+									Path:              agentTokenName,
+									Audience:          tenantControlPlane.Status.Addons.Konnectivity.ClusterRoleBinding.Name,
+									ExpirationSeconds: pointer.Int64(3600),
+								},
+							},
+						},
+						DefaultMode: pointer.Int32Ptr(420),
+					},
+				},
+			},
+		}
+
+		if len(r.resource.Spec.Template.Spec.Containers) != 1 {
+			r.resource.Spec.Template.Spec.Containers = make([]corev1.Container, 1)
+		}
+
+		r.resource.Spec.Template.Spec.Containers[0].Image = fmt.Sprintf("%s:%s", tenantControlPlane.Spec.Addons.Konnectivity.AgentImage, tenantControlPlane.Spec.Addons.Konnectivity.Version)
+		r.resource.Spec.Template.Spec.Containers[0].Name = AgentName
+		r.resource.Spec.Template.Spec.Containers[0].Command = []string{"/proxy-agent"}
+		r.resource.Spec.Template.Spec.Containers[0].Args = []string{
+			"-v=8",
+			"--logtostderr=true",
+			"--ca-cert=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt",
+			fmt.Sprintf("--proxy-server-host=%s", address),
+			fmt.Sprintf("--proxy-server-port=%d", tenantControlPlane.Spec.Addons.Konnectivity.ProxyPort),
+			"--admin-server-port=8133",
+			"--health-server-port=8134",
+			"--service-account-token-path=/var/run/secrets/tokens/konnectivity-agent-token",
+		}
+		r.resource.Spec.Template.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{
+			{
+				MountPath: "/var/run/secrets/tokens",
+				Name:      agentTokenName,
+			},
+		}
+		r.resource.Spec.Template.Spec.Containers[0].LivenessProbe = &corev1.Probe{
+			ProbeHandler: corev1.ProbeHandler{
+				HTTPGet: &corev1.HTTPGetAction{
+					Path:   "/healthz",
+					Port:   intstr.FromInt(8134),
+					Scheme: corev1.URISchemeHTTP,
+				},
+			},
+			InitialDelaySeconds: 15,
+			TimeoutSeconds:      15,
+			PeriodSeconds:       10,
+			SuccessThreshold:    1,
+			FailureThreshold:    3,
+		}
+
+		return nil
+	}
+}

internal/resources/konnectivity/certificate_resource.go

@@ -0,0 +1,180 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package konnectivity
+
+import (
+	"bytes"
+	"context"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"fmt"
+	"math/big"
+	"math/rand"
+	"time"
+
+	"github.com/go-logr/logr"
+	corev1 "k8s.io/api/core/v1"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	k8stypes "k8s.io/apimachinery/pkg/types"
+	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+	"github.com/clastix/kamaji/internal/crypto"
+	"github.com/clastix/kamaji/internal/kubeadm"
+	"github.com/clastix/kamaji/internal/utilities"
+)
+
+type CertificateResource struct {
+	resource *corev1.Secret
+	Client   client.Client
+	Log      logr.Logger
+	Name     string
+}
+
+func (r *CertificateResource) ShouldStatusBeUpdated(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
+	return tenantControlPlane.Status.Addons.Konnectivity.Certificate.SecretName != r.resource.GetName() ||
+		tenantControlPlane.Status.Addons.Konnectivity.Certificate.ResourceVersion != r.resource.ResourceVersion
+}
+
+func (r *CertificateResource) ShouldCleanup(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
+	return tenantControlPlane.Spec.Addons.Konnectivity == nil
+}
+
+func (r *CertificateResource) CleanUp(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) {
+	if err := r.Client.Delete(ctx, r.resource); err != nil {
+		if !k8serrors.IsNotFound(err) {
+			return false, err
+		}
+
+		return false, nil
+	}
+
+	return true, nil
+}
+
+func (r *CertificateResource) Define(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+	r.resource = &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      r.getPrefixedName(tenantControlPlane),
+			Namespace: tenantControlPlane.GetNamespace(),
+		},
+	}
+
+	return nil
+}
+
+func (r *CertificateResource) getPrefixedName(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) string {
+	return utilities.AddTenantPrefix(r.Name, tenantControlPlane)
+}
+
+func (r *CertificateResource) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) {
+	return controllerutil.CreateOrUpdate(ctx, r.Client, r.resource, r.mutate(ctx, tenantControlPlane))
+}
+
+func (r *CertificateResource) GetName() string {
+	return r.Name
+}
+
+func (r *CertificateResource) UpdateTenantControlPlaneStatus(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+	if tenantControlPlane.Spec.Addons.Konnectivity != nil {
+		tenantControlPlane.Status.Addons.Konnectivity.Certificate.LastUpdate = metav1.Now()
+		tenantControlPlane.Status.Addons.Konnectivity.Certificate.SecretName = r.resource.GetName()
+		tenantControlPlane.Status.Addons.Konnectivity.Certificate.ResourceVersion = r.resource.ResourceVersion
+		tenantControlPlane.Status.Addons.Konnectivity.Enabled = true
+
+		return nil
+	}
+
+	tenantControlPlane.Status.Addons.Konnectivity.Certificate = kamajiv1alpha1.CertificatePrivateKeyPairStatus{}
+	tenantControlPlane.Status.Addons.Konnectivity.Enabled = false
+
+	return nil
+}
+
+func (r *CertificateResource) mutate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) controllerutil.MutateFn {
+	return func() error {
+		latestCARV := tenantControlPlane.Status.Certificates.CA.ResourceVersion
+		actualCARV := r.resource.GetLabels()["latest-ca-rv"]
+		if latestCARV == actualCARV {
+			isValid, err := isCertificateAndKeyPairValid(
+				r.resource.Data[corev1.TLSCertKey],
+				r.resource.Data[corev1.TLSPrivateKeyKey],
+			)
+			if err != nil {
+				r.Log.Info(fmt.Sprintf("%s certificate-private_key pair is not valid: %s", konnectivityCertAndKeyBaseName, err.Error()))
+			}
+			if isValid {
+				return nil
+			}
+		}
+
+		namespacedName := k8stypes.NamespacedName{Namespace: tenantControlPlane.GetNamespace(), Name: tenantControlPlane.Status.Certificates.CA.SecretName}
+		secretCA := &corev1.Secret{}
+		if err := r.Client.Get(ctx, namespacedName, secretCA); err != nil {
+			return err
+		}
+
+		ca := kubeadm.CertificatePrivateKeyPair{
+			Name:        kubeadmconstants.CACertAndKeyBaseName,
+			Certificate: secretCA.Data[kubeadmconstants.CACertName],
+			PrivateKey:  secretCA.Data[kubeadmconstants.CAKeyName],
+		}
+		cert, privKey, err := getCertificateAndKeyPair(ca.Certificate, ca.PrivateKey)
+		if err != nil {
+			return err
+		}
+
+		r.resource.Type = corev1.SecretTypeTLS
+		r.resource.Data = map[string][]byte{
+			corev1.TLSCertKey:       cert.Bytes(),
+			corev1.TLSPrivateKeyKey: privKey.Bytes(),
+		}
+		r.resource.SetLabels(utilities.MergeMaps(
+			utilities.KamajiLabels(),
+			map[string]string{
+				"latest-ca-rv":                latestCARV,
+				"kamaji.clastix.io/name":      tenantControlPlane.GetName(),
+				"kamaji.clastix.io/component": r.GetName(),
+			},
+		))
+
+		return ctrl.SetControllerReference(tenantControlPlane, r.resource, r.Client.Scheme())
+	}
+}
+
+func getCertificateAndKeyPair(caCert []byte, caPrivKey []byte) (*bytes.Buffer, *bytes.Buffer, error) {
+	template := getCertTemplate()
+
+	return crypto.GetCertificateAndKeyPair(template, caCert, caPrivKey)
+}
+
+func isCertificateAndKeyPairValid(cert []byte, privKey []byte) (bool, error) {
+	return crypto.IsValidCertificateKeyPairBytes(cert, privKey)
+}
+
+func getCertTemplate() *x509.Certificate {
+	serialNumber := big.NewInt(rand.Int63())
+
+	return &x509.Certificate{
+		PublicKeyAlgorithm: x509.RSA,
+		SerialNumber:       serialNumber,
+		Subject: pkix.Name{
+			CommonName:   CertCommonName,
+			Organization: []string{certOrganization},
+		},
+		NotBefore:    time.Now(),
+		NotAfter:     time.Now().AddDate(certExpirationDelayYears, 0, 0),
+		SubjectKeyId: []byte{1, 2, 3, 4, 6},
+		ExtKeyUsage: []x509.ExtKeyUsage{
+			x509.ExtKeyUsageClientAuth,
+			x509.ExtKeyUsageServerAuth,
+			x509.ExtKeyUsageCodeSigning,
+		},
+		KeyUsage: x509.KeyUsageDigitalSignature,
+	}
+}

internal/resources/konnectivity/cluster_role_binding_resource.go

@@ -0,0 +1,115 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package konnectivity
+
+import (
+	"context"
+
+	rbacv1 "k8s.io/api/rbac/v1"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+	"github.com/clastix/kamaji/internal/utilities"
+)
+
+type ClusterRoleBindingResource struct {
+	resource     *rbacv1.ClusterRoleBinding
+	Client       client.Client
+	Name         string
+	tenantClient client.Client
+}
+
+func (r *ClusterRoleBindingResource) ShouldStatusBeUpdated(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
+	return tenantControlPlane.Status.Addons.Konnectivity.ClusterRoleBinding.Name != r.resource.GetName() ||
+		tenantControlPlane.Status.Addons.Konnectivity.ClusterRoleBinding.RV != r.resource.ObjectMeta.ResourceVersion
+}
+
+func (r *ClusterRoleBindingResource) ShouldCleanup(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
+	return tenantControlPlane.Spec.Addons.Konnectivity == nil
+}
+
+func (r *ClusterRoleBindingResource) CleanUp(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) {
+	if err := r.tenantClient.Delete(ctx, r.resource); err != nil {
+		if !k8serrors.IsNotFound(err) {
+			return false, err
+		}
+
+		return false, nil
+	}
+
+	return true, nil
+}
+
+func (r *ClusterRoleBindingResource) Define(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+	r.resource = &rbacv1.ClusterRoleBinding{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: CertCommonName,
+		},
+	}
+
+	client, err := utilities.GetTenantClient(ctx, r.Client, tenantControlPlane)
+	if err != nil {
+		return err
+	}
+
+	r.tenantClient = client
+
+	return nil
+}
+
+func (r *ClusterRoleBindingResource) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) {
+	return controllerutil.CreateOrUpdate(ctx, r.tenantClient, r.resource, r.mutate(ctx, tenantControlPlane))
+}
+
+func (r *ClusterRoleBindingResource) GetName() string {
+	return r.Name
+}
+
+func (r *ClusterRoleBindingResource) UpdateTenantControlPlaneStatus(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+	if tenantControlPlane.Spec.Addons.Konnectivity != nil {
+		tenantControlPlane.Status.Addons.Konnectivity.Enabled = true
+		tenantControlPlane.Status.Addons.Konnectivity.ClusterRoleBinding = kamajiv1alpha1.ExternalKubernetesObjectStatus{
+			Name: r.resource.GetName(),
+			RV:   r.resource.ObjectMeta.ResourceVersion,
+		}
+
+		return nil
+	}
+
+	tenantControlPlane.Status.Addons.Konnectivity.ClusterRoleBinding = kamajiv1alpha1.ExternalKubernetesObjectStatus{}
+	tenantControlPlane.Status.Addons.Konnectivity.Enabled = false
+
+	return nil
+}
+
+func (r *ClusterRoleBindingResource) mutate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) controllerutil.MutateFn {
+	return func() error {
+		r.resource.SetLabels(utilities.MergeMaps(
+			utilities.KamajiLabels(),
+			map[string]string{
+				"kubernetes.io/cluster-service":   "true",
+				"addonmanager.kubernetes.io/mode": "Reconcile",
+			},
+		))
+
+		r.resource.RoleRef = rbacv1.RoleRef{
+			APIGroup: rbacv1.GroupName,
+			Kind:     "ClusterRole",
+			Name:     roleAuthDelegator,
+		}
+
+		r.resource.Subjects = []rbacv1.Subject{
+			{
+				APIGroup: rbacv1.GroupName,
+				Kind:     rbacv1.UserKind,
+				Name:     CertCommonName,
+			},
+		}
+
+		return nil
+	}
+}

internal/resources/konnectivity/constants.go

@@ -0,0 +1,22 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package konnectivity
+
+const (
+	AgentName      = "konnectivity-agent"
+	CertCommonName = "system:konnectivity-server"
+
+	agentTokenName                  = "konnectivity-agent-token"
+	apiServerAPIVersion             = "apiserver.k8s.io/v1beta1"
+	certExpirationDelayYears        = 10
+	certOrganization                = "system:master"
+	defaultClusterName              = "kubernetes"
+	defaultUDSName                  = "/run/konnectivity/konnectivity-server.socket"
+	egressSelectorConfigurationKind = "EgressSelectorConfiguration"
+	egressSelectorConfigurationName = "cluster"
+	konnectivityCertAndKeyBaseName  = "konnectivity"
+	konnectivityKubeconfigFileName  = "konnectivity-server.conf"
+	kubeconfigAPIVersion            = "v1"
+	roleAuthDelegator               = "system:auth-delegator"
+)

internal/resources/konnectivity/deployment_resource.go

@@ -0,0 +1,327 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package konnectivity
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/pkg/errors"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/utils/pointer"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+	"sigs.k8s.io/controller-runtime/pkg/log"
+
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+	"github.com/clastix/kamaji/internal/types"
+	"github.com/clastix/kamaji/internal/utilities"
+)
+
+const (
+	konnectivityEgressSelectorConfigurationPath = "/etc/kubernetes/konnectivity/configurations/egress-selector-configuration.yaml"
+	konnectivityServerName                      = "konnectivity-server"
+	konnectivityServerPath                      = "/run/konnectivity"
+
+	egressSelectorConfigurationVolume  = "egress-selector-configuration"
+	konnectivityUDSVolume              = "konnectivity-uds"
+	konnectivityServerKubeconfigVolume = "konnectivity-server-kubeconfig"
+)
+
+type KubernetesDeploymentResource struct {
+	resource               *appsv1.Deployment
+	Client                 client.Client
+	ETCDStorageType        types.ETCDStorageType
+	ETCDEndpoints          []string
+	ETCDCompactionInterval string
+	Name                   string
+}
+
+func (r *KubernetesDeploymentResource) isStatusEqual(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
+	return r.resource.Status.String() == tenantControlPlane.Status.Kubernetes.Deployment.DeploymentStatus.String()
+}
+
+func (r *KubernetesDeploymentResource) ShouldStatusBeUpdated(context.Context, *kamajiv1alpha1.TenantControlPlane) bool {
+	return false
+}
+
+func (r *KubernetesDeploymentResource) ShouldCleanup(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
+	return tenantControlPlane.Spec.Addons.Konnectivity == nil
+}
+
+func (r *KubernetesDeploymentResource) CleanUp(ctx context.Context, _ *kamajiv1alpha1.TenantControlPlane) (bool, error) {
+	logger := log.FromContext(ctx)
+
+	logger.Info("performing clean-up from Deployment of Konnectivity")
+
+	res, err := utilities.CreateOrUpdateWithConflict(ctx, r.Client, r.resource, func() error {
+		if found, index := utilities.HasNamedContainer(r.resource.Spec.Template.Spec.Containers, konnectivityServerName); found {
+			logger.Info("removing Konnectivity container")
+
+			var containers []corev1.Container
+
+			containers = append(containers, r.resource.Spec.Template.Spec.Containers[:index]...)
+			containers = append(containers, r.resource.Spec.Template.Spec.Containers[index+1:]...)
+
+			r.resource.Spec.Template.Spec.Containers = containers
+		}
+
+		if found, index := utilities.HasNamedContainer(r.resource.Spec.Template.Spec.Containers, "kube-apiserver"); found {
+			argsMap := utilities.ArgsFromSliceToMap(r.resource.Spec.Template.Spec.Containers[index].Args)
+
+			if utilities.ArgsRemoveFlag(argsMap, "--egress-selector-config-file") {
+				logger.Info("removing egress selector configuration file from kube-apiserver container")
+
+				r.resource.Spec.Template.Spec.Containers[index].Args = utilities.ArgsFromMapToSlice(argsMap)
+			}
+
+			for _, volumeName := range []string{konnectivityUDSVolume, egressSelectorConfigurationVolume} {
+				if volumeFound, volumeIndex := utilities.HasNamedVolume(r.resource.Spec.Template.Spec.Volumes, egressSelectorConfigurationVolume); volumeFound {
+					logger.Info("removing Konnectivity volume " + volumeName)
+
+					var volumes []corev1.Volume
+
+					volumes = append(volumes, r.resource.Spec.Template.Spec.Volumes[:volumeIndex]...)
+					volumes = append(volumes, r.resource.Spec.Template.Spec.Volumes[volumeIndex+1:]...)
+
+					r.resource.Spec.Template.Spec.Volumes = volumes
+				}
+			}
+		}
+
+		return nil
+	})
+
+	return res == controllerutil.OperationResultUpdated, err
+}
+
+func (r *KubernetesDeploymentResource) Define(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+	r.resource = &appsv1.Deployment{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      tenantControlPlane.GetName(),
+			Namespace: tenantControlPlane.GetNamespace(),
+		},
+	}
+
+	return nil
+}
+
+func (r *KubernetesDeploymentResource) syncContainer(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+	found, index := utilities.HasNamedContainer(r.resource.Spec.Template.Spec.Containers, konnectivityServerName)
+	if !found {
+		r.resource.Spec.Template.Spec.Containers = append(r.resource.Spec.Template.Spec.Containers, corev1.Container{})
+		index = len(r.resource.Spec.Template.Spec.Containers) - 1
+	}
+
+	r.resource.Spec.Template.Spec.Containers[index].Name = konnectivityServerName
+	r.resource.Spec.Template.Spec.Containers[index].Image = fmt.Sprintf("%s:%s", tenantControlPlane.Spec.Addons.Konnectivity.ServerImage, tenantControlPlane.Spec.Addons.Konnectivity.Version)
+	r.resource.Spec.Template.Spec.Containers[index].Command = []string{"/proxy-server"}
+	r.resource.Spec.Template.Spec.Containers[index].Args = []string{
+		"-v=8",
+		"--logtostderr=true",
+		fmt.Sprintf("--uds-name=%s/konnectivity-server.socket", konnectivityServerPath),
+		"--cluster-cert=/etc/kubernetes/pki/apiserver.crt",
+		"--cluster-key=/etc/kubernetes/pki/apiserver.key",
+		"--mode=grpc",
+		"--server-port=0",
+		fmt.Sprintf("--agent-port=%d", tenantControlPlane.Spec.Addons.Konnectivity.ProxyPort),
+		"--admin-port=8133",
+		"--health-port=8134",
+		"--agent-namespace=kube-system",
+		fmt.Sprintf("--agent-service-account=%s", AgentName),
+		"--kubeconfig=/etc/kubernetes/konnectivity-server.conf",
+		fmt.Sprintf("--authentication-audience=%s", CertCommonName),
+		fmt.Sprintf("--server-count=%d", tenantControlPlane.Spec.ControlPlane.Deployment.Replicas),
+	}
+	r.resource.Spec.Template.Spec.Containers[index].LivenessProbe = &corev1.Probe{
+		InitialDelaySeconds: 30,
+		TimeoutSeconds:      60,
+		PeriodSeconds:       10,
+		SuccessThreshold:    1,
+		FailureThreshold:    3,
+		ProbeHandler: corev1.ProbeHandler{
+			HTTPGet: &corev1.HTTPGetAction{
+				Path:   "/healthz",
+				Port:   intstr.FromInt(8134),
+				Scheme: corev1.URISchemeHTTP,
+			},
+		},
+	}
+	r.resource.Spec.Template.Spec.Containers[index].Ports = []corev1.ContainerPort{
+		{
+			Name:          "agentport",
+			ContainerPort: tenantControlPlane.Spec.Addons.Konnectivity.ProxyPort,
+			Protocol:      corev1.ProtocolTCP,
+		},
+		{
+			Name:          "adminport",
+			ContainerPort: 8133,
+			Protocol:      corev1.ProtocolTCP,
+		},
+		{
+			Name:          "healthport",
+			ContainerPort: 8134,
+			Protocol:      corev1.ProtocolTCP,
+		},
+	}
+	r.resource.Spec.Template.Spec.Containers[index].VolumeMounts = []corev1.VolumeMount{
+		{
+			Name:      "etc-kubernetes-pki",
+			MountPath: "/etc/kubernetes/pki",
+			ReadOnly:  true,
+		},
+		{
+			Name:      "konnectivity-server-kubeconfig",
+			MountPath: "/etc/kubernetes/konnectivity-server.conf",
+			SubPath:   "konnectivity-server.conf",
+			ReadOnly:  true,
+		},
+		{
+			Name:      "konnectivity-uds",
+			MountPath: konnectivityServerPath,
+			ReadOnly:  false,
+		},
+	}
+	r.resource.Spec.Template.Spec.Containers[index].ImagePullPolicy = corev1.PullAlways
+	r.resource.Spec.Template.Spec.Containers[index].Resources = corev1.ResourceRequirements{
+		Limits:   nil,
+		Requests: nil,
+	}
+
+	if resources := tenantControlPlane.Spec.Addons.Konnectivity.Resources; resources != nil {
+		r.resource.Spec.Template.Spec.Containers[index].Resources = *resources
+	}
+
+	return nil
+}
+
+func (r *KubernetesDeploymentResource) mutate(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) controllerutil.MutateFn {
+	return func() (err error) {
+		// If konnectivity is disabled, no operation is required:
+		// removal of the container will be performed by clean-up.
+		if tenantControlPlane.Spec.Addons.Konnectivity == nil {
+			return nil
+		}
+
+		if len(r.resource.Spec.Template.Spec.Containers) == 0 {
+			return fmt.Errorf("the Deployment resource is not ready to be mangled for Konnectivity server enrichment")
+		}
+
+		if err = r.syncContainer(tenantControlPlane); err != nil {
+			return errors.Wrap(err, "cannot sync konnectivity-server container")
+		}
+		if err = r.patchKubeAPIServerContainer(); err != nil {
+			return errors.Wrap(err, "cannot sync patch kube-apiserver container")
+		}
+		if err = r.syncVolumes(tenantControlPlane); err != nil {
+			return errors.Wrap(err, "cannot patch required konnectivity volumes")
+		}
+
+		return nil
+	}
+}
+
+func (r *KubernetesDeploymentResource) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) {
+	return utilities.CreateOrUpdateWithConflict(ctx, r.Client, r.resource, r.mutate(ctx, tenantControlPlane))
+}
+
+func (r *KubernetesDeploymentResource) GetName() string {
+	return r.Name
+}
+
+func (r *KubernetesDeploymentResource) UpdateTenantControlPlaneStatus(context.Context, *kamajiv1alpha1.TenantControlPlane) error {
+	return nil
+}
+
+func (r *KubernetesDeploymentResource) patchKubeAPIServerContainer() error {
+	// Patching VolumesMounts
+	found, index := false, 0
+
+	found, index = utilities.HasNamedContainer(r.resource.Spec.Template.Spec.Containers, "kube-apiserver")
+	if !found {
+		return fmt.Errorf("missing kube-apiserver container, cannot patch arguments")
+	}
+	// Adding the egress selector config file flag
+	args := utilities.ArgsFromSliceToMap(r.resource.Spec.Template.Spec.Containers[index].Args)
+
+	if utilities.ArgsAddFlagValue(args, "--egress-selector-config-file", konnectivityEgressSelectorConfigurationPath) {
+		// LOG
+	}
+
+	r.resource.Spec.Template.Spec.Containers[index].Args = utilities.ArgsFromMapToSlice(args)
+
+	vFound, vIndex := false, 0
+	// Patching the volume mounts
+	if vFound, vIndex = utilities.HasNamedVolumeMount(r.resource.Spec.Template.Spec.Containers[index].VolumeMounts, konnectivityUDSVolume); !vFound {
+		r.resource.Spec.Template.Spec.Containers[index].VolumeMounts = append(r.resource.Spec.Template.Spec.Containers[index].VolumeMounts, corev1.VolumeMount{})
+		vIndex = len(r.resource.Spec.Template.Spec.Containers[index].VolumeMounts) - 1
+	}
+
+	r.resource.Spec.Template.Spec.Containers[index].VolumeMounts[vIndex].Name = konnectivityUDSVolume
+	r.resource.Spec.Template.Spec.Containers[index].VolumeMounts[vIndex].ReadOnly = false
+	r.resource.Spec.Template.Spec.Containers[index].VolumeMounts[vIndex].MountPath = konnectivityServerPath
+
+	if vFound, vIndex = utilities.HasNamedVolumeMount(r.resource.Spec.Template.Spec.Containers[index].VolumeMounts, egressSelectorConfigurationVolume); !vFound {
+		r.resource.Spec.Template.Spec.Containers[index].VolumeMounts = append(r.resource.Spec.Template.Spec.Containers[index].VolumeMounts, corev1.VolumeMount{})
+		vIndex = len(r.resource.Spec.Template.Spec.Containers[index].VolumeMounts) - 1
+	}
+
+	r.resource.Spec.Template.Spec.Containers[index].VolumeMounts[vIndex].Name = egressSelectorConfigurationVolume
+	r.resource.Spec.Template.Spec.Containers[index].VolumeMounts[vIndex].ReadOnly = false
+	r.resource.Spec.Template.Spec.Containers[index].VolumeMounts[vIndex].MountPath = "/etc/kubernetes/konnectivity/configurations"
+
+	return nil
+}
+
+func (r *KubernetesDeploymentResource) syncVolumes(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+	found, index := false, 0
+	// Defining volumes for the UDS socket
+	found, index = utilities.HasNamedVolume(r.resource.Spec.Template.Spec.Volumes, konnectivityUDSVolume)
+	if !found {
+		r.resource.Spec.Template.Spec.Volumes = append(r.resource.Spec.Template.Spec.Volumes, corev1.Volume{})
+		index = len(r.resource.Spec.Template.Spec.Volumes) - 1
+	}
+
+	r.resource.Spec.Template.Spec.Volumes[index].Name = konnectivityUDSVolume
+	r.resource.Spec.Template.Spec.Volumes[index].VolumeSource = corev1.VolumeSource{
+		EmptyDir: &corev1.EmptyDirVolumeSource{
+			Medium: "Memory",
+		},
+	}
+	// Defining volumes for the egress selector configuration
+	found, index = utilities.HasNamedVolume(r.resource.Spec.Template.Spec.Volumes, egressSelectorConfigurationVolume)
+	if !found {
+		r.resource.Spec.Template.Spec.Volumes = append(r.resource.Spec.Template.Spec.Volumes, corev1.Volume{})
+		index = len(r.resource.Spec.Template.Spec.Volumes) - 1
+	}
+
+	r.resource.Spec.Template.Spec.Volumes[index].Name = egressSelectorConfigurationVolume
+	r.resource.Spec.Template.Spec.Volumes[index].VolumeSource = corev1.VolumeSource{
+		ConfigMap: &corev1.ConfigMapVolumeSource{
+			LocalObjectReference: corev1.LocalObjectReference{
+				Name: tenantControlPlane.Status.Addons.Konnectivity.EgressSelectorConfiguration,
+			},
+			DefaultMode: pointer.Int32Ptr(420),
+		},
+	}
+	// Defining volume for the Konnectivity kubeconfig
+	found, index = utilities.HasNamedVolume(r.resource.Spec.Template.Spec.Volumes, konnectivityServerKubeconfigVolume)
+	if !found {
+		r.resource.Spec.Template.Spec.Volumes = append(r.resource.Spec.Template.Spec.Volumes, corev1.Volume{})
+		index = len(r.resource.Spec.Template.Spec.Volumes) - 1
+	}
+
+	r.resource.Spec.Template.Spec.Volumes[index].Name = konnectivityServerKubeconfigVolume
+	r.resource.Spec.Template.Spec.Volumes[index].VolumeSource = corev1.VolumeSource{
+		Secret: &corev1.SecretVolumeSource{
+			SecretName:  tenantControlPlane.Status.Addons.Konnectivity.Kubeconfig.SecretName,
+			DefaultMode: pointer.Int32Ptr(420),
+		},
+	}
+
+	return nil
+}

internal/resources/konnectivity/egress_selector_configuration_resource.go

@@ -0,0 +1,118 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+package konnectivity
+
+import (
+	"context"
+
+	corev1 "k8s.io/api/core/v1"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	apiserverv1alpha1 "k8s.io/apiserver/pkg/apis/apiserver/v1alpha1"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+	"github.com/clastix/kamaji/internal/utilities"
+)
+
+type EgressSelectorConfigurationResource struct {
+	resource *corev1.ConfigMap
+	Client   client.Client
+	Name     string
+}
+
+func (r *EgressSelectorConfigurationResource) Define(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+	r.resource = &corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      r.getPrefixedName(tenantControlPlane),
+			Namespace: tenantControlPlane.GetNamespace(),
+		},
+	}
+
+	return nil
+}
+
+func (r *EgressSelectorConfigurationResource) ShouldCleanup(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
+	return tenantControlPlane.Spec.Addons.Konnectivity == nil
+}
+
+func (r *EgressSelectorConfigurationResource) CleanUp(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) {
+	if err := r.Client.Delete(ctx, r.resource); err != nil {
+		if !k8serrors.IsNotFound(err) {
+			return false, err
+		}
+
+		return false, nil
+	}
+
+	return true, nil
+}
+
+func (r *EgressSelectorConfigurationResource) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) {
+	return controllerutil.CreateOrUpdate(ctx, r.Client, r.resource, r.mutate(ctx, tenantControlPlane))
+}
+
+func (r *EgressSelectorConfigurationResource) GetName() string {
+	return r.Name
+}
+
+func (r *EgressSelectorConfigurationResource) ShouldStatusBeUpdated(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
+	return tenantControlPlane.Status.Addons.Konnectivity.EgressSelectorConfiguration != r.resource.GetName()
+}
+
+func (r *EgressSelectorConfigurationResource) UpdateTenantControlPlaneStatus(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+	if tenantControlPlane.Spec.Addons.Konnectivity != nil {
+		tenantControlPlane.Status.Addons.Konnectivity.Enabled = true
+		tenantControlPlane.Status.Addons.Konnectivity.EgressSelectorConfiguration = r.resource.GetName()
+
+		return nil
+	}
+
+	tenantControlPlane.Status.Addons.Konnectivity.Enabled = true
+	tenantControlPlane.Status.Addons.Konnectivity.EgressSelectorConfiguration = ""
+
+	return nil
+}
+
+func (r *EgressSelectorConfigurationResource) mutate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) func() error {
+	return func() error {
+		r.resource.SetLabels(utilities.MergeMaps(r.resource.GetLabels(), utilities.KamajiLabels()))
+
+		configuration := &apiserverv1alpha1.EgressSelectorConfiguration{
+			TypeMeta: metav1.TypeMeta{
+				Kind:       egressSelectorConfigurationKind,
+				APIVersion: apiServerAPIVersion,
+			},
+			EgressSelections: []apiserverv1alpha1.EgressSelection{
+				{
+					Name: egressSelectorConfigurationName,
+					Connection: apiserverv1alpha1.Connection{
+						ProxyProtocol: apiserverv1alpha1.ProtocolGRPC,
+						Transport: &apiserverv1alpha1.Transport{
+							UDS: &apiserverv1alpha1.UDSTransport{
+								UDSName: defaultUDSName,
+							},
+						},
+					},
+				},
+			},
+		}
+
+		yamlConfiguration, err := utilities.EncondeToYaml(configuration)
+		if err != nil {
+			return err
+		}
+
+		r.resource.Data = map[string]string{
+			"egress-selector-configuration.yaml": string(yamlConfiguration),
+		}
+
+		return ctrl.SetControllerReference(tenantControlPlane, r.resource, r.Client.Scheme())
+	}
+}
+
+func (r *EgressSelectorConfigurationResource) getPrefixedName(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) string {
+	return utilities.AddTenantPrefix(r.Name, tenantControlPlane)
+}

internal/resources/konnectivity/kubeconfig_resource.go

@@ -0,0 +1,169 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package konnectivity
+
+import (
+	"context"
+	"fmt"
+
+	corev1 "k8s.io/api/core/v1"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	k8stypes "k8s.io/apimachinery/pkg/types"
+	clientcmdapiv1 "k8s.io/client-go/tools/clientcmd/api/v1"
+	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+	"github.com/clastix/kamaji/internal/utilities"
+)
+
+type KubeconfigResource struct {
+	resource *corev1.Secret
+	Client   client.Client
+	Name     string
+}
+
+func (r *KubeconfigResource) ShouldStatusBeUpdated(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
+	return tenantControlPlane.Status.Addons.Konnectivity.Kubeconfig.SecretName != r.resource.GetName()
+}
+
+func (r *KubeconfigResource) ShouldCleanup(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
+	return tenantControlPlane.Spec.Addons.Konnectivity == nil
+}
+
+func (r *KubeconfigResource) CleanUp(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) {
+	if err := r.Client.Delete(ctx, r.resource); err != nil {
+		if !k8serrors.IsNotFound(err) {
+			return false, err
+		}
+
+		return false, nil
+	}
+
+	return true, nil
+}
+
+func (r *KubeconfigResource) Define(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+	r.resource = &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      r.getPrefixedName(tenantControlPlane),
+			Namespace: tenantControlPlane.GetNamespace(),
+		},
+	}
+
+	return nil
+}
+
+func (r *KubeconfigResource) getPrefixedName(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) string {
+	return utilities.AddTenantPrefix(r.Name, tenantControlPlane)
+}
+
+func (r *KubeconfigResource) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) {
+	return controllerutil.CreateOrUpdate(ctx, r.Client, r.resource, r.mutate(ctx, tenantControlPlane))
+}
+
+func (r *KubeconfigResource) GetName() string {
+	return r.Name
+}
+
+func (r *KubeconfigResource) UpdateTenantControlPlaneStatus(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+	if tenantControlPlane.Spec.Addons.Konnectivity != nil {
+		tenantControlPlane.Status.Addons.Konnectivity.Kubeconfig.LastUpdate = metav1.Now()
+		tenantControlPlane.Status.Addons.Konnectivity.Kubeconfig.SecretName = r.resource.GetName()
+		tenantControlPlane.Status.Addons.Konnectivity.Enabled = true
+
+		return nil
+	}
+
+	tenantControlPlane.Status.Addons.Konnectivity.Enabled = false
+	tenantControlPlane.Status.Addons.Konnectivity.Kubeconfig = kamajiv1alpha1.KubeconfigStatus{}
+
+	return nil
+}
+
+func (r *KubeconfigResource) mutate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) controllerutil.MutateFn {
+	return func() error {
+		latestCARV := tenantControlPlane.Status.Addons.Konnectivity.Certificate.ResourceVersion
+		actualCARV := r.resource.GetLabels()["latest-certificate-rv"]
+		if latestCARV == actualCARV {
+			return nil
+		}
+
+		caNamespacedName := k8stypes.NamespacedName{Namespace: tenantControlPlane.GetNamespace(), Name: tenantControlPlane.Status.Certificates.CA.SecretName}
+		secretCA := &corev1.Secret{}
+		if err := r.Client.Get(ctx, caNamespacedName, secretCA); err != nil {
+			return err
+		}
+
+		certificateNamespacedName := k8stypes.NamespacedName{Namespace: tenantControlPlane.GetNamespace(), Name: tenantControlPlane.Status.Addons.Konnectivity.Certificate.SecretName}
+		secretCertificate := &corev1.Secret{}
+		if err := r.Client.Get(ctx, certificateNamespacedName, secretCertificate); err != nil {
+			return err
+		}
+
+		userName := CertCommonName
+		clusterName := defaultClusterName
+		contextName := fmt.Sprintf("%s@%s", userName, clusterName)
+
+		kubeconfig := &clientcmdapiv1.Config{
+			Kind:       "Config",
+			APIVersion: kubeconfigAPIVersion,
+			AuthInfos: []clientcmdapiv1.NamedAuthInfo{
+				{
+					Name: userName,
+					AuthInfo: clientcmdapiv1.AuthInfo{
+						ClientKeyData:         secretCertificate.Data[corev1.TLSPrivateKeyKey],
+						ClientCertificateData: secretCertificate.Data[corev1.TLSCertKey],
+					},
+				},
+			},
+			Clusters: []clientcmdapiv1.NamedCluster{
+				{
+					Name: clusterName,
+					Cluster: clientcmdapiv1.Cluster{
+						Server:                   r.getServer(*tenantControlPlane),
+						CertificateAuthorityData: secretCA.Data[kubeadmconstants.CACertName],
+					},
+				},
+			},
+			Contexts: []clientcmdapiv1.NamedContext{
+				{
+					Name: contextName,
+					Context: clientcmdapiv1.Context{
+						Cluster:  clusterName,
+						AuthInfo: userName,
+					},
+				},
+			},
+			CurrentContext: contextName,
+		}
+
+		kubeconfigBytes, err := utilities.EncondeToYaml(kubeconfig)
+		if err != nil {
+			return err
+		}
+
+		r.resource.Data = map[string][]byte{
+			konnectivityKubeconfigFileName: kubeconfigBytes,
+		}
+
+		r.resource.SetLabels(utilities.MergeMaps(
+			utilities.KamajiLabels(),
+			map[string]string{
+				"latest-certificate-rv":       latestCARV,
+				"kamaji.clastix.io/name":      tenantControlPlane.GetName(),
+				"kamaji.clastix.io/component": r.GetName(),
+			},
+		))
+
+		return ctrl.SetControllerReference(tenantControlPlane, r.resource, r.Client.Scheme())
+	}
+}
+
+func (r *KubeconfigResource) getServer(tenantControlPlane kamajiv1alpha1.TenantControlPlane) string {
+	return fmt.Sprintf("https://%s:%d", "localhost", tenantControlPlane.Spec.NetworkProfile.Port)
+}

internal/resources/konnectivity/service_account_resource.go

@@ -0,0 +1,104 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package konnectivity
+
+import (
+	"context"
+
+	corev1 "k8s.io/api/core/v1"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+	"github.com/clastix/kamaji/internal/utilities"
+)
+
+type ServiceAccountResource struct {
+	resource     *corev1.ServiceAccount
+	Client       client.Client
+	Name         string
+	tenantClient client.Client
+}
+
+func (r *ServiceAccountResource) ShouldStatusBeUpdated(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
+	return tenantControlPlane.Status.Addons.Konnectivity.ServiceAccount.Name != r.resource.GetName() ||
+		tenantControlPlane.Status.Addons.Konnectivity.ServiceAccount.Namespace != r.resource.GetNamespace() ||
+		tenantControlPlane.Status.Addons.Konnectivity.ServiceAccount.RV != r.resource.ObjectMeta.ResourceVersion
+}
+
+func (r *ServiceAccountResource) ShouldCleanup(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
+	return tenantControlPlane.Spec.Addons.Konnectivity == nil
+}
+
+func (r *ServiceAccountResource) CleanUp(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) {
+	if err := r.tenantClient.Delete(ctx, r.resource); err != nil {
+		if !k8serrors.IsNotFound(err) {
+			return false, err
+		}
+
+		return false, nil
+	}
+
+	return true, nil
+}
+
+func (r *ServiceAccountResource) Define(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+	r.resource = &corev1.ServiceAccount{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "konnectivity-agent",
+			Namespace: agentNamespace,
+		},
+	}
+
+	client, err := utilities.GetTenantClient(ctx, r.Client, tenantControlPlane)
+	if err != nil {
+		return err
+	}
+
+	r.tenantClient = client
+
+	return nil
+}
+
+func (r *ServiceAccountResource) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) {
+	return controllerutil.CreateOrUpdate(ctx, r.tenantClient, r.resource, r.mutate(ctx, tenantControlPlane))
+}
+
+func (r *ServiceAccountResource) GetName() string {
+	return r.Name
+}
+
+func (r *ServiceAccountResource) UpdateTenantControlPlaneStatus(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+	if tenantControlPlane.Spec.Addons.Konnectivity != nil {
+		tenantControlPlane.Status.Addons.Konnectivity.ServiceAccount = kamajiv1alpha1.ExternalKubernetesObjectStatus{
+			Name:      r.resource.GetName(),
+			Namespace: r.resource.GetNamespace(),
+			RV:        r.resource.ObjectMeta.ResourceVersion,
+		}
+		tenantControlPlane.Status.Addons.Konnectivity.Enabled = true
+
+		return nil
+	}
+
+	tenantControlPlane.Status.Addons.Konnectivity.Enabled = false
+	tenantControlPlane.Status.Addons.Konnectivity.ServiceAccount = kamajiv1alpha1.ExternalKubernetesObjectStatus{}
+
+	return nil
+}
+
+func (r *ServiceAccountResource) mutate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) controllerutil.MutateFn {
+	return func() error {
+		r.resource.SetLabels(utilities.MergeMaps(
+			utilities.KamajiLabels(),
+			map[string]string{
+				"kubernetes.io/cluster-service":   "true",
+				"addonmanager.kubernetes.io/mode": "Reconcile",
+			},
+		))
+
+		return nil
+	}
+}

internal/resources/konnectivity/service_resource.go

@@ -0,0 +1,147 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package konnectivity
+
+import (
+	"context"
+	"fmt"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+	"github.com/clastix/kamaji/internal/utilities"
+)
+
+type ServiceResource struct {
+	resource *corev1.Service
+	Client   client.Client
+	Name     string
+}
+
+func (r *ServiceResource) ShouldStatusBeUpdated(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
+	if tenantControlPlane.Status.Addons.Konnectivity.Service.Name != r.resource.GetName() {
+		return true
+	}
+
+	if tenantControlPlane.Status.Addons.Konnectivity.Service.Namespace != r.resource.GetNamespace() {
+		return true
+	}
+
+	if tenantControlPlane.Status.Addons.Konnectivity.Service.Port != r.resource.Spec.Ports[1].Port {
+		return true
+	}
+
+	if len(r.resource.Status.Conditions) != len(tenantControlPlane.Status.Addons.Konnectivity.Service.Conditions) {
+		return true
+	}
+
+	resourceIngresses := tenantControlPlane.Status.Addons.Konnectivity.Service.LoadBalancer.Ingress
+	statusIngresses := r.resource.Status.LoadBalancer.Ingress
+
+	if len(resourceIngresses) != len(statusIngresses) {
+		return true
+	}
+
+	for i := 0; i < len(resourceIngresses); i++ {
+		if resourceIngresses[i].Hostname != statusIngresses[i].Hostname ||
+			resourceIngresses[i].IP != statusIngresses[i].IP ||
+			len(resourceIngresses[i].Ports) != len(statusIngresses[i].Ports) {
+			return true
+		}
+
+		resourcePorts := resourceIngresses[i].Ports
+		statusPorts := statusIngresses[i].Ports
+		for j := 0; j < len(resourcePorts); j++ {
+			if resourcePorts[j].Port != statusPorts[j].Port ||
+				resourcePorts[j].Protocol != statusPorts[j].Protocol {
+				return true
+			}
+		}
+	}
+
+	return false
+}
+
+func (r *ServiceResource) ShouldCleanup(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
+	return tenantControlPlane.Spec.Addons.Konnectivity == nil
+}
+
+func (r *ServiceResource) CleanUp(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) {
+	res, err := utilities.CreateOrUpdateWithConflict(ctx, r.Client, r.resource, func() error {
+		for index, port := range r.resource.Spec.Ports {
+			if port.Name == "konnectivity-server" {
+				ports := make([]corev1.ServicePort, 0, len(r.resource.Spec.Ports)-1)
+
+				ports = append(ports, r.resource.Spec.Ports[:index]...)
+				ports = append(ports, r.resource.Spec.Ports[index+1:]...)
+
+				r.resource.Spec.Ports = ports
+
+				break
+			}
+		}
+
+		return nil
+	})
+
+	return res == controllerutil.OperationResultUpdated, err
+}
+
+func (r *ServiceResource) UpdateTenantControlPlaneStatus(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+	if tenantControlPlane.Spec.Addons.Konnectivity != nil {
+		tenantControlPlane.Status.Addons.Konnectivity.Service.Name = r.resource.GetName()
+		tenantControlPlane.Status.Addons.Konnectivity.Service.Namespace = r.resource.GetNamespace()
+		tenantControlPlane.Status.Addons.Konnectivity.Service.Port = r.resource.Spec.Ports[1].Port
+		tenantControlPlane.Status.Addons.Konnectivity.Service.ServiceStatus = r.resource.Status
+		tenantControlPlane.Status.Addons.Konnectivity.Enabled = true
+
+		return nil
+	}
+
+	tenantControlPlane.Status.Addons.Konnectivity.Service = kamajiv1alpha1.KubernetesServiceStatus{}
+	tenantControlPlane.Status.Addons.Konnectivity.Enabled = false
+
+	return nil
+}
+
+func (r *ServiceResource) Define(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
+	r.resource = &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      tenantControlPlane.GetName(),
+			Namespace: tenantControlPlane.GetNamespace(),
+		},
+	}
+
+	return nil
+}
+
+func (r *ServiceResource) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) {
+	return controllerutil.CreateOrUpdate(ctx, r.Client, r.resource, r.mutate(ctx, tenantControlPlane))
+}
+
+func (r *ServiceResource) mutate(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) func() error {
+	return func() (err error) {
+		switch len(r.resource.Spec.Ports) {
+		case 0:
+			return fmt.Errorf("current state of the Service is not ready to be mangled for Konnectivity")
+		case 1:
+			r.resource.Spec.Ports = append(r.resource.Spec.Ports, corev1.ServicePort{})
+		}
+
+		r.resource.Spec.Ports[1].Name = "konnectivity-server"
+		r.resource.Spec.Ports[1].Protocol = corev1.ProtocolTCP
+		r.resource.Spec.Ports[1].Port = tenantControlPlane.Spec.Addons.Konnectivity.ProxyPort
+		r.resource.Spec.Ports[1].TargetPort = intstr.FromInt(int(tenantControlPlane.Spec.Addons.Konnectivity.ProxyPort))
+
+		return controllerutil.SetControllerReference(tenantControlPlane, r.resource, r.Client.Scheme())
+	}
+}
+
+func (r *ServiceResource) GetName() string {
+	return r.Name
+}

internal/resources/kubeadm_addons.go

@@ -9,14 +9,13 @@ import (
 
 	"github.com/go-logr/logr"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	clientset "k8s.io/client-go/kubernetes"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 
-	kamajiapi "github.com/clastix/kamaji/api"
 	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
 	"github.com/clastix/kamaji/internal/kubeadm"
+	"github.com/clastix/kamaji/internal/utilities"
 )
 
 type KubeadmAddon int
@@ -31,19 +30,14 @@ func (d KubeadmAddon) String() string {
 }
 
 type KubeadmAddonResource struct {
-	Client                       client.Client
-	Log                          logr.Logger
-	Name                         string
-	KubeadmAddon                 KubeadmAddon
-	kubeadmConfigResourceVersion string
+	Client                client.Client
+	Log                   logr.Logger
+	Name                  string
+	KubeadmAddon          KubeadmAddon
+	kubeadmConfigChecksum string
 }
 
 func (r *KubeadmAddonResource) isStatusEqual(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
-	addonSpec, err := r.getSpec(tenantControlPlane)
-	if err != nil {
-		return false
-	}
-
 	i, err := r.GetStatus(tenantControlPlane)
 	if err != nil {
 		return false
@@ -54,11 +48,11 @@ func (r *KubeadmAddonResource) isStatusEqual(tenantControlPlane *kamajiv1alpha1.
 		return false
 	}
 
-	return *addonSpec.Enabled == addonStatus.Enabled
+	return addonStatus.Checksum == r.kubeadmConfigChecksum
 }
 
-func (r *KubeadmAddonResource) SetKubeadmConfigResourceVersion(rv string) {
-	r.kubeadmConfigResourceVersion = rv
+func (r *KubeadmAddonResource) SetKubeadmConfigChecksum(checksum string) {
+	r.kubeadmConfigChecksum = checksum
 }
 
 func (r *KubeadmAddonResource) ShouldStatusBeUpdated(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
@@ -71,11 +65,11 @@ func (r *KubeadmAddonResource) ShouldCleanup(tenantControlPlane *kamajiv1alpha1.
 		return false
 	}
 
-	return !*spec.Enabled
+	return spec == nil
 }
 
 func (r *KubeadmAddonResource) CleanUp(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) {
-	client, err := GetRESTClient(ctx, r, tenantControlPlane)
+	client, err := utilities.GetTenantRESTClient(ctx, r.Client, tenantControlPlane)
 	if err != nil {
 		return false, err
 	}
@@ -136,29 +130,17 @@ func (r *KubeadmAddonResource) GetName() string {
 }
 
 func (r *KubeadmAddonResource) UpdateTenantControlPlaneStatus(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
-	i, err := r.GetStatus(tenantControlPlane)
+	status, err := r.GetStatus(tenantControlPlane)
 	if err != nil {
 		return err
 	}
 
-	addonSpec, err := r.getSpec(tenantControlPlane)
-	if err != nil {
-		return err
-	}
-
-	status, ok := i.(*kamajiv1alpha1.AddonStatus)
-	if !ok {
-		return fmt.Errorf("error addon status")
-	}
-
-	status.Enabled = *addonSpec.Enabled
-	status.LastUpdate = metav1.Now()
-	status.KubeadmConfigResourceVersion = r.kubeadmConfigResourceVersion
+	status.SetChecksum(r.kubeadmConfigChecksum)
 
 	return nil
 }
 
-func (r *KubeadmAddonResource) GetStatus(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (kamajiapi.KubeadmConfigResourceVersionDependant, error) {
+func (r *KubeadmAddonResource) GetStatus(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (kamajiv1alpha1.KubeadmConfigChecksumDependant, error) {
 	switch r.KubeadmAddon {
 	case AddonCoreDNS:
 		return &tenantControlPlane.Status.Addons.CoreDNS, nil
@@ -172,9 +154,9 @@ func (r *KubeadmAddonResource) GetStatus(tenantControlPlane *kamajiv1alpha1.Tena
 func (r *KubeadmAddonResource) getSpec(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (*kamajiv1alpha1.AddonSpec, error) {
 	switch r.KubeadmAddon {
 	case AddonCoreDNS:
-		return &tenantControlPlane.Spec.Addons.CoreDNS, nil
+		return tenantControlPlane.Spec.Addons.CoreDNS, nil
 	case AddonKubeProxy:
-		return &tenantControlPlane.Spec.Addons.KubeProxy, nil
+		return tenantControlPlane.Spec.Addons.KubeProxy, nil
 	default:
 		return nil, fmt.Errorf("%s has no spec", r.KubeadmAddon)
 	}

internal/resources/kubeadm_config.go

@@ -7,10 +7,8 @@ import (
 	"context"
 	"fmt"
 
-	"github.com/go-logr/logr"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/runtime"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
@@ -23,28 +21,21 @@ import (
 type KubeadmConfigResource struct {
 	resource               *corev1.ConfigMap
 	Client                 client.Client
-	Scheme                 *runtime.Scheme
-	Log                    logr.Logger
 	Name                   string
-	Port                   int32
-	Domain                 string
-	PodCIDR                string
-	ServiceCIDR            string
-	KubernetesVersion      string
 	ETCDs                  []string
 	ETCDCompactionInterval string
 	TmpDirectory           string
 }
 
 func (r *KubeadmConfigResource) ShouldStatusBeUpdated(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
-	address, err := tenantControlPlane.GetAddress(ctx, r.Client)
+	address, port, err := tenantControlPlane.AssignedControlPlaneAddress()
 	if err != nil {
 		return true
 	}
 
-	return !(tenantControlPlane.Status.KubeadmConfig.ResourceVersion == r.resource.ObjectMeta.ResourceVersion &&
+	return !(tenantControlPlane.Status.KubeadmConfig.Checksum == r.resource.GetAnnotations()["checksum"] &&
 		tenantControlPlane.Status.KubeadmConfig.ConfigmapName == r.resource.GetName() &&
-		tenantControlPlane.Status.ControlPlaneEndpoint == r.getControlPlaneEndpoint(tenantControlPlane, address))
+		tenantControlPlane.Status.ControlPlaneEndpoint == r.getControlPlaneEndpoint(tenantControlPlane.Spec.ControlPlane.Ingress, address, port))
 }
 
 func (r *KubeadmConfigResource) ShouldCleanup(plane *kamajiv1alpha1.TenantControlPlane) bool {
@@ -71,12 +62,7 @@ func (r *KubeadmConfigResource) getPrefixedName(tenantControlPlane *kamajiv1alph
 }
 
 func (r *KubeadmConfigResource) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) {
-	address, err := tenantControlPlane.GetAddress(ctx, r.Client)
-	if err != nil {
-		return controllerutil.OperationResultNone, err
-	}
-
-	return controllerutil.CreateOrUpdate(ctx, r.Client, r.resource, r.mutate(tenantControlPlane, address))
+	return utilities.CreateOrUpdateWithConflict(ctx, r.Client, r.resource, r.mutate(tenantControlPlane))
 }
 
 func (r *KubeadmConfigResource) GetName() string {
@@ -84,42 +70,40 @@ func (r *KubeadmConfigResource) GetName() string {
 }
 
 func (r *KubeadmConfigResource) UpdateTenantControlPlaneStatus(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
-	address, _ := tenantControlPlane.GetAddress(ctx, r.Client)
-
 	tenantControlPlane.Status.KubeadmConfig.LastUpdate = metav1.Now()
-	tenantControlPlane.Status.KubeadmConfig.ResourceVersion = r.resource.ObjectMeta.ResourceVersion
+	tenantControlPlane.Status.KubeadmConfig.Checksum = r.resource.GetAnnotations()["checksum"]
 	tenantControlPlane.Status.KubeadmConfig.ConfigmapName = r.resource.GetName()
-	tenantControlPlane.Status.ControlPlaneEndpoint = r.getControlPlaneEndpoint(tenantControlPlane, address)
 
 	return nil
 }
 
-func (r *KubeadmConfigResource) getControlPlaneEndpoint(tenantControlPlane *kamajiv1alpha1.TenantControlPlane, address string) string {
-	if !tenantControlPlane.Spec.ControlPlane.Ingress.Enabled {
-		return fmt.Sprintf("%s:%d", address, tenantControlPlane.Spec.NetworkProfile.Port)
+func (r *KubeadmConfigResource) getControlPlaneEndpoint(ingress kamajiv1alpha1.IngressSpec, address string, port int32) string {
+	if hostname := ingress.Hostname; len(hostname) > 0 {
+		return hostname
 	}
 
-	if tenantControlPlane.Spec.ControlPlane.Ingress.Hostname != "" {
-		return tenantControlPlane.Spec.ControlPlane.Ingress.Hostname
-	}
-
-	return getTenantControllerExternalFQDN(*tenantControlPlane)
+	return fmt.Sprintf("%s:%d", address, port)
 }
 
-func (r *KubeadmConfigResource) mutate(tenantControlPlane *kamajiv1alpha1.TenantControlPlane, address string) controllerutil.MutateFn {
+func (r *KubeadmConfigResource) mutate(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) controllerutil.MutateFn {
 	return func() error {
+		address, port, err := tenantControlPlane.AssignedControlPlaneAddress()
+		if err != nil {
+			return err
+		}
+
 		r.resource.SetLabels(utilities.KamajiLabels())
 
 		params := kubeadm.Parameters{
+			TenantControlPlaneAddress:     address,
+			TenantControlPlanePort:        port,
 			TenantControlPlaneName:        tenantControlPlane.GetName(),
 			TenantControlPlaneNamespace:   tenantControlPlane.GetNamespace(),
-			TenantControlPlaneEndpoint:    r.getControlPlaneEndpoint(tenantControlPlane, address),
-			TenantControlPlaneAddress:     address,
-			TenantControlPlanePort:        r.Port,
-			TenantControlPlaneDomain:      r.Domain,
-			TenantControlPlanePodCIDR:     r.PodCIDR,
-			TenantControlPlaneServiceCIDR: r.ServiceCIDR,
-			TenantControlPlaneVersion:     r.KubernetesVersion,
+			TenantControlPlaneEndpoint:    r.getControlPlaneEndpoint(tenantControlPlane.Spec.ControlPlane.Ingress, address, port),
+			TenantControlPlaneCertSANs:    tenantControlPlane.Spec.NetworkProfile.CertSANs,
+			TenantControlPlanePodCIDR:     tenantControlPlane.Spec.NetworkProfile.PodCIDR,
+			TenantControlPlaneServiceCIDR: tenantControlPlane.Spec.NetworkProfile.ServiceCIDR,
+			TenantControlPlaneVersion:     tenantControlPlane.Spec.Kubernetes.Version,
 			ETCDs:                         r.ETCDs,
 			ETCDCompactionInterval:        r.ETCDCompactionInterval,
 			CertificatesDir:               r.TmpDirectory,
@@ -132,6 +116,9 @@ func (r *KubeadmConfigResource) mutate(tenantControlPlane *kamajiv1alpha1.Tenant
 		}
 
 		r.resource.Data = data
+		r.resource.SetAnnotations(map[string]string{
+			"checksum": utilities.CalculateConfigMapChecksum(data),
+		})
 
 		if err := ctrl.SetControllerReference(tenantControlPlane, r.resource, r.Client.Scheme()); err != nil {
 			return err