fix(helm): etcd.compactionInterval not being respect (#506)

Signed-off-by: Dario Tranchitella <dario@tranchitella.eu>

.github/workflows/docker-ci.yml

@@ -4,6 +4,7 @@ on:
   push:
     tags:        
       - "v*"
+      - "edge-*"
 
 jobs:
   docker-ci:

.golangci.yml

@@ -3,7 +3,7 @@ linters-settings:
     sections:
       - standard
       - default
-      - prefix(github.com/clastix/kamaji)
+      - prefix(github.com/clastix/kamaji/)
   goheader:
     template: |-
       Copyright 2022 Clastix Labs

ADOPTERS.md

@@ -7,6 +7,7 @@ Feel free to open a Pull-Request to get yours listed.
 
 | Type | Name | Since | Website | Use-Case |
 |:-|:-|:-|:-|:-|
+| Vendor | DCloud | 2024 | [link](https://dcloud.co.id) | DCloud is an Indonesian Cloud Provider using Kamaji to build and offer [Managed Kubernetes Service](https://dcloud.co.id/dkubes.html). |
 | End-user | Sicuro Tech Lab | 2024 | [link](https://sicurotechlab.it/) | Sicuro Tech Lab offers cloud infrastructure for Web Agencies and uses kamaji to provide managed k8s services. |
 | R&D | TIM | 2024 | [link](https://www.gruppotim.it) | TIM is an Italian telecommunications company using Kamaji for experimental research and development purposes. |
 | End-user | KINX | 2024 | [link](https://kinx.net/?lang=en) | KINX is an Internet infrastructure service provider and will use kamaji for its new [Managed Kubernetes Service](https://kinx.net/service/cloud/kubernetes/intro/?lang=en). |

Makefile

@@ -3,7 +3,7 @@
 # To re-generate a bundle for another specific version without changing the standard setup, you can:
 # - use the VERSION as arg of the bundle target (e.g make bundle VERSION=0.0.2)
 # - use environment variables to overwrite this value (e.g export VERSION=0.0.2)
-VERSION ?= 0.6.1
+VERSION ?= 1.0.0
 
 # CHANNELS define the bundle channels used in the bundle.
 # Add a new line here if you would like to change its default config. (E.g CHANNELS = "candidate,fast,stable")
@@ -77,7 +77,7 @@ helm: ## Download helm locally if necessary.
 
 GINKGO = $(shell pwd)/bin/ginkgo
 ginkgo: ## Download ginkgo locally if necessary.
-	$(call go-install-tool,$(GINKGO),github.com/onsi/ginkgo/v2/ginkgo@v2.6.0)
+	$(call go-install-tool,$(GINKGO),github.com/onsi/ginkgo/v2/ginkgo)
 
 KIND = $(shell pwd)/bin/kind
 kind: ## Download kind locally if necessary.
@@ -307,7 +307,7 @@ env:
 
 .PHONY: e2e
 e2e: env load helm ginkgo cert-manager ## Create a KinD cluster, install Kamaji on it and run the test suite.
-	$(HELM) upgrade --debug --install kamaji ./charts/kamaji --create-namespace --namespace kamaji-system --set "image.pullPolicy=Never"
+	$(HELM) upgrade --debug --install kamaji ./charts/kamaji --create-namespace --namespace kamaji-system --set "image.pullPolicy=Never" --set "telemetry.disabled=true"
 	$(MAKE) datastores
 	$(GINKGO) -v ./e2e
 

README.md

@@ -122,6 +122,7 @@ Since Kamaji is just focusing on the Control Plane a [Kamaji's Cluster API Contr
 - YouTube ▶️ [Equinix, Kamaji, and Cluster API](https://www.youtube.com/watch?v=TLBTqROj_wA)
 - YouTube ▶️ [Rancher & Kamaji: solving multitenancy challenges in the Kubernetes world](https://www.youtube.com/watch?v=VXHNrMmlF8U)
 - YouTube ▶️ [Enabling Self-Service Kubernetes clusters with Kamaji and Paralus](https://www.youtube.com/watch?v=JWA2LwZazM0)
+- YouTube ▶️ [Hosted Control Plane on Kubernetes (HPC) with Kamaji and K0mostron by Hervé Leclerc, ALTER WAY](https://www.youtube.com/watch?v=vmRdE2ngn78)
 
 ### 🏷️ Versioning
 
@@ -157,4 +158,4 @@ The code is provided as-is with no warranties.
 If you're looking to run Kamaji in production and would like to learn more, **CLASTIX** can help by offering [Open Source support plans](https://clastix.io/support),
 as well as providing a comprehensive Enterprise Platform named [CLASTIX Enterprise Platform](https://clastix.cloud/), built on top of the Kamaji and [Capsule](https://capsule.clastix.io/) project (now donated to CNCF as a Sandbox project).
 
-Feel free to get in touch with the provided [Contact form](https://clastix.io/contact).
+Feel free to get in touch with the provided [Contact form](https://clastix.io/contact).

api/v1alpha1/tenantcontrolplane_types.go

@@ -204,7 +204,7 @@ type KonnectivityServerSpec struct {
 	// The port which Konnectivity server is listening to.
 	Port int32 `json:"port"`
 	// Container image version of the Konnectivity server.
-	// +kubebuilder:default=v0.0.32
+	// +kubebuilder:default=v0.28.6
 	Version string `json:"version,omitempty"`
 	// Container image used by the Konnectivity server.
 	// +kubebuilder:default=registry.k8s.io/kas-network-proxy/proxy-server
@@ -219,7 +219,7 @@ type KonnectivityAgentSpec struct {
 	// +kubebuilder:default=registry.k8s.io/kas-network-proxy/proxy-agent
 	Image string `json:"image,omitempty"`
 	// Version for Konnectivity agent.
-	// +kubebuilder:default=v0.0.32
+	// +kubebuilder:default=v0.28.6
 	Version string `json:"version,omitempty"`
 	// Tolerations for the deployed agent.
 	// Can be customized to start the konnectivity-agent even if the nodes are not ready or tainted.
@@ -230,9 +230,9 @@ type KonnectivityAgentSpec struct {
 
 // KonnectivitySpec defines the spec for Konnectivity.
 type KonnectivitySpec struct {
-	// +kubebuilder:default={version:"v0.0.32",image:"registry.k8s.io/kas-network-proxy/proxy-server",port:8132}
+	// +kubebuilder:default={version:"v0.28.6",image:"registry.k8s.io/kas-network-proxy/proxy-server",port:8132}
 	KonnectivityServerSpec KonnectivityServerSpec `json:"server,omitempty"`
-	// +kubebuilder:default={version:"v0.0.32",image:"registry.k8s.io/kas-network-proxy/proxy-agent"}
+	// +kubebuilder:default={version:"v0.28.6",image:"registry.k8s.io/kas-network-proxy/proxy-agent"}
 	KonnectivityAgentSpec KonnectivityAgentSpec `json:"agent,omitempty"`
 }
 

charts/kamaji/Chart.yaml

@@ -1,5 +1,5 @@
 apiVersion: v2
-appVersion: v0.6.1
+appVersion: v1.0.0
 description: Kamaji is the Hosted Control Plane Manager for Kubernetes.
 home: https://github.com/clastix/kamaji
 icon: https://github.com/clastix/kamaji/raw/master/assets/logo-colored.png
@@ -17,7 +17,7 @@ name: kamaji
 sources:
 - https://github.com/clastix/kamaji
 type: application
-version: 0.16.1
+version: 1.0.0
 annotations:
   catalog.cattle.io/certified: partner
   catalog.cattle.io/release-name: kamaji

charts/kamaji/README.md

@@ -1,6 +1,6 @@
 # kamaji
 
-![Version: 0.16.1](https://img.shields.io/badge/Version-0.16.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v0.6.1](https://img.shields.io/badge/AppVersion-v0.6.1-informational?style=flat-square)
+![Version: 1.0.0](https://img.shields.io/badge/Version-1.0.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v1.0.0](https://img.shields.io/badge/AppVersion-v1.0.0-informational?style=flat-square)
 
 Kamaji is the Hosted Control Plane Manager for Kubernetes.
 
@@ -134,6 +134,7 @@ Here the values you can override:
 | serviceAccount.create | bool | `true` |  |
 | serviceAccount.name | string | `"kamaji-controller-manager"` |  |
 | serviceMonitor.enabled | bool | `false` | Toggle the ServiceMonitor true if you have Prometheus Operator installed and configured |
+| telemetry | object | `{"disabled":false}` | Disable the analytics traces collection |
 | temporaryDirectoryPath | string | `"/tmp/kamaji"` | Directory which will be used to work with temporary files. (default "/tmp/kamaji") |
 | tolerations | list | `[]` | Kubernetes node taints that the Kamaji controller pods would tolerate |
 

charts/kamaji/crds/tenantcontrolplane.yaml

@@ -104,7 +104,7 @@ spec:
                         agent:
                           default:
                             image: registry.k8s.io/kas-network-proxy/proxy-agent
-                            version: v0.0.32
+                            version: v0.28.6
                           properties:
                             extraArgs:
                               description: |-
@@ -165,7 +165,7 @@ spec:
                                 type: object
                               type: array
                             version:
-                              default: v0.0.32
+                              default: v0.28.6
                               description: Version for Konnectivity agent.
                               type: string
                           type: object
@@ -173,7 +173,7 @@ spec:
                           default:
                             image: registry.k8s.io/kas-network-proxy/proxy-server
                             port: 8132
-                            version: v0.0.32
+                            version: v0.28.6
                           properties:
                             extraArgs:
                               description: |-
@@ -252,7 +252,7 @@ spec:
                                   type: object
                               type: object
                             version:
-                              default: v0.0.32
+                              default: v0.28.6
                               description: Container image version of the Konnectivity
                                 server.
                               type: string

charts/kamaji/templates/controller.yaml

@@ -34,6 +34,9 @@ spec:
         - --metrics-bind-address={{ .Values.metricsBindAddress }}
         - --tmp-directory={{ .Values.temporaryDirectoryPath }}
         - --datastore={{ include "datastore.fullname" . }}
+        {{- if .Values.telemetry.disabled }}
+        - --disable-telemetry
+        {{- end }}
         {{- if .Values.loggingDevel.enable }}
         - --zap-devel
         {{- end }}

charts/kamaji/templates/etcd_sts.yaml

@@ -61,7 +61,7 @@ spec:
             - --peer-cert-file=/etc/etcd/pki/peer.pem
             - --peer-key-file=/etc/etcd/pki/peer-key.pem
             - --auto-compaction-mode=periodic
-            - --auto-compaction-retention=5m
+            - --auto-compaction-retention= {{ .Values.etcd.compactionInterval }}
             - --snapshot-count=10000
             - --quota-backend-bytes=8589934592
             - --v=8

charts/kamaji/templates/validatingwebhookconfiguration.yaml

@@ -8,6 +8,27 @@ metadata:
     {{- include "kamaji.labels" $data | nindent 4 }}
   name: kamaji-validating-webhook-configuration
 webhooks:
+  - admissionReviewVersions:
+      - v1
+    clientConfig:
+      service:
+        name: {{ include "kamaji.webhookServiceName" . }}
+        namespace: {{ .Release.Namespace }}
+        path: /telemetry
+    failurePolicy: Ignore
+    name: telemetry.kamaji.clastix.io
+    rules:
+      - apiGroups:
+          - kamaji.clastix.io
+        apiVersions:
+          - v1alpha1
+        operations:
+          - CREATE
+          - UPDATE
+          - DELETE
+        resources:
+          - tenantcontrolplanes
+    sideEffects: None
   - admissionReviewVersions:
       - v1
     clientConfig:

charts/kamaji/values.yaml

@@ -219,4 +219,9 @@ datastore:
 cfssl:
   image:
     repository: cfssl/cfssl
-    tag: latest
+    tag: latest
+
+# -- Disable the analytics traces collection
+telemetry:
+  disabled: false 
+  

cmd/manager/cmd.go

@@ -7,10 +7,12 @@ import (
 	"flag"
 	"fmt"
 	"io"
+	"net/http"
 	"os"
 	goRuntime "runtime"
 	"time"
 
+	telemetryclient "github.com/clastix/kamaji-telemetry/pkg/client"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -53,6 +55,7 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command {
 		webhookCABundle            []byte
 		migrateJobImage            string
 		maxConcurrentReconciles    int
+		disableTelemetry           bool
 
 		webhookCAPath string
 	)
@@ -95,8 +98,14 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command {
 			setupLog.Info(fmt.Sprintf("Build date: %s", internal.BuildTime))
 			setupLog.Info(fmt.Sprintf("Go Version: %s", goRuntime.Version()))
 			setupLog.Info(fmt.Sprintf("Go OS/Arch: %s/%s", goRuntime.GOOS, goRuntime.GOARCH))
+			setupLog.Info(fmt.Sprintf("Telemetry enabled: %t", !disableTelemetry))
 
-			mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
+			telemetryClient := telemetryclient.New(http.Client{Timeout: 5 * time.Second}, "https://telemetry.clastix.io")
+			if disableTelemetry {
+				telemetryClient = telemetryclient.NewNewOp()
+			}
+
+			ctrlOpts := ctrl.Options{
 				Scheme: scheme,
 				Metrics: metricsserver.Options{
 					BindAddress: metricsBindAddress,
@@ -107,13 +116,15 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command {
 				HealthProbeBindAddress:  healthProbeBindAddress,
 				LeaderElection:          leaderElect,
 				LeaderElectionNamespace: managerNamespace,
-				LeaderElectionID:        "799b98bc.clastix.io",
+				LeaderElectionID:        "kamaji.clastix.io",
 				NewCache: func(config *rest.Config, opts cache.Options) (cache.Cache, error) {
 					opts.SyncPeriod = &cacheResyncPeriod
 
 					return cache.New(config, opts)
 				},
-			})
+			}
+
+			mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrlOpts)
 			if err != nil {
 				setupLog.Error(err, "unable to start manager")
 
@@ -152,6 +163,29 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command {
 				return err
 			}
 
+			k8sVersion, versionErr := cmdutils.KubernetesVersion(mgr.GetConfig())
+			if versionErr != nil {
+				setupLog.Error(err, "unable to get kubernetes version")
+
+				k8sVersion = "Unknown"
+			}
+
+			if !disableTelemetry {
+				err = mgr.Add(&controllers.TelemetryController{
+					Client:                  mgr.GetClient(),
+					KubernetesVersion:       k8sVersion,
+					KamajiVersion:           internal.GitTag,
+					TelemetryClient:         telemetryClient,
+					LeaderElectionNamespace: ctrlOpts.LeaderElectionNamespace,
+					LeaderElectionID:        ctrlOpts.LeaderElectionID,
+				})
+				if err != nil {
+					setupLog.Error(err, "unable to create controller", "controller", "TelemetryController")
+
+					return err
+				}
+			}
+
 			if err = (&controllers.CertificateLifecycle{Channel: certChannel}).SetupWithManager(mgr); err != nil {
 				setupLog.Error(err, "unable to create controller", "controller", "CertificateLifecycle")
 
@@ -196,6 +230,14 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command {
 					},
 					handlers.TenantControlPlaneServiceCIDR{},
 				},
+				routes.TenantControlPlaneTelemetry{}: {
+					handlers.TenantControlPlaneTelemetry{
+						Enabled:           !disableTelemetry,
+						TelemetryClient:   telemetryClient,
+						KamajiVersion:     internal.GitTag,
+						KubernetesVersion: k8sVersion,
+					},
+				},
 				routes.DataStoreValidate{}: {
 					handlers.DataStoreValidation{Client: mgr.GetClient()},
 				},
@@ -255,7 +297,7 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command {
 	cmd.Flags().StringVar(&healthProbeBindAddress, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
 	cmd.Flags().BoolVar(&leaderElect, "leader-elect", true, "Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.")
 	cmd.Flags().StringVar(&tmpDirectory, "tmp-directory", "/tmp/kamaji", "Directory which will be used to work with temporary files.")
-	cmd.Flags().StringVar(&kineImage, "kine-image", "rancher/kine:v0.9.2-amd64", "Container image along with tag to use for the Kine sidecar container (used only if etcd-storage-type is set to one of kine strategies).")
+	cmd.Flags().StringVar(&kineImage, "kine-image", "rancher/kine:v0.11.10-amd64", "Container image along with tag to use for the Kine sidecar container (used only if etcd-storage-type is set to one of kine strategies).")
 	cmd.Flags().StringVar(&datastore, "datastore", "etcd", "The default DataStore that should be used by Kamaji to setup the required storage.")
 	cmd.Flags().StringVar(&migrateJobImage, "migrate-image", fmt.Sprintf("clastix/kamaji:%s", internal.GitTag), "Specify the container image to launch when a TenantControlPlane is migrated to a new datastore.")
 	cmd.Flags().IntVar(&maxConcurrentReconciles, "max-concurrent-tcp-reconciles", 1, "Specify the number of workers for the Tenant Control Plane controller (beware of CPU consumption)")
@@ -265,6 +307,7 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command {
 	cmd.Flags().StringVar(&webhookCAPath, "webhook-ca-path", "/tmp/k8s-webhook-server/serving-certs/ca.crt", "Path to the Manager webhook server CA, required for the TenantControlPlane migration jobs.")
 	cmd.Flags().DurationVar(&controllerReconcileTimeout, "controller-reconcile-timeout", 30*time.Second, "The reconciliation request timeout before the controller withdraw the external resource calls, such as dealing with the Datastore, or the Tenant Control Plane API endpoint.")
 	cmd.Flags().DurationVar(&cacheResyncPeriod, "cache-resync-period", 10*time.Hour, "The controller-runtime.Manager cache resync period.")
+	cmd.Flags().BoolVar(&disableTelemetry, "disable-telemetry", false, "Disable the analytics traces collection.")
 
 	cobra.OnInitialize(func() {
 		viper.AutomaticEnv()

cmd/utils/k8s_version.go

@@ -0,0 +1,24 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package utils
+
+import (
+	"github.com/pkg/errors"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+)
+
+func KubernetesVersion(config *rest.Config) (string, error) {
+	cs, csErr := kubernetes.NewForConfig(config)
+	if csErr != nil {
+		return "", errors.Wrap(csErr, "cannot create kubernetes clientset")
+	}
+
+	sv, svErr := cs.ServerVersion()
+	if svErr != nil {
+		return "", errors.Wrap(svErr, "cannot get Kubernetes version")
+	}
+
+	return sv.GitVersion, nil
+}

config/crd/bases/kamaji.clastix.io_tenantcontrolplanes.yaml

@@ -95,7 +95,7 @@ spec:
                       agent:
                         default:
                           image: registry.k8s.io/kas-network-proxy/proxy-agent
-                          version: v0.0.32
+                          version: v0.28.6
                         properties:
                           extraArgs:
                             description: |-
@@ -156,7 +156,7 @@ spec:
                               type: object
                             type: array
                           version:
-                            default: v0.0.32
+                            default: v0.28.6
                             description: Version for Konnectivity agent.
                             type: string
                         type: object
@@ -164,7 +164,7 @@ spec:
                         default:
                           image: registry.k8s.io/kas-network-proxy/proxy-server
                           port: 8132
-                          version: v0.0.32
+                          version: v0.28.6
                         properties:
                           extraArgs:
                             description: |-
@@ -243,7 +243,7 @@ spec:
                                 type: object
                             type: object
                           version:
-                            default: v0.0.32
+                            default: v0.28.6
                             description: Container image version of the Konnectivity
                               server.
                             type: string

config/install.yaml

@@ -394,7 +394,7 @@ spec:
                       agent:
                         default:
                           image: registry.k8s.io/kas-network-proxy/proxy-agent
-                          version: v0.0.32
+                          version: v0.28.6
                         properties:
                           extraArgs:
                             description: |-
@@ -454,7 +454,7 @@ spec:
                               type: object
                             type: array
                           version:
-                            default: v0.0.32
+                            default: v0.28.6
                             description: Version for Konnectivity agent.
                             type: string
                         type: object
@@ -462,7 +462,7 @@ spec:
                         default:
                           image: registry.k8s.io/kas-network-proxy/proxy-server
                           port: 8132
-                          version: v0.0.32
+                          version: v0.28.6
                         properties:
                           extraArgs:
                             description: |-
@@ -537,7 +537,7 @@ spec:
                                 type: object
                             type: object
                           version:
-                            default: v0.0.32
+                            default: v0.28.6
                             description: Container image version of the Konnectivity server.
                             type: string
                         required:
@@ -8016,7 +8016,7 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: spec.serviceAccountName
-        image: clastix/kamaji:v0.6.1
+        image: clastix/kamaji:v1.0.0
         imagePullPolicy: Always
         livenessProbe:
           httpGet:
@@ -8208,6 +8208,27 @@ metadata:
     cluster.x-k8s.io/provider: kamaji-core
   name: kamaji-validating-webhook-configuration
 webhooks:
+- admissionReviewVersions:
+  - v1
+  clientConfig:
+    service:
+      name: kamaji-webhook-service
+      namespace: kamaji-system
+      path: /telemetry
+  failurePolicy: Ignore
+  name: telemetry.kamaji.clastix.io
+  rules:
+  - apiGroups:
+    - kamaji.clastix.io
+    apiVersions:
+    - v1alpha1
+    operations:
+    - CREATE
+    - UPDATE
+    - DELETE
+    resources:
+    - tenantcontrolplanes
+  sideEffects: None
 - admissionReviewVersions:
   - v1
   clientConfig:

config/manager/kustomization.yaml

@@ -13,4 +13,4 @@ kind: Kustomization
 images:
 - name: controller
   newName: clastix/kamaji
-  newTag: v0.6.1
+  newTag: v1.0.0

config/webhook/manifests.yaml

@@ -30,6 +30,27 @@ kind: ValidatingWebhookConfiguration
 metadata:
   name: validating-webhook-configuration
 webhooks:
+- admissionReviewVersions:
+  - v1
+  clientConfig:
+    service:
+      name: webhook-service
+      namespace: system
+      path: /telemetry
+  failurePolicy: Ignore
+  name: telemetry.kamaji.clastix.io
+  rules:
+  - apiGroups:
+    - kamaji.clastix.io
+    apiVersions:
+    - v1alpha1
+    operations:
+    - CREATE
+    - UPDATE
+    - DELETE
+    resources:
+    - tenantcontrolplanes
+  sideEffects: None
 - admissionReviewVersions:
   - v1
   clientConfig:

controllers/datastore_controller.go

@@ -19,7 +19,6 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/predicate"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
-	"sigs.k8s.io/controller-runtime/pkg/source"
 
 	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
 )
@@ -98,7 +97,7 @@ func (r *DataStore) SetupWithManager(mgr controllerruntime.Manager) error {
 		For(&kamajiv1alpha1.DataStore{}, builder.WithPredicates(
 			predicate.ResourceVersionChangedPredicate{},
 		)).
-		WatchesRawSource(source.Kind(mgr.GetCache(), &kamajiv1alpha1.TenantControlPlane{}), handler.Funcs{
+		Watches(&kamajiv1alpha1.TenantControlPlane{}, handler.Funcs{
 			CreateFunc: func(_ context.Context, createEvent event.CreateEvent, limitingInterface workqueue.RateLimitingInterface) {
 				enqueueFn(createEvent.Object.(*kamajiv1alpha1.TenantControlPlane), limitingInterface)
 			},

controllers/soot/controllers/coredns.go

@@ -79,7 +79,7 @@ func (c *CoreDNS) SetupWithManager(mgr manager.Manager) error {
 		For(&rbacv1.ClusterRoleBinding{}, builder.WithPredicates(predicate.NewPredicateFuncs(func(object client.Object) bool {
 			return object.GetName() == kubeadm.CoreDNSClusterRoleBindingName
 		}))).
-		WatchesRawSource(&source.Channel{Source: c.TriggerChannel}, &handler.EnqueueRequestForObject{}).
+		WatchesRawSource(source.Channel(c.TriggerChannel, &handler.EnqueueRequestForObject{})).
 		Owns(&rbacv1.ClusterRole{}).
 		Owns(&corev1.ServiceAccount{}).
 		Owns(&corev1.Service{}).

controllers/soot/controllers/konnectivity.go

@@ -80,7 +80,7 @@ func (k *KonnectivityAgent) SetupWithManager(mgr manager.Manager) error {
 		For(&appsv1.DaemonSet{}, builder.WithPredicates(predicate.NewPredicateFuncs(func(object client.Object) bool {
 			return object.GetName() == konnectivity.AgentName && object.GetNamespace() == konnectivity.AgentNamespace
 		}))).
-		WatchesRawSource(source.Kind(mgr.GetCache(), &corev1.ServiceAccount{}), handler.EnqueueRequestsFromMapFunc(func(_ context.Context, object client.Object) []reconcile.Request {
+		Watches(&corev1.ServiceAccount{}, handler.EnqueueRequestsFromMapFunc(func(_ context.Context, object client.Object) []reconcile.Request {
 			if object.GetName() == konnectivity.AgentName && object.GetNamespace() == konnectivity.AgentNamespace {
 				return []reconcile.Request{
 					{
@@ -94,7 +94,7 @@ func (k *KonnectivityAgent) SetupWithManager(mgr manager.Manager) error {
 
 			return nil
 		})).
-		WatchesRawSource(source.Kind(mgr.GetCache(), &v1.ClusterRoleBinding{}), handler.EnqueueRequestsFromMapFunc(func(_ context.Context, object client.Object) []reconcile.Request {
+		Watches(&v1.ClusterRoleBinding{}, handler.EnqueueRequestsFromMapFunc(func(_ context.Context, object client.Object) []reconcile.Request {
 			if object.GetName() == konnectivity.CertCommonName {
 				return []reconcile.Request{
 					{
@@ -107,6 +107,6 @@ func (k *KonnectivityAgent) SetupWithManager(mgr manager.Manager) error {
 
 			return nil
 		})).
-		WatchesRawSource(&source.Channel{Source: k.TriggerChannel}, &handler.EnqueueRequestForObject{}).
+		WatchesRawSource(source.Channel(k.TriggerChannel, &handler.EnqueueRequestForObject{})).
 		Complete(k)
 }

controllers/soot/controllers/kubeadm_phase.go

@@ -67,6 +67,6 @@ func (k *KubeadmPhase) SetupWithManager(mgr manager.Manager) error {
 
 	return controllerruntime.NewControllerManagedBy(mgr).
 		For(k.Phase.GetWatchedObject(), builder.WithPredicates(predicate.NewPredicateFuncs(k.Phase.GetPredicateFunc()))).
-		WatchesRawSource(&source.Channel{Source: k.TriggerChannel}, &handler.EnqueueRequestForObject{}).
+		WatchesRawSource(source.Channel(k.TriggerChannel, &handler.EnqueueRequestForObject{})).
 		Complete(k)
 }

controllers/soot/controllers/kubeproxy.go

@@ -79,7 +79,7 @@ func (k *KubeProxy) SetupWithManager(mgr manager.Manager) error {
 		For(&rbacv1.ClusterRoleBinding{}, builder.WithPredicates(predicate.NewPredicateFuncs(func(object client.Object) bool {
 			return object.GetName() == kubeadm.KubeProxyClusterRoleBindingName
 		}))).
-		WatchesRawSource(&source.Channel{Source: k.TriggerChannel}, &handler.EnqueueRequestForObject{}).
+		WatchesRawSource(source.Channel(k.TriggerChannel, &handler.EnqueueRequestForObject{})).
 		Owns(&corev1.ServiceAccount{}).
 		Owns(&rbacv1.Role{}).
 		Owns(&rbacv1.RoleBinding{}).

controllers/soot/controllers/migrate.go

@@ -187,7 +187,7 @@ func (m *Migrate) SetupWithManager(mgr manager.Manager) error {
 
 			return object.GetName() == vwc.GetName()
 		}))).
-		WatchesRawSource(&source.Channel{Source: m.TriggerChannel}, &handler.EnqueueRequestForObject{}).
+		WatchesRawSource(source.Channel(m.TriggerChannel, &handler.EnqueueRequestForObject{})).
 		Complete(m)
 }
 

controllers/soot/manager.go

@@ -302,7 +302,7 @@ func (m *Manager) SetupWithManager(mgr manager.Manager) error {
 	m.sootMap = make(map[string]sootItem)
 
 	return controllerruntime.NewControllerManagedBy(mgr).
-		WatchesRawSource(&source.Channel{Source: m.sootManagerErrChan}, &handler.EnqueueRequestForObject{}).
+		WatchesRawSource(source.Channel(m.sootManagerErrChan, &handler.EnqueueRequestForObject{})).
 		For(&kamajiv1alpha1.TenantControlPlane{}, builder.WithPredicates(predicate.NewPredicateFuncs(func(object client.Object) bool {
 			obj := object.(*kamajiv1alpha1.TenantControlPlane) //nolint:forcetypeassert
 			// status is required to understand if we have to start or stop the soot manager

controllers/telemetry_controller.go

@@ -0,0 +1,120 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package controllers
+
+import (
+	"context"
+	"time"
+
+	"github.com/clastix/kamaji-telemetry/api"
+	telemetry "github.com/clastix/kamaji-telemetry/pkg/client"
+	"github.com/pkg/errors"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/log"
+
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+)
+
+type TelemetryController struct {
+	Client                  client.Client
+	TelemetryClient         telemetry.Client
+	LeaderElectionNamespace string
+	LeaderElectionID        string
+	KamajiVersion           string
+	KubernetesVersion       string
+}
+
+func (m *TelemetryController) retrieveControllerUID(ctx context.Context) (string, error) {
+	var defaultSvc corev1.Service
+	if err := m.Client.Get(ctx, types.NamespacedName{Name: "kubernetes", Namespace: "default"}, &defaultSvc); err != nil {
+		return "", errors.Wrap(err, "cannot start the telemetry controller")
+	}
+
+	return string(defaultSvc.UID), nil
+}
+
+func (m *TelemetryController) Start(ctx context.Context) error {
+	logger := log.FromContext(ctx)
+
+	ticker := time.NewTicker(5 * time.Minute)
+	defer ticker.Stop()
+
+	uid, err := m.retrieveControllerUID(ctx)
+	if err != nil {
+		logger.Error(err, "cannot retrieve controller UID")
+
+		return err
+	}
+	// First run to avoid waiting 5 minutes
+	go m.collectStats(ctx, uid)
+
+	for {
+		select {
+		case <-ctx.Done():
+			return nil
+		case <-ticker.C:
+			go m.collectStats(ctx, uid)
+		}
+	}
+}
+
+func (m *TelemetryController) collectStats(ctx context.Context, uid string) {
+	logger := log.FromContext(ctx)
+
+	stats := api.Stats{
+		UUID:                uid,
+		KamajiVersion:       m.KamajiVersion,
+		KubernetesVersion:   m.KubernetesVersion,
+		TenantControlPlanes: api.TenantControlPlane{},
+		Datastores:          api.Datastores{},
+	}
+
+	var tcpList kamajiv1alpha1.TenantControlPlaneList
+	if err := m.Client.List(ctx, &tcpList); err != nil {
+		logger.Error(err, "cannot list TenantControlPlane")
+
+		return
+	}
+
+	for _, tcp := range tcpList.Items {
+		switch {
+		case tcp.Spec.ControlPlane.Deployment.Replicas == nil || *tcp.Spec.ControlPlane.Deployment.Replicas == 0:
+			stats.TenantControlPlanes.Sleeping++
+		case tcp.Status.Kubernetes.Version.Status != nil && *tcp.Status.Kubernetes.Version.Status == kamajiv1alpha1.VersionNotReady:
+			stats.TenantControlPlanes.NotReady++
+		case tcp.Status.Kubernetes.Version.Status != nil && *tcp.Status.Kubernetes.Version.Status == kamajiv1alpha1.VersionUpgrading:
+			stats.TenantControlPlanes.Upgrading++
+		default:
+			stats.TenantControlPlanes.Running++
+		}
+	}
+
+	var datastoreList kamajiv1alpha1.DataStoreList
+	if err := m.Client.List(ctx, &datastoreList); err != nil {
+		logger.Error(err, "cannot list DataStores")
+
+		return
+	}
+
+	for _, ds := range datastoreList.Items {
+		switch ds.Spec.Driver {
+		case kamajiv1alpha1.EtcdDriver:
+			stats.Datastores.Etcd.ShardCount++
+			stats.Datastores.Etcd.UsedByCount += len(ds.Status.UsedBy)
+		case kamajiv1alpha1.KinePostgreSQLDriver:
+			stats.Datastores.PostgreSQL.ShardCount++
+			stats.Datastores.PostgreSQL.UsedByCount += len(ds.Status.UsedBy)
+		case kamajiv1alpha1.KineMySQLDriver:
+			stats.Datastores.MySQL.ShardCount++
+			stats.Datastores.MySQL.UsedByCount += len(ds.Status.UsedBy)
+		case kamajiv1alpha1.KineNatsDriver:
+			stats.Datastores.NATS.ShardCount++
+			stats.Datastores.NATS.UsedByCount += len(ds.Status.UsedBy)
+		}
+	}
+
+	m.TelemetryClient.PushStats(ctx, stats)
+}

controllers/tenantcontrolplane_controller.go

@@ -227,29 +227,29 @@ func (r *TenantControlPlaneReconciler) SetupWithManager(mgr ctrl.Manager) error
 	r.clock = clock.RealClock{}
 
 	return ctrl.NewControllerManagedBy(mgr).
-		WatchesRawSource(&source.Channel{Source: r.CertificateChan}, handler.Funcs{GenericFunc: func(_ context.Context, genericEvent event.GenericEvent, limitingInterface workqueue.RateLimitingInterface) {
+		WatchesRawSource(source.Channel(r.CertificateChan, handler.Funcs{GenericFunc: func(_ context.Context, genericEvent event.GenericEvent, limitingInterface workqueue.RateLimitingInterface) {
 			limitingInterface.AddRateLimited(ctrl.Request{
 				NamespacedName: k8stypes.NamespacedName{
 					Namespace: genericEvent.Object.GetNamespace(),
 					Name:      genericEvent.Object.GetName(),
 				},
 			})
-		}}).
-		WatchesRawSource(&source.Channel{Source: r.TriggerChan}, handler.Funcs{GenericFunc: func(_ context.Context, genericEvent event.GenericEvent, limitingInterface workqueue.RateLimitingInterface) {
+		}})).
+		WatchesRawSource(source.Channel(r.TriggerChan, handler.Funcs{GenericFunc: func(_ context.Context, genericEvent event.GenericEvent, limitingInterface workqueue.RateLimitingInterface) {
 			limitingInterface.AddRateLimited(ctrl.Request{
 				NamespacedName: k8stypes.NamespacedName{
 					Namespace: genericEvent.Object.GetNamespace(),
 					Name:      genericEvent.Object.GetName(),
 				},
 			})
-		}}).
+		}})).
 		For(&kamajiv1alpha1.TenantControlPlane{}).
 		Owns(&corev1.Secret{}).
 		Owns(&corev1.ConfigMap{}).
 		Owns(&appsv1.Deployment{}).
 		Owns(&corev1.Service{}).
 		Owns(&networkingv1.Ingress{}).
-		WatchesRawSource(source.Kind(mgr.GetCache(), &batchv1.Job{}), handler.EnqueueRequestsFromMapFunc(func(_ context.Context, object client.Object) []reconcile.Request {
+		Watches(&batchv1.Job{}, handler.EnqueueRequestsFromMapFunc(func(_ context.Context, object client.Object) []reconcile.Request {
 			labels := object.GetLabels()
 
 			name, namespace := labels["tcp.kamaji.clastix.io/name"], labels["tcp.kamaji.clastix.io/namespace"]

deploy/kine/mysql/kine.yaml

@@ -44,7 +44,7 @@ spec:
           secretName: mysql-certs
       containers:
         - name: kine-tenant
-          image: rancher/kine:v0.9.2-amd64
+          image: rancher/kine:v0.11.10-amd64
           ports:
             - containerPort: 2379
               name: server

docs/content/reference/api.md

@@ -13509,7 +13509,7 @@ Enables the Konnectivity addon in the Tenant Cluster, required if the worker nod
         <td>
           <br/>
           <br/>
-            <i>Default</i>: map[image:registry.k8s.io/kas-network-proxy/proxy-agent version:v0.0.32]<br/>
+            <i>Default</i>: map[image:registry.k8s.io/kas-network-proxy/proxy-agent version:v0.28.6]<br/>
         </td>
         <td>false</td>
       </tr><tr>
@@ -13518,7 +13518,7 @@ Enables the Konnectivity addon in the Tenant Cluster, required if the worker nod
         <td>
           <br/>
           <br/>
-            <i>Default</i>: map[image:registry.k8s.io/kas-network-proxy/proxy-server port:8132 version:v0.0.32]<br/>
+            <i>Default</i>: map[image:registry.k8s.io/kas-network-proxy/proxy-server port:8132 version:v0.28.6]<br/>
         </td>
         <td>false</td>
       </tr></tbody>
@@ -13575,7 +13575,7 @@ Can be customized to start the konnectivity-agent even if the nodes are not read
         <td>
           Version for Konnectivity agent.<br/>
           <br/>
-            <i>Default</i>: v0.0.32<br/>
+            <i>Default</i>: v0.28.6<br/>
         </td>
         <td>false</td>
       </tr></tbody>
@@ -13704,7 +13704,7 @@ unxpected ways. Only modify if you know what you are doing.<br/>
         <td>
           Container image version of the Konnectivity server.<br/>
           <br/>
-            <i>Default</i>: v0.0.32<br/>
+            <i>Default</i>: v0.28.6<br/>
         </td>
         <td>false</td>
       </tr></tbody>

docs/content/reference/configuration.md

@@ -10,7 +10,7 @@ Available flags are the following:
 | `--health-probe-bind-address`     | The address the probe endpoint binds to.                                                                                                                                           | `:8081`                                        |
 | `--leader-elect`                  | Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.                                                              | `true`                                         |
 | `--tmp-directory`                 | Directory which will be used to work with temporary files.                                                                                                                         | `/tmp/kamaji`                                  |
-| `--kine-image`                    | Container image along with tag to use for the Kine sidecar container (used only if etcd-storage-type is set to one of kine strategies).                                            | `rancher/kine:v0.9.2-amd64`                    |
+| `--kine-image`                    | Container image along with tag to use for the Kine sidecar container (used only if etcd-storage-type is set to one of kine strategies).                                            | `rancher/kine:v0.11.10-amd64`                  |
 | `--datastore`                     | The default DataStore that should be used by Kamaji to setup the required storage.                                                                                                 | `etcd`                                         |
 | `--migrate-image`                 | Specify the container image to launch when a TenantControlPlane is migrated to a new datastore.                                                                                    | `migrate-image`                                |
 | `--max-concurrent-tcp-reconciles` | Specify the number of workers for the Tenant Control Plane controller (beware of CPU consumption).                                                                                 | `1`                                            |

docs/content/reference/conformance.md

@@ -12,6 +12,8 @@ All the _“Tenant Clusters”_ built with Kamaji are CNCF conformant:
 - [v1.26](https://github.com/cncf/k8s-conformance/pull/2787)
 - [v1.27](https://github.com/cncf/k8s-conformance/pull/2786)
 - [v1.28](https://github.com/cncf/k8s-conformance/pull/2785)
+- [v1.29](https://github.com/cncf/k8s-conformance/pull/3273)
+- [v1.30](https://github.com/cncf/k8s-conformance/pull/3274)
 
 <p align="left" style="padding: 6px 6px">
   <img src="https://raw.githubusercontent.com/cncf/artwork/master/projects/kubernetes/certified-kubernetes/versionless/color/certified-kubernetes-color.png" width="100" />

docs/content/reference/versioning.md

@@ -1,12 +1,26 @@
-# Versioning
+# Releases and Versions
+
+[Clastix Labs](https://github.com/clastix) organization publishes Kamaji's versions that correspond to specific project milestones and sets of new features. These versions are available in different types of release artifacts.
+
+## Types of release artifacts
+
+### Edge Releases
+
+Edge Release artifacts are published on a monthly basis as part of the open source project. Versioning follows the form `edge-{year}.{month}.{incremental}` where incremental refers to the monthly release. For example, `edge-24.7.1` is the first edge release shipped in July 2024. The full list of edge release artifacts can be found on the Kamaji's GitHub [releases page](https://github.com/clastix/kamaji/releases).
+
+Edge Release artifacts contain the code in from the main branch at the point in time when they were cut. This means they always have the latest features and fixes, and have undergone automated testing as well as maintainer code review. Edge Releases may involve partial features that are later modified or backed out. They may also involve breaking changes, of course, we do our best to avoid this. Edge Releases are generally considered production ready, and the project will mark specific releases as “_not recommended_” if bugs are discovered after release.
+
+Using Edge Release artifacts and reporting bugs helps us ensure a rapid pace of development and is a great way to help maintainers. We publish edge release guidance as part of the release notes and strive to always provide production-ready artifacts.
+
+### Stable Releases
+
+Stable Release artifacts of Kamaji follow semantic versioning, whereby changes in major version denote large feature additions and possible breaking changes and changes in minor versions denote safe upgrades without breaking changes.
+
+As of July 2024, [Clastix Labs](https://github.com/clastix) organization does not no longer provide stable release artifacts. Stable Release artefacts are offered on a subscription basis by [CLASTIX](https://clastix.io), the main Kamaji project contributor.
 
-In Kamaji, there are different components that might require independent versioning and support level:
 
 | Kamaji | Management Cluster | Tenant Cluster       |
 |--------|--------------------|----------------------|
-| v0.0   | v1.22+             | [v1.21.0 .. v1.23.5] |
-| v0.1   | v1.22+             | [v1.21.0 .. v1.25.0] |
-| v0.2   | v1.22+             | [v1.21.0 .. v1.27.0] |
 | v0.3.0 | v1.22+             | [v1.21.0 .. v1.27.0] |
 | v0.3.1 | v1.22+             | [v1.21.0 .. v1.27.3] |
 | v0.3.2 | v1.22+             | [v1.21.0 .. v1.27.3] |
@@ -19,3 +33,4 @@ In Kamaji, there are different components that might require independent version
 | v0.4.2 | v1.22+             | [v1.21.0 .. v1.29.1] |
 | v0.5.0 | v1.22+             | [v1.21.0 .. v1.30.0] |
 | v0.6.0 | v1.22+             | [v1.21.0 .. v1.30.1] |
+| v1.0.0 | v1.22+             | [v1.21.0 .. v1.30.2] |

docs/content/telemetry.md

@@ -0,0 +1,49 @@
+# Telemetry
+
+This document outlines the telemetry feature within the Kamaji project, detailing the rationale behind data collection, the nature of the data collected, data handling practices, and instructions for opting out.
+
+## Why We Collect Telemetry
+
+The Kamaji project, being open source, benefits from insights into how it is used. These insights help the project maintainers make informed decisions regarding feature prioritization, test automation, and bug fixes. Without this data, decisions on feature deprecation and enhancements would be based on limited information, potentially hindering the project's evolution and maintainability. Our goal is to ensure Kamaji's development is driven by the needs of its community, and telemetry data plays a crucial role in achieving this.
+
+## What We Collect and How
+
+It's important to clarify that our interest lies in the usage patterns of Kamaji, not in personal information about its users. We collect data about Kamaji version and Tenant Control Planes.
+
+### Telemetry Payload Example
+
+Below is a simplified example of what a telemetry payload might look like:
+
+```json
+// General status
+{"uuid": "56279633-3131-436b-b8f2-9008a49a2f12", "running": 1, "sleeping": 0, "not_ready": 0, "upgrading": 0, "kamaji_version": "v0.6.1", "kubernetes_version": "v1.27.3"}
+```
+
+```json
+// Creating a TCP
+{"tcp_version": "v1.26.0", "kamaji_version": "v0.6.1", "kubernetes_version": "v1.27.3"}
+```
+
+```json
+// Modifying a TCP (n.b.: version upgrade)
+{"kamaji_version": "v0.6.1", "new_tcp_version": "v1.27.0", "old_tcp_version": "v1.26.0", "kubernetes_version": "v1.27.3"}
+```
+
+```json
+// Deleting a TCP
+{"status": "Ready", "tcp_version": "v1.27.0", "kamaji_version": "v0.6.1", "kubernetes_version": "v1.27.3"}
+```
+
+Data is collected through a component within Kamaji, which periodically sends this information to our backend. The telemetry backend, managed by the Kamaji maintainers, ensures data is stored securely and access is strictly controlled.
+
+## Telemetry Opt-Out
+We respect the privacy and autonomy of our users. If you prefer not to participate in telemetry, Kamaji provides an easy opt-out mechanism.
+
+For Helm based deployments, include the `--set telemetry.disabled=true` flag when installing or upgrading Kamaji.
+
+For manual deployments, set the flag `--disable-telemetry=true` int the Kamaji controller configuration to disable telemetry reporting.
+
+To re-enable telemetry, simply reverse the opt-out process using the appropriate method for your deployment.
+
+## Conclusion
+Telemetry in Kamaji is designed to foster a data-driven development process that aligns with the needs and preferences of our user community. Your participation helps us make Kamaji better for everyone.

docs/mkdocs.yml

@@ -76,4 +76,5 @@ nav:
   - reference/conformance.md
   - reference/versioning.md
   - reference/api.md
+- 'Telemetry': telemetry.md
 - 'Contribute': contribute.md

go.mod

@@ -7,18 +7,19 @@ toolchain go1.22.1
 require (
 	github.com/JamesStewy/go-mysqldump v0.2.2
 	github.com/blang/semver v3.5.1+incompatible
-	github.com/go-logr/logr v1.4.1
+	github.com/clastix/kamaji-telemetry v1.0.0
+	github.com/go-logr/logr v1.4.2
 	github.com/go-pg/pg/v10 v10.10.6
 	github.com/go-sql-driver/mysql v1.6.0
 	github.com/google/go-cmp v0.6.0
-	github.com/google/uuid v1.3.0
+	github.com/google/uuid v1.6.0
 	github.com/json-iterator/go v1.1.12
 	github.com/juju/mutex/v2 v2.0.0
 	github.com/nats-io/nats.go v1.34.1
 	github.com/onsi/ginkgo/v2 v2.17.1
 	github.com/onsi/gomega v1.32.0
 	github.com/pkg/errors v0.9.1
-	github.com/spf13/cobra v1.7.0
+	github.com/spf13/cobra v1.8.1
 	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.10.1
 	github.com/testcontainers/testcontainers-go v0.13.0
@@ -35,7 +36,7 @@ require (
 	k8s.io/kubelet v0.0.0
 	k8s.io/kubernetes v1.30.2
 	k8s.io/utils v0.0.0-20230726121419-3b25d923346b
-	sigs.k8s.io/controller-runtime v0.17.1-0.20240416095710-67b27f27e514
+	sigs.k8s.io/controller-runtime v0.18.4
 )
 
 require (

go.sum

@@ -861,6 +861,8 @@ github.com/cilium/ebpf v0.0.0-20200702112145-1c8d4c9ef775/go.mod h1:7cR51M8ViRLI
 github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs=
 github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs=
 github.com/cilium/ebpf v0.6.2/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs=
+github.com/clastix/kamaji-telemetry v1.0.0 h1:/s7TVsyQpunD+cBKIaWmZ1yCXXYXgf4uQ4TeXio4moY=
+github.com/clastix/kamaji-telemetry v1.0.0/go.mod h1:yhK/I0qEmKQw4mtEZRUnQfjsbbrIZtuR/XXISBrrETU=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
@@ -992,6 +994,7 @@ github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfc
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY=
@@ -1106,8 +1109,9 @@ github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbV
 github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
-github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ=
 github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
+github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
 github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ=
 github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg=
@@ -1250,8 +1254,9 @@ github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8=
 github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8=
 github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg=
@@ -1275,8 +1280,9 @@ github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+
 github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gorilla/handlers v0.0.0-20150720190736-60c7bfde3e33/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ=
-github.com/gorilla/mux v1.7.2 h1:zoNxOV7WjqXptQOVngLmcSQgXmgk4NMz1HibBchjl/I=
 github.com/gorilla/mux v1.7.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
+github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
+github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
@@ -1692,8 +1698,9 @@ github.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee/go.mod h1:1l0Ry5zgKv
 github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
 github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE=
 github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo=
-github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I=
 github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0=
+github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM=
+github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
 github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk=
 github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
@@ -2851,8 +2858,8 @@ rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.29.0/go.mod h1:z7+wmGM2dfIiLRfrC6jb5kV2Mq/sK1ZP303cxzkV5Y4=
-sigs.k8s.io/controller-runtime v0.17.1-0.20240416095710-67b27f27e514 h1:jHkHzzQ/HXBagSiTfTnfGVdPhE7O0jC7QRXDZ9MK4Xg=
-sigs.k8s.io/controller-runtime v0.17.1-0.20240416095710-67b27f27e514/go.mod h1:TLM3OvUJgcqHVBLVRlNylmfbOlOukMLFHtc6jo3EtIQ=
+sigs.k8s.io/controller-runtime v0.18.4 h1:87+guW1zhvuPLh1PHybKdYFLU0YJp4FhJRmiHvm5BZw=
+sigs.k8s.io/controller-runtime v0.18.4/go.mod h1:TVoGrfdpbA9VRFaRnKgk9P5/atA0pMwq+f+msb9M8Sg=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
 sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3 h1:XX3Ajgzov2RKUdc5jW3t5jwY7Bo7dcRm+tFxT+NfgY0=

internal/datastore/datastore.go

@@ -23,7 +23,7 @@ type ConnectionEndpoint struct {
 }
 
 func (r ConnectionEndpoint) String() string {
-	return fmt.Sprintf("%s:%d", r.Host, r.Port)
+	return net.JoinHostPort(r.Host, strconv.FormatInt(int64(r.Port), 10))
 }
 
 type ConnectionConfig struct {

internal/resources/k8s_ingress_resource.go

@@ -30,20 +30,49 @@ func (r *KubernetesIngressResource) ShouldStatusBeUpdated(_ context.Context, tcp
 	case tcp.Spec.ControlPlane.Ingress == nil && tcp.Status.Kubernetes.Ingress == nil:
 		// No update in case of no ingress in spec, neither in status.
 		return false
-	case tcp.Spec.ControlPlane.Ingress != nil && tcp.Status.Kubernetes.Ingress == nil,
-		// Must be updated when TCP is using an Ingress, and status is not tracking it
-		// or
-		// Must be updated when the status is referring to an Ingress, although spec doesn't.
-		tcp.Spec.ControlPlane.Ingress == nil && tcp.Status.Kubernetes.Ingress != nil:
+	case tcp.Spec.ControlPlane.Ingress != nil && tcp.Status.Kubernetes.Ingress == nil, // TCP is using an Ingress, Status not tracking it
+		tcp.Spec.ControlPlane.Ingress == nil && tcp.Status.Kubernetes.Ingress != nil: // Status tracks an Ingress, Spec doesn't
 		return true
-	case len(r.resource.Status.LoadBalancer.Ingress) > 0 && tcp.Status.Kubernetes.Ingress == nil || tcp.Status.Kubernetes.Ingress.LoadBalancer.Ingress == nil:
-		// Must be updated since missing the Ingress status
-		return true
-	case r.resource.Status.LoadBalancer.Ingress[0].IP != tcp.Status.Kubernetes.Ingress.LoadBalancer.Ingress[0].IP:
-		// Must bne updated, Ingress load balancer IP is slightly different
+	case len(tcp.Status.Kubernetes.Ingress.IngressStatus.LoadBalancer.Ingress) != len(r.resource.Status.LoadBalancer.Ingress):
+		// Mismatch count of tracked LoadBalancer Ingress
 		return true
 	default:
-		return tcp.Status.Kubernetes.Ingress.Name != r.resource.GetName() || tcp.Status.Kubernetes.Ingress.Namespace != r.resource.GetNamespace()
+		statusIngress := tcp.Status.Kubernetes.Ingress.IngressStatus.LoadBalancer.Ingress
+
+		for i, ingress := range r.resource.Status.LoadBalancer.Ingress {
+			if ingress.IP != statusIngress[i].IP {
+				return true
+			}
+
+			if len(ingress.Ports) != len(statusIngress[i].Ports) {
+				return true
+			}
+
+			for p, port := range ingress.Ports {
+				if port.Port != statusIngress[i].Ports[p].Port {
+					return true
+				}
+
+				if port.Protocol != statusIngress[i].Ports[p].Protocol {
+					return true
+				}
+
+				if port.Error == nil && statusIngress[i].Ports[p].Error != nil ||
+					port.Error != nil && statusIngress[i].Ports[p].Error == nil {
+					return true
+				}
+
+				if port.Error == nil && statusIngress[i].Ports[p].Error == nil {
+					continue
+				}
+
+				if *port.Error != *statusIngress[i].Ports[p].Error {
+					return true
+				}
+			}
+		}
+
+		return false
 	}
 }
 

internal/resources/k8s_service_resource.go

@@ -5,7 +5,8 @@ package resources
 
 import (
 	"context"
-	"fmt"
+	"net"
+	"strconv"
 
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -53,8 +54,7 @@ func (r *KubernetesServiceResource) UpdateTenantControlPlaneStatus(ctx context.C
 
 		return err
 	}
-
-	tenantControlPlane.Status.ControlPlaneEndpoint = fmt.Sprintf("%s:%d", address, tenantControlPlane.Spec.NetworkProfile.Port)
+	tenantControlPlane.Status.ControlPlaneEndpoint = net.JoinHostPort(address, strconv.FormatInt(int64(tenantControlPlane.Spec.NetworkProfile.Port), 10))
 
 	return nil
 }

internal/resources/kubeadm_config.go

@@ -5,7 +5,8 @@ package resources
 
 import (
 	"context"
-	"fmt"
+	"net"
+	"strconv"
 
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -74,7 +75,7 @@ func (r *KubeadmConfigResource) getControlPlaneEndpoint(ingress *kamajiv1alpha1.
 		address, port = utilities.GetControlPlaneAddressAndPortFromHostname(ingress.Hostname, port)
 	}
 
-	return fmt.Sprintf("%s:%d", address, port)
+	return net.JoinHostPort(address, strconv.FormatInt(int64(port), 10))
 }
 
 func (r *KubeadmConfigResource) mutate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) controllerutil.MutateFn {

internal/webhook/handlers/tcp_telemetry.go

@@ -0,0 +1,97 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package handlers
+
+import (
+	"context"
+
+	"github.com/clastix/kamaji-telemetry/api"
+	kamajitelemetry "github.com/clastix/kamaji-telemetry/pkg/client"
+	"k8s.io/apimachinery/pkg/runtime"
+
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+	"github.com/clastix/kamaji/internal/webhook/utils"
+)
+
+type TenantControlPlaneTelemetry struct {
+	Enabled           bool
+	TelemetryClient   kamajitelemetry.Client
+	KamajiVersion     string
+	KubernetesVersion string
+}
+
+func (t TenantControlPlaneTelemetry) OnCreate(object runtime.Object) AdmissionResponse {
+	if t.Enabled {
+		tcp := object.(*kamajiv1alpha1.TenantControlPlane) //nolint:forcetypeassert
+
+		go t.TelemetryClient.PushCreate(context.Background(), api.Create{
+			KamajiVersion:     t.KamajiVersion,
+			KubernetesVersion: t.KubernetesVersion,
+			TenantVersion:     tcp.Spec.Kubernetes.Version,
+			ClusterAPIOwned: func() bool {
+				for _, owner := range tcp.OwnerReferences {
+					if owner.Kind == "KamajiControlPlane" {
+						return true
+					}
+				}
+
+				return false
+			}(),
+			NetworkProfile: func() string {
+				switch {
+				case tcp.Spec.ControlPlane.Ingress != nil:
+					return api.NetworkProfileIngress
+				case tcp.Spec.ControlPlane.Service.ServiceType == kamajiv1alpha1.ServiceTypeLoadBalancer:
+					return api.NetworkProfileLB
+				case tcp.Spec.ControlPlane.Service.ServiceType == kamajiv1alpha1.ServiceTypeNodePort:
+					return api.NetworkProfileNodePort
+				case tcp.Spec.ControlPlane.Service.ServiceType == kamajiv1alpha1.ServiceTypeClusterIP:
+					return api.NetworkProfileClusterIP
+				default:
+					return "Unknown"
+				}
+			}(),
+		})
+	}
+
+	return utils.NilOp()
+}
+
+func (t TenantControlPlaneTelemetry) OnDelete(object runtime.Object) AdmissionResponse {
+	if t.Enabled {
+		tcp := object.(*kamajiv1alpha1.TenantControlPlane) //nolint:forcetypeassert
+
+		go t.TelemetryClient.PushDelete(context.Background(), api.Delete{
+			KamajiVersion:     t.KamajiVersion,
+			KubernetesVersion: t.KubernetesVersion,
+			TenantVersion:     tcp.Spec.Kubernetes.Version,
+			Status:            t.extractTCPVersion(tcp.Status.Kubernetes.Version.Status),
+		})
+	}
+
+	return utils.NilOp()
+}
+
+func (t TenantControlPlaneTelemetry) OnUpdate(newObject runtime.Object, prevObject runtime.Object) AdmissionResponse {
+	if t.Enabled {
+		prevTCP, newTCP := prevObject.(*kamajiv1alpha1.TenantControlPlane), newObject.(*kamajiv1alpha1.TenantControlPlane) //nolint:forcetypeassert
+
+		go t.TelemetryClient.PushUpdate(context.Background(), api.Update{
+			KamajiVersion:     t.KamajiVersion,
+			KubernetesVersion: t.KubernetesVersion,
+			OldTenantVersion:  prevTCP.Status.Kubernetes.Version.Version,
+			NewTenantVersion:  newTCP.Spec.Kubernetes.Version,
+		})
+	}
+
+	return utils.NilOp()
+}
+
+func (t TenantControlPlaneTelemetry) extractTCPVersion(status *kamajiv1alpha1.KubernetesVersionStatus) string {
+	if status == nil {
+		return "Unknown"
+	}
+
+	return string(*status)
+}

internal/webhook/routes/tcp_telemetry.go

@@ -0,0 +1,22 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package routes
+
+import (
+	"k8s.io/apimachinery/pkg/runtime"
+
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+)
+
+//+kubebuilder:webhook:path=/telemetry,mutating=false,failurePolicy=ignore,sideEffects=None,groups=kamaji.clastix.io,resources=tenantcontrolplanes,verbs=create;update;delete,versions=v1alpha1,name=telemetry.kamaji.clastix.io,admissionReviewVersions=v1
+
+type TenantControlPlaneTelemetry struct{}
+
+func (t TenantControlPlaneTelemetry) GetPath() string {
+	return "/telemetry"
+}
+
+func (t TenantControlPlaneTelemetry) GetObject() runtime.Object {
+	return &kamajiv1alpha1.TenantControlPlane{}
+}