feat: add ObservedGeneration (#1069)

* feat: add ObservedGeneration to all status types

Add ObservedGeneration field to DataStoreStatus, KubeconfigGeneratorStatus,
and TenantControlPlaneStatus to track which generation the controller has
processed. This enables clients and tools like kstatus to determine if the
controller has reconciled the latest spec changes.

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

* refactor: follow Cluster API pattern for ObservedGeneration

Move ObservedGeneration setting for TenantControlPlane from intermediate
status updates to the final successful reconciliation completion. This
follows Cluster API conventions where ObservedGeneration indicates the
controller has fully processed the given generation.

Previously, ObservedGeneration was set on every status update during
resource processing, which could mislead clients into thinking the spec
was fully reconciled when the controller was still mid-reconciliation
or had hit a transient error.

Now:
- DataStore: Sets ObservedGeneration before single status update (simple controller)
- KubeconfigGenerator: Sets ObservedGeneration before single status update (simple controller)
- TenantControlPlane: Sets ObservedGeneration only after ALL resources processed successfully

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

* test: verify ObservedGeneration equals Generation after reconciliation

Add assertion to e2e test to verify that status.observedGeneration
equals metadata.generation after a TenantControlPlane is successfully
reconciled.

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

* chore: regenerate CRDs with ObservedGeneration field

Run make crds to regenerate CRDs with the new ObservedGeneration
field in status types.

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

* Run make manifests

* Run make apidoc

* Remove rbac role

* Remove webhook manifest

---------

Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>

api/v1alpha1/datastore_types.go

@@ -90,6 +90,9 @@ type SecretReference struct {
 
 // DataStoreStatus defines the observed state of DataStore.
 type DataStoreStatus struct {
+	// ObservedGeneration represents the .metadata.generation that was last reconciled.
+	// +optional
+	ObservedGeneration int64 `json:"observedGeneration,omitempty"`
 	// List of the Tenant Control Planes, namespaced named, using this data store.
 	UsedBy []string `json:"usedBy,omitempty"`
 }

api/v1alpha1/kubeconfiggenerator_types.go

@@ -66,6 +66,9 @@ type KubeconfigGeneratorStatusError struct {
 
 // KubeconfigGeneratorStatus defines the observed state of KubeconfigGenerator.
 type KubeconfigGeneratorStatus struct {
+	// ObservedGeneration represents the .metadata.generation that was last reconciled.
+	// +optional
+	ObservedGeneration int64 `json:"observedGeneration,omitempty"`
 	// Resources is the sum of targeted TenantControlPlane objects.
 	//+kubebuilder:default=0
 	Resources int `json:"resources"`

api/v1alpha1/tenantcontrolplane_status.go

@@ -162,6 +162,9 @@ type AddonsStatus struct {
 
 // TenantControlPlaneStatus defines the observed state of TenantControlPlane.
 type TenantControlPlaneStatus struct {
+	// ObservedGeneration represents the .metadata.generation that was last reconciled.
+	// +optional
+	ObservedGeneration int64 `json:"observedGeneration,omitempty"`
 	// Storage Status contains information about Kubernetes storage system
 	Storage StorageStatus `json:"storage,omitempty"`
 	// Certificates contains information about the different certificates

charts/kamaji-crds/hack/kamaji.clastix.io_datastores_spec.yaml

@@ -275,6 +275,10 @@ versions:
           status:
             description: DataStoreStatus defines the observed state of DataStore.
             properties:
+              observedGeneration:
+                description: ObservedGeneration represents the .metadata.generation that was last reconciled.
+                format: int64
+                type: integer
               usedBy:
                 description: List of the Tenant Control Planes, namespaced named, using this data store.
                 items:

charts/kamaji-crds/hack/kamaji.clastix.io_kubeconfiggenerators_spec.yaml

@@ -199,6 +199,10 @@ versions:
                     - resource
                   type: object
                 type: array
+              observedGeneration:
+                description: ObservedGeneration represents the .metadata.generation that was last reconciled.
+                format: int64
+                type: integer
               resources:
                 default: 0
                 description: Resources is the sum of targeted TenantControlPlane objects.

charts/kamaji-crds/hack/kamaji.clastix.io_tenantcontrolplanes_spec.yaml

@@ -8788,6 +8788,10 @@ versions:
                         type: string
                     type: object
                 type: object
+              observedGeneration:
+                description: ObservedGeneration represents the .metadata.generation that was last reconciled.
+                format: int64
+                type: integer
               storage:
                 description: Storage Status contains information about Kubernetes storage system
                 properties:

charts/kamaji/crds/kamaji.clastix.io_datastores.yaml

@@ -284,6 +284,10 @@ spec:
             status:
               description: DataStoreStatus defines the observed state of DataStore.
               properties:
+                observedGeneration:
+                  description: ObservedGeneration represents the .metadata.generation that was last reconciled.
+                  format: int64
+                  type: integer
                 usedBy:
                   description: List of the Tenant Control Planes, namespaced named, using this data store.
                   items:

charts/kamaji/crds/kamaji.clastix.io_kubeconfiggenerators.yaml

@@ -207,6 +207,10 @@ spec:
                       - resource
                     type: object
                   type: array
+                observedGeneration:
+                  description: ObservedGeneration represents the .metadata.generation that was last reconciled.
+                  format: int64
+                  type: integer
                 resources:
                   default: 0
                   description: Resources is the sum of targeted TenantControlPlane objects.

charts/kamaji/crds/kamaji.clastix.io_tenantcontrolplanes.yaml

@@ -8796,6 +8796,10 @@ spec:
                           type: string
                       type: object
                   type: object
+                observedGeneration:
+                  description: ObservedGeneration represents the .metadata.generation that was last reconciled.
+                  format: int64
+                  type: integer
                 storage:
                   description: Storage Status contains information about Kubernetes storage system
                   properties:

controllers/datastore_controller.go

@@ -73,6 +73,7 @@ func (r *DataStore) Reconcile(ctx context.Context, request reconcile.Request) (r
 			tcpSets.Insert(getNamespacedName(tcp.GetNamespace(), tcp.GetName()).String())
 		}
 
+		ds.Status.ObservedGeneration = ds.Generation
 		ds.Status.UsedBy = tcpSets.List()
 
 		if sErr := r.Client.Status().Update(ctx, &ds); sErr != nil {

controllers/kubeconfiggenerator_controller.go

@@ -88,6 +88,7 @@ func (r *KubeconfigGeneratorReconciler) Reconcile(ctx context.Context, req ctrl.
 	}
 
 	generator.Status = status
+	generator.Status.ObservedGeneration = generator.Generation
 
 	if statusErr := r.Client.Status().Update(ctx, &generator); statusErr != nil {
 		logger.Error(statusErr, "cannot update resource status")

controllers/tenantcontrolplane_controller.go

@@ -18,6 +18,7 @@ import (
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	k8stypes "k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/discovery"
+	"k8s.io/client-go/util/retry"
 	"k8s.io/client-go/util/workqueue"
 	"k8s.io/utils/clock"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -260,6 +261,23 @@ func (r *TenantControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R
 
 	log.Info(fmt.Sprintf("%s has been reconciled", tenantControlPlane.GetName()))
 
+	// Set ObservedGeneration only on successful reconciliation completion.
+	// This follows Cluster API conventions where ObservedGeneration indicates
+	// the controller has fully processed the given generation.
+	if err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
+		if getErr := r.Client.Get(ctx, req.NamespacedName, tenantControlPlane); getErr != nil {
+			return getErr
+		}
+
+		tenantControlPlane.Status.ObservedGeneration = tenantControlPlane.Generation
+
+		return r.Client.Status().Update(ctx, tenantControlPlane)
+	}); err != nil {
+		log.Error(err, "failed to update ObservedGeneration")
+
+		return ctrl.Result{}, err
+	}
+
 	return ctrl.Result{}, nil
 }
 

docs/content/reference/api.md

@@ -27982,6 +27982,15 @@ DataStoreStatus defines the observed state of DataStore.
         </tr>
     </thead>
     <tbody><tr>
+        <td><b>observedGeneration</b></td>
+        <td>integer</td>
+        <td>
+          ObservedGeneration represents the .metadata.generation that was last reconciled.<br/>
+          <br/>
+            <i>Format</i>: int64<br/>
+        </td>
+        <td>false</td>
+      </tr><tr>
         <td><b>usedBy</b></td>
         <td>[]string</td>
         <td>
@@ -28365,6 +28374,15 @@ In case of a different value compared to Resources, check the field errors.<br/>
           Errors is the list of failed kubeconfig generations.<br/>
         </td>
         <td>false</td>
+      </tr><tr>
+        <td><b>observedGeneration</b></td>
+        <td>integer</td>
+        <td>
+          ObservedGeneration represents the .metadata.generation that was last reconciled.<br/>
+          <br/>
+            <i>Format</i>: int64<br/>
+        </td>
+        <td>false</td>
       </tr></tbody>
 </table>
 
@@ -42555,6 +42573,15 @@ that are necessary to run a kubernetes control plane<br/>
           Kubernetes contains information about the reconciliation of the required Kubernetes resources deployed in the admin cluster<br/>
         </td>
         <td>false</td>
+      </tr><tr>
+        <td><b>observedGeneration</b></td>
+        <td>integer</td>
+        <td>
+          ObservedGeneration represents the .metadata.generation that was last reconciled.<br/>
+          <br/>
+            <i>Format</i>: int64<br/>
+        </td>
+        <td>false</td>
       </tr><tr>
         <td><b><a href="#tenantcontrolplanestatusstorage">storage</a></b></td>
         <td>object</td>

e2e/tcp_ready_test.go

@@ -5,10 +5,12 @@ package e2e
 
 import (
 	"context"
+	"time"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
 	pointer "k8s.io/utils/ptr"
 
 	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
@@ -59,5 +61,15 @@ var _ = Describe("Deploy a TenantControlPlane resource", func() {
 	// Check if TenantControlPlane resource has been created
 	It("Should be Ready", func() {
 		StatusMustEqualTo(tcp, kamajiv1alpha1.VersionReady)
+		// ObservedGeneration is set at the end of successful reconciliation,
+		// after status becomes Ready, so we need to wait for it.
+		Eventually(func() int64 {
+			if err := k8sClient.Get(context.Background(), types.NamespacedName{Name: tcp.GetName(), Namespace: tcp.GetNamespace()}, tcp); err != nil {
+				return -1
+			}
+
+			return tcp.Status.ObservedGeneration
+		}, 30*time.Second, time.Second).Should(Equal(tcp.Generation),
+			"ObservedGeneration should equal Generation after successful reconciliation")
 	})
 })