fix: migrate to dedicate resourcequota controller (#2104)

* chore * perfromance improvements Signed-off-by: Oliver Baehler <oliver@sudo-i.net> * perfromance improvements Signed-off-by: Oliver Baehler <oliver@sudo-i.net> * feat(performance): removed duplicate client calls from all admission paths Signed-off-by: Oliver Baehler <oliver@sudo-i.net> * feat(performance): removed duplicate client calls from all admission paths Signed-off-by: Oliver Baehler <oliver@sudo-i.net> * feat(performance): removed duplicate client calls from all admission paths Signed-off-by: Oliver Baehler <oliver@sudo-i.net> * feat(performance): removed duplicate client calls from all admission paths Signed-off-by: Oliver Baehler <oliver@sudo-i.net> * feat(performance): removed duplicate client calls from all admission paths Signed-off-by: Oliver Baehler <oliver@sudo-i.net> * feat(performance): removed duplicate client calls from all admission paths Signed-off-by: Oliver Baehler <oliver@sudo-i.net> * feat(performance): removed duplicate client calls from all admission paths Signed-off-by: Oliver Baehler <oliver@sudo-i.net> * feat: add globalresourcequota api Signed-off-by: Oliver Baehler <oliver@sudo-i.net> * feat: add globalresourcequota api Signed-off-by: Oliver Baehler <oliver@sudo-i.net> * feat: add globalresourcequota api Signed-off-by: Oliver Baehler <oliver@sudo-i.net> * feat: add globalresourcequota api Signed-off-by: Oliver Baehler <oliver@sudo-i.net> * feat: add globalresourcequota api Signed-off-by: Oliver Baehler <oliver@sudo-i.net> * feat: add globalresourcequota api Signed-off-by: Oliver Baehler <oliver@sudo-i.net> * fix: migrate to dedicate resourcequota controller Signed-off-by: Oliver Baehler <oliver@sudo-i.net> --------- Signed-off-by: Oliver Baehler <oliver@sudo-i.net>
2026-08-25 16:07:24 +00:00 · 2026-08-25 11:31:29 +02:00
parent 931bdfb5a9
commit def254e672
5 changed files with 260 additions and 57 deletions
@@ -20,7 +20,6 @@ import (
 	schedulingv1 "k8s.io/api/scheduling/v1"
 	storagev1 "k8s.io/api/storage/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apiserver/pkg/authentication/serviceaccount"
 	"k8s.io/client-go/discovery"
@@ -95,24 +94,6 @@ func (r *Manager) SetupWithManager(mgr ctrl.Manager, ctrlConfig utils.Controller
 				predicates.DeletionChangedPredicate{},
 			)),
 		).
-		Watches(
-			&corev1.ResourceQuota{},
-			handler.Funcs{
-				CreateFunc: func(ctx context.Context, e event.TypedCreateEvent[client.Object], _ workqueue.TypedRateLimitingInterface[reconcile.Request]) {
-					r.syncResourceQuotasForResourceQuota(ctx, e.Object)
-				},
-				UpdateFunc: func(ctx context.Context, e event.TypedUpdateEvent[client.Object], _ workqueue.TypedRateLimitingInterface[reconcile.Request]) {
-					r.syncResourceQuotasForResourceQuota(ctx, e.ObjectNew)
-				},
-				DeleteFunc: func(ctx context.Context, e event.TypedDeleteEvent[client.Object], _ workqueue.TypedRateLimitingInterface[reconcile.Request]) {
-					r.syncResourceQuotasForResourceQuota(ctx, e.Object)
-				},
-			},
-			builder.WithPredicates(predicate.Or(
-				predicates.TenantManagedResourceChangedPredicate{},
-				predicates.ResourceQuotaUsageChangedPredicate{},
-			)),
-		).
 		Owns(&rbacv1.RoleBinding{}, builder.WithPredicates(predicates.TenantManagedResourceChangedPredicate{})).
 		Watches(
 			&capsulev1beta2.CapsuleConfiguration{},
@@ -263,7 +244,11 @@ func (r *Manager) SetupWithManager(mgr ctrl.Manager, ctrlConfig utils.Controller
 		)
 	}

-	return ctrlBuilder.Complete(r)
+	if err := ctrlBuilder.Complete(r); err != nil {
+		return err
+	}
+
+	return r.setupResourceQuotaController(mgr, ctrlConfig)
 }

 func (r *Manager) Reconcile(ctx context.Context, request ctrl.Request) (result ctrl.Result, err error) {
@@ -445,36 +430,3 @@ func (r *Manager) reconcile(ctx context.Context, log logr.Logger, instance *caps

 	return err
 }
-
-func (r *Manager) syncResourceQuotasForResourceQuota(ctx context.Context, quota client.Object) {
-	owner := metav1.GetControllerOf(quota)
-	if owner == nil || owner.APIVersion != capsulev1beta2.GroupVersion.String() || owner.Kind != "Tenant" {
-		return
-	}
-
-	reader := r.reader
-	if reader == nil {
-		reader = r.Client
-	}
-
-	tenant := &capsulev1beta2.Tenant{}
-	if err := reader.Get(ctx, client.ObjectKey{Name: owner.Name}, tenant); err != nil {
-		if !apierrors.IsNotFound(err) {
-			r.Log.Error(err, "cannot retrieve Tenant for ResourceQuota sync", "tenant", owner.Name)
-		}
-
-		return
-	}
-
-	if tenant.DeletionTimestamp != nil {
-		return
-	}
-
-	if err := r.syncCustomResourceQuotaUsages(ctx, tenant); err != nil {
-		r.Log.Error(err, "cannot update custom ResourceQuota usages", "tenant", tenant.Name)
-	}
-
-	if err := r.syncResourceQuotas(ctx, r.Log, tenant); err != nil {
-		r.Log.Error(err, "cannot sync ResourceQuotas", "tenant", tenant.Name)
-	}
-}
@@ -0,0 +1,63 @@
+// Copyright 2020-2026 Project Capsule Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package tenant
+
+import (
+	"context"
+	"testing"
+
+	"github.com/go-logr/logr"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+
+	capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2"
+	"github.com/projectcapsule/capsule/internal/metrics"
+)
+
+func TestActiveTenantReconcilePrunesMissingNamespacesFromStatus(t *testing.T) {
+	t.Parallel()
+
+	scheme := runtime.NewScheme()
+	if err := corev1.AddToScheme(scheme); err != nil {
+		t.Fatalf("add core scheme: %v", err)
+	}
+	if err := capsulev1beta2.AddToScheme(scheme); err != nil {
+		t.Fatalf("add Capsule scheme: %v", err)
+	}
+
+	tenant := &capsulev1beta2.Tenant{
+		ObjectMeta: metav1.ObjectMeta{Name: "tenant-a"},
+		Status: capsulev1beta2.TenantStatus{
+			Namespaces: []string{"gone"},
+			Spaces: []*capsulev1beta2.TenantStatusNamespaceItem{{
+				Name: "gone",
+			}},
+		},
+	}
+
+	cl := fake.NewClientBuilder().
+		WithScheme(scheme).
+		WithIndex(
+			&corev1.Namespace{},
+			".metadata.ownerReferences[*].capsule",
+			func(client.Object) []string { return nil },
+		).
+		Build()
+	manager := &Manager{Client: cl, Metrics: metrics.NewTenantRecorder()}
+
+	if err := manager.reconcileActiveTenantNamespaces(context.Background(), logr.Discard(), tenant); err != nil {
+		t.Fatalf("reconcile active Tenant namespaces: %v", err)
+	}
+
+	if len(tenant.Status.Namespaces) != 0 {
+		t.Fatalf("status.namespaces = %v, want empty", tenant.Status.Namespaces)
+	}
+
+	if len(tenant.Status.Spaces) != 0 {
+		t.Fatalf("status.spaces = %v, want empty", tenant.Status.Spaces)
+	}
+}
@@ -8,15 +8,21 @@ import (
 	"testing"

 	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	k8smeta "k8s.io/apimachinery/pkg/api/meta"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/types"
 	dynamicfake "k8s.io/client-go/dynamic/fake"
 	k8stesting "k8s.io/client-go/testing"
+	"k8s.io/client-go/util/workqueue"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+	"sigs.k8s.io/controller-runtime/pkg/event"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"

 	capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2"
 	"github.com/projectcapsule/capsule/internal/metrics"
@@ -24,7 +30,7 @@ import (
 	"github.com/projectcapsule/capsule/pkg/api/meta"
 )

-func TestResourceQuotaWatchSyncsOnlyTheOwnerTenantQuotas(t *testing.T) {
+func TestResourceQuotaReconcileSyncsTenantQuotas(t *testing.T) {
 	t.Parallel()

 	scheme := runtime.NewScheme()
@@ -73,7 +79,11 @@ func TestResourceQuotaWatchSyncsOnlyTheOwnerTenantQuotas(t *testing.T) {
 	})
 	manager := &Manager{Client: cl, reader: cl, DynamicClient: dynamicClient, Metrics: metrics.NewTenantRecorder()}

-	manager.syncResourceQuotasForResourceQuota(context.Background(), trigger)
+	if _, err := manager.reconcileResourceQuotas(context.Background(), reconcile.Request{
+		NamespacedName: types.NamespacedName{Name: tenant.Name},
+	}); err != nil {
+		t.Fatalf("reconcile ResourceQuotas: %v", err)
+	}

 	updated := &corev1.ResourceQuota{}
 	if err := cl.Get(context.Background(), client.ObjectKeyFromObject(trigger), updated); err != nil {
@@ -94,3 +104,93 @@ func TestResourceQuotaWatchSyncsOnlyTheOwnerTenantQuotas(t *testing.T) {
 		t.Fatalf("custom ResourceQuota usage annotation = %q, want 1", got)
 	}
 }
+
+func TestResourceQuotaInitialEventsCoalesceByOwnerTenant(t *testing.T) {
+	t.Parallel()
+
+	scheme := runtime.NewScheme()
+	if err := capsulev1beta2.AddToScheme(scheme); err != nil {
+		t.Fatalf("add Capsule scheme: %v", err)
+	}
+
+	mapper := k8smeta.NewDefaultRESTMapper([]schema.GroupVersion{capsulev1beta2.GroupVersion})
+	mapper.Add(capsulev1beta2.GroupVersion.WithKind("Tenant"), k8smeta.RESTScopeRoot)
+
+	h := resourceQuotaEventHandler(scheme, mapper)
+	queue := workqueue.NewTypedRateLimitingQueue(workqueue.DefaultTypedControllerRateLimiter[reconcile.Request]())
+	defer queue.ShutDown()
+
+	controller := true
+	for i := 0; i < 100; i++ {
+		quota := &corev1.ResourceQuota{ObjectMeta: metav1.ObjectMeta{
+			Name:      "quota",
+			Namespace: "team-a",
+			OwnerReferences: []metav1.OwnerReference{{
+				APIVersion: capsulev1beta2.GroupVersion.String(),
+				Kind:       "Tenant",
+				Name:       "tenant-a",
+				Controller: &controller,
+			}},
+		}}
+
+		h.Create(context.Background(), event.CreateEvent{Object: quota, IsInInitialList: true}, queue)
+	}
+
+	if got := queue.Len(); got != 1 {
+		t.Fatalf("queued requests = %d, want 1", got)
+	}
+}
+
+type namespaceGoneClient struct {
+	client.Client
+}
+
+func (c *namespaceGoneClient) Create(
+	ctx context.Context,
+	obj client.Object,
+	opts ...client.CreateOption,
+) error {
+	if quota, ok := obj.(*corev1.ResourceQuota); ok {
+		return apierrors.NewNotFound(corev1.Resource("namespaces"), quota.Namespace)
+	}
+
+	return c.Client.Create(ctx, obj, opts...)
+}
+
+func TestResourceQuotaReconcileSkipsMissingStatusNamespace(t *testing.T) {
+	t.Parallel()
+
+	scheme := runtime.NewScheme()
+	if err := corev1.AddToScheme(scheme); err != nil {
+		t.Fatalf("add core scheme: %v", err)
+	}
+	if err := capsulev1beta2.AddToScheme(scheme); err != nil {
+		t.Fatalf("add Capsule scheme: %v", err)
+	}
+
+	tenant := &capsulev1beta2.Tenant{
+		ObjectMeta: metav1.ObjectMeta{Name: "tenant-a", UID: "tenant-a"},
+		Spec: capsulev1beta2.TenantSpec{ResourceQuota: api.ResourceQuotaSpec{
+			Scope: api.ResourceQuotaScopeNamespace,
+			Items: []corev1.ResourceQuotaSpec{{Hard: corev1.ResourceList{
+				corev1.ResourceLimitsCPU: resource.MustParse("1"),
+			}}},
+		}},
+		Status: capsulev1beta2.TenantStatus{
+			Namespaces: []string{"gone"},
+			Spaces: []*capsulev1beta2.TenantStatusNamespaceItem{{
+				Name: "gone",
+			}},
+		},
+	}
+
+	base := fake.NewClientBuilder().WithScheme(scheme).WithObjects(tenant).Build()
+	cl := &namespaceGoneClient{Client: base}
+	manager := &Manager{Client: cl, reader: cl, Metrics: metrics.NewTenantRecorder()}
+
+	if _, err := manager.reconcileResourceQuotas(context.Background(), reconcile.Request{
+		NamespacedName: types.NamespacedName{Name: tenant.Name},
+	}); err != nil {
+		t.Fatalf("reconcile ResourceQuotas with missing namespace: %v", err)
+	}
+}
@@ -388,9 +388,9 @@ func (r *Manager) syncResourceQuota(ctx context.Context, log logr.Logger, tenant
 			return retryErr
 		})
 		if err != nil {
-			if apierrors.HasStatusCause(err, corev1.NamespaceTerminatingCause) {
+			if apierrors.IsNotFound(err) || apierrors.HasStatusCause(err, corev1.NamespaceTerminatingCause) {
 				log.V(4).Info(
-					"skipping ResourceQuota sync because namespace is terminating",
+					"skipping ResourceQuota sync because namespace is gone or terminating",
 					"name", target.Name,
 					"namespace", target.Namespace,
 					"tenant", tenant.Name,
@@ -0,0 +1,88 @@
+// Copyright 2020-2026 Project Capsule Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package tenant
+
+import (
+	"context"
+	"errors"
+	"fmt"
+
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	k8smeta "k8s.io/apimachinery/pkg/api/meta"
+	"k8s.io/apimachinery/pkg/runtime"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/builder"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
+	"sigs.k8s.io/controller-runtime/pkg/predicate"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+
+	capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2"
+	"github.com/projectcapsule/capsule/internal/controllers/utils"
+	"github.com/projectcapsule/capsule/pkg/runtime/predicates"
+)
+
+func (r *Manager) setupResourceQuotaController(mgr ctrl.Manager, ctrlConfig utils.ControllerOptions) error {
+	// Keep ResourceQuota work behind a controller queue. Informer handlers must
+	// process their initial object list before controller workers can start, so
+	// running a Tenant-wide quota sync directly from a handler can prevent the
+	// source from ever reporting that it has synced.
+	return ctrl.NewControllerManagedBy(mgr).
+		Named("capsule/tenant-resourcequotas").
+		Watches(
+			&corev1.ResourceQuota{},
+			resourceQuotaEventHandler(mgr.GetScheme(), mgr.GetRESTMapper()),
+			builder.WithPredicates(predicate.Or(
+				predicates.TenantManagedResourceChangedPredicate{},
+				predicates.ResourceQuotaUsageChangedPredicate{},
+			)),
+		).
+		WithOptions(ctrlConfig.Runtime.ToControllerOptions()).
+		Complete(reconcile.Func(r.reconcileResourceQuotas))
+}
+
+func resourceQuotaEventHandler(scheme *runtime.Scheme, mapper k8smeta.RESTMapper) handler.EventHandler {
+	return handler.EnqueueRequestForOwner(
+		scheme,
+		mapper,
+		&capsulev1beta2.Tenant{},
+		handler.OnlyControllerOwner(),
+	)
+}
+
+func (r *Manager) reconcileResourceQuotas(
+	ctx context.Context,
+	request reconcile.Request,
+) (reconcile.Result, error) {
+	reader := r.reader
+	if reader == nil {
+		reader = r.Client
+	}
+
+	tenant := &capsulev1beta2.Tenant{}
+	if err := reader.Get(ctx, client.ObjectKey{Name: request.Name}, tenant); err != nil {
+		if apierrors.IsNotFound(err) {
+			return reconcile.Result{}, nil
+		}
+
+		return reconcile.Result{}, fmt.Errorf("retrieve Tenant for ResourceQuota sync: %w", err)
+	}
+
+	if tenant.DeletionTimestamp != nil {
+		return reconcile.Result{}, nil
+	}
+
+	var errs []error
+
+	if err := r.syncCustomResourceQuotaUsages(ctx, tenant); err != nil {
+		errs = append(errs, fmt.Errorf("update custom ResourceQuota usages: %w", err))
+	}
+
+	if err := r.syncResourceQuotas(ctx, r.Log, tenant); err != nil {
+		errs = append(errs, fmt.Errorf("sync ResourceQuotas: %w", err))
+	}
+
+	return reconcile.Result{}, errors.Join(errs...)
+}