diff --git a/internal/controllers/tenant/manager.go b/internal/controllers/tenant/manager.go index dc4da2e1..900b3dae 100644 --- a/internal/controllers/tenant/manager.go +++ b/internal/controllers/tenant/manager.go @@ -20,7 +20,6 @@ import ( schedulingv1 "k8s.io/api/scheduling/v1" storagev1 "k8s.io/api/storage/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apiserver/pkg/authentication/serviceaccount" "k8s.io/client-go/discovery" @@ -95,24 +94,6 @@ func (r *Manager) SetupWithManager(mgr ctrl.Manager, ctrlConfig utils.Controller predicates.DeletionChangedPredicate{}, )), ). - Watches( - &corev1.ResourceQuota{}, - handler.Funcs{ - CreateFunc: func(ctx context.Context, e event.TypedCreateEvent[client.Object], _ workqueue.TypedRateLimitingInterface[reconcile.Request]) { - r.syncResourceQuotasForResourceQuota(ctx, e.Object) - }, - UpdateFunc: func(ctx context.Context, e event.TypedUpdateEvent[client.Object], _ workqueue.TypedRateLimitingInterface[reconcile.Request]) { - r.syncResourceQuotasForResourceQuota(ctx, e.ObjectNew) - }, - DeleteFunc: func(ctx context.Context, e event.TypedDeleteEvent[client.Object], _ workqueue.TypedRateLimitingInterface[reconcile.Request]) { - r.syncResourceQuotasForResourceQuota(ctx, e.Object) - }, - }, - builder.WithPredicates(predicate.Or( - predicates.TenantManagedResourceChangedPredicate{}, - predicates.ResourceQuotaUsageChangedPredicate{}, - )), - ). Owns(&rbacv1.RoleBinding{}, builder.WithPredicates(predicates.TenantManagedResourceChangedPredicate{})). Watches( &capsulev1beta2.CapsuleConfiguration{}, @@ -263,7 +244,11 @@ func (r *Manager) SetupWithManager(mgr ctrl.Manager, ctrlConfig utils.Controller ) } - return ctrlBuilder.Complete(r) + if err := ctrlBuilder.Complete(r); err != nil { + return err + } + + return r.setupResourceQuotaController(mgr, ctrlConfig) } func (r *Manager) Reconcile(ctx context.Context, request ctrl.Request) (result ctrl.Result, err error) { @@ -445,36 +430,3 @@ func (r *Manager) reconcile(ctx context.Context, log logr.Logger, instance *caps return err } - -func (r *Manager) syncResourceQuotasForResourceQuota(ctx context.Context, quota client.Object) { - owner := metav1.GetControllerOf(quota) - if owner == nil || owner.APIVersion != capsulev1beta2.GroupVersion.String() || owner.Kind != "Tenant" { - return - } - - reader := r.reader - if reader == nil { - reader = r.Client - } - - tenant := &capsulev1beta2.Tenant{} - if err := reader.Get(ctx, client.ObjectKey{Name: owner.Name}, tenant); err != nil { - if !apierrors.IsNotFound(err) { - r.Log.Error(err, "cannot retrieve Tenant for ResourceQuota sync", "tenant", owner.Name) - } - - return - } - - if tenant.DeletionTimestamp != nil { - return - } - - if err := r.syncCustomResourceQuotaUsages(ctx, tenant); err != nil { - r.Log.Error(err, "cannot update custom ResourceQuota usages", "tenant", tenant.Name) - } - - if err := r.syncResourceQuotas(ctx, r.Log, tenant); err != nil { - r.Log.Error(err, "cannot sync ResourceQuotas", "tenant", tenant.Name) - } -} diff --git a/internal/controllers/tenant/namespaces_test.go b/internal/controllers/tenant/namespaces_test.go new file mode 100644 index 00000000..759116ca --- /dev/null +++ b/internal/controllers/tenant/namespaces_test.go @@ -0,0 +1,63 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package tenant + +import ( + "context" + "testing" + + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/internal/metrics" +) + +func TestActiveTenantReconcilePrunesMissingNamespacesFromStatus(t *testing.T) { + t.Parallel() + + scheme := runtime.NewScheme() + if err := corev1.AddToScheme(scheme); err != nil { + t.Fatalf("add core scheme: %v", err) + } + if err := capsulev1beta2.AddToScheme(scheme); err != nil { + t.Fatalf("add Capsule scheme: %v", err) + } + + tenant := &capsulev1beta2.Tenant{ + ObjectMeta: metav1.ObjectMeta{Name: "tenant-a"}, + Status: capsulev1beta2.TenantStatus{ + Namespaces: []string{"gone"}, + Spaces: []*capsulev1beta2.TenantStatusNamespaceItem{{ + Name: "gone", + }}, + }, + } + + cl := fake.NewClientBuilder(). + WithScheme(scheme). + WithIndex( + &corev1.Namespace{}, + ".metadata.ownerReferences[*].capsule", + func(client.Object) []string { return nil }, + ). + Build() + manager := &Manager{Client: cl, Metrics: metrics.NewTenantRecorder()} + + if err := manager.reconcileActiveTenantNamespaces(context.Background(), logr.Discard(), tenant); err != nil { + t.Fatalf("reconcile active Tenant namespaces: %v", err) + } + + if len(tenant.Status.Namespaces) != 0 { + t.Fatalf("status.namespaces = %v, want empty", tenant.Status.Namespaces) + } + + if len(tenant.Status.Spaces) != 0 { + t.Fatalf("status.spaces = %v, want empty", tenant.Status.Spaces) + } +} diff --git a/internal/controllers/tenant/resourcequota_watch_test.go b/internal/controllers/tenant/resourcequota_watch_test.go index 66e3cac9..0d49445d 100644 --- a/internal/controllers/tenant/resourcequota_watch_test.go +++ b/internal/controllers/tenant/resourcequota_watch_test.go @@ -8,15 +8,21 @@ import ( "testing" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + k8smeta "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" dynamicfake "k8s.io/client-go/dynamic/fake" k8stesting "k8s.io/client-go/testing" + "k8s.io/client-go/util/workqueue" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/reconcile" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" "github.com/projectcapsule/capsule/internal/metrics" @@ -24,7 +30,7 @@ import ( "github.com/projectcapsule/capsule/pkg/api/meta" ) -func TestResourceQuotaWatchSyncsOnlyTheOwnerTenantQuotas(t *testing.T) { +func TestResourceQuotaReconcileSyncsTenantQuotas(t *testing.T) { t.Parallel() scheme := runtime.NewScheme() @@ -73,7 +79,11 @@ func TestResourceQuotaWatchSyncsOnlyTheOwnerTenantQuotas(t *testing.T) { }) manager := &Manager{Client: cl, reader: cl, DynamicClient: dynamicClient, Metrics: metrics.NewTenantRecorder()} - manager.syncResourceQuotasForResourceQuota(context.Background(), trigger) + if _, err := manager.reconcileResourceQuotas(context.Background(), reconcile.Request{ + NamespacedName: types.NamespacedName{Name: tenant.Name}, + }); err != nil { + t.Fatalf("reconcile ResourceQuotas: %v", err) + } updated := &corev1.ResourceQuota{} if err := cl.Get(context.Background(), client.ObjectKeyFromObject(trigger), updated); err != nil { @@ -94,3 +104,93 @@ func TestResourceQuotaWatchSyncsOnlyTheOwnerTenantQuotas(t *testing.T) { t.Fatalf("custom ResourceQuota usage annotation = %q, want 1", got) } } + +func TestResourceQuotaInitialEventsCoalesceByOwnerTenant(t *testing.T) { + t.Parallel() + + scheme := runtime.NewScheme() + if err := capsulev1beta2.AddToScheme(scheme); err != nil { + t.Fatalf("add Capsule scheme: %v", err) + } + + mapper := k8smeta.NewDefaultRESTMapper([]schema.GroupVersion{capsulev1beta2.GroupVersion}) + mapper.Add(capsulev1beta2.GroupVersion.WithKind("Tenant"), k8smeta.RESTScopeRoot) + + h := resourceQuotaEventHandler(scheme, mapper) + queue := workqueue.NewTypedRateLimitingQueue(workqueue.DefaultTypedControllerRateLimiter[reconcile.Request]()) + defer queue.ShutDown() + + controller := true + for i := 0; i < 100; i++ { + quota := &corev1.ResourceQuota{ObjectMeta: metav1.ObjectMeta{ + Name: "quota", + Namespace: "team-a", + OwnerReferences: []metav1.OwnerReference{{ + APIVersion: capsulev1beta2.GroupVersion.String(), + Kind: "Tenant", + Name: "tenant-a", + Controller: &controller, + }}, + }} + + h.Create(context.Background(), event.CreateEvent{Object: quota, IsInInitialList: true}, queue) + } + + if got := queue.Len(); got != 1 { + t.Fatalf("queued requests = %d, want 1", got) + } +} + +type namespaceGoneClient struct { + client.Client +} + +func (c *namespaceGoneClient) Create( + ctx context.Context, + obj client.Object, + opts ...client.CreateOption, +) error { + if quota, ok := obj.(*corev1.ResourceQuota); ok { + return apierrors.NewNotFound(corev1.Resource("namespaces"), quota.Namespace) + } + + return c.Client.Create(ctx, obj, opts...) +} + +func TestResourceQuotaReconcileSkipsMissingStatusNamespace(t *testing.T) { + t.Parallel() + + scheme := runtime.NewScheme() + if err := corev1.AddToScheme(scheme); err != nil { + t.Fatalf("add core scheme: %v", err) + } + if err := capsulev1beta2.AddToScheme(scheme); err != nil { + t.Fatalf("add Capsule scheme: %v", err) + } + + tenant := &capsulev1beta2.Tenant{ + ObjectMeta: metav1.ObjectMeta{Name: "tenant-a", UID: "tenant-a"}, + Spec: capsulev1beta2.TenantSpec{ResourceQuota: api.ResourceQuotaSpec{ + Scope: api.ResourceQuotaScopeNamespace, + Items: []corev1.ResourceQuotaSpec{{Hard: corev1.ResourceList{ + corev1.ResourceLimitsCPU: resource.MustParse("1"), + }}}, + }}, + Status: capsulev1beta2.TenantStatus{ + Namespaces: []string{"gone"}, + Spaces: []*capsulev1beta2.TenantStatusNamespaceItem{{ + Name: "gone", + }}, + }, + } + + base := fake.NewClientBuilder().WithScheme(scheme).WithObjects(tenant).Build() + cl := &namespaceGoneClient{Client: base} + manager := &Manager{Client: cl, reader: cl, Metrics: metrics.NewTenantRecorder()} + + if _, err := manager.reconcileResourceQuotas(context.Background(), reconcile.Request{ + NamespacedName: types.NamespacedName{Name: tenant.Name}, + }); err != nil { + t.Fatalf("reconcile ResourceQuotas with missing namespace: %v", err) + } +} diff --git a/internal/controllers/tenant/resourcequotas.go b/internal/controllers/tenant/resourcequotas.go index 2be7d3db..c9b99897 100644 --- a/internal/controllers/tenant/resourcequotas.go +++ b/internal/controllers/tenant/resourcequotas.go @@ -388,9 +388,9 @@ func (r *Manager) syncResourceQuota(ctx context.Context, log logr.Logger, tenant return retryErr }) if err != nil { - if apierrors.HasStatusCause(err, corev1.NamespaceTerminatingCause) { + if apierrors.IsNotFound(err) || apierrors.HasStatusCause(err, corev1.NamespaceTerminatingCause) { log.V(4).Info( - "skipping ResourceQuota sync because namespace is terminating", + "skipping ResourceQuota sync because namespace is gone or terminating", "name", target.Name, "namespace", target.Namespace, "tenant", tenant.Name, diff --git a/internal/controllers/tenant/resourcequotas_controller.go b/internal/controllers/tenant/resourcequotas_controller.go new file mode 100644 index 00000000..4c59e4cf --- /dev/null +++ b/internal/controllers/tenant/resourcequotas_controller.go @@ -0,0 +1,88 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package tenant + +import ( + "context" + "errors" + "fmt" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + k8smeta "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/internal/controllers/utils" + "github.com/projectcapsule/capsule/pkg/runtime/predicates" +) + +func (r *Manager) setupResourceQuotaController(mgr ctrl.Manager, ctrlConfig utils.ControllerOptions) error { + // Keep ResourceQuota work behind a controller queue. Informer handlers must + // process their initial object list before controller workers can start, so + // running a Tenant-wide quota sync directly from a handler can prevent the + // source from ever reporting that it has synced. + return ctrl.NewControllerManagedBy(mgr). + Named("capsule/tenant-resourcequotas"). + Watches( + &corev1.ResourceQuota{}, + resourceQuotaEventHandler(mgr.GetScheme(), mgr.GetRESTMapper()), + builder.WithPredicates(predicate.Or( + predicates.TenantManagedResourceChangedPredicate{}, + predicates.ResourceQuotaUsageChangedPredicate{}, + )), + ). + WithOptions(ctrlConfig.Runtime.ToControllerOptions()). + Complete(reconcile.Func(r.reconcileResourceQuotas)) +} + +func resourceQuotaEventHandler(scheme *runtime.Scheme, mapper k8smeta.RESTMapper) handler.EventHandler { + return handler.EnqueueRequestForOwner( + scheme, + mapper, + &capsulev1beta2.Tenant{}, + handler.OnlyControllerOwner(), + ) +} + +func (r *Manager) reconcileResourceQuotas( + ctx context.Context, + request reconcile.Request, +) (reconcile.Result, error) { + reader := r.reader + if reader == nil { + reader = r.Client + } + + tenant := &capsulev1beta2.Tenant{} + if err := reader.Get(ctx, client.ObjectKey{Name: request.Name}, tenant); err != nil { + if apierrors.IsNotFound(err) { + return reconcile.Result{}, nil + } + + return reconcile.Result{}, fmt.Errorf("retrieve Tenant for ResourceQuota sync: %w", err) + } + + if tenant.DeletionTimestamp != nil { + return reconcile.Result{}, nil + } + + var errs []error + + if err := r.syncCustomResourceQuotaUsages(ctx, tenant); err != nil { + errs = append(errs, fmt.Errorf("update custom ResourceQuota usages: %w", err)) + } + + if err := r.syncResourceQuotas(ctx, r.Log, tenant); err != nil { + errs = append(errs, fmt.Errorf("sync ResourceQuotas: %w", err)) + } + + return reconcile.Result{}, errors.Join(errs...) +}