fix: consistently reconcile quotas from rules (#2083)

* fix: do not mutate on update and bound pvcs Signed-off-by: Oliver Baehler <oliver@sudo-i.net> * fix: do not mutate on update and bound pvcs Signed-off-by: Oliver Baehler <oliver@sudo-i.net> * fix: consistently reconcile quotas from rules Signed-off-by: Oliver Baehler <oliver@sudo-i.net> --------- Signed-off-by: Oliver Baehler <oliver@sudo-i.net>
2026-08-25 16:07:24 +00:00 · 2026-08-17 10:49:56 +02:00
parent bddb97644a
commit 305cabd4f6
8 changed files with 511 additions and 62 deletions
@@ -310,31 +310,7 @@ func validateGlobalResourceQuota(quota *capsulev1beta2.GlobalResourceQuota) erro
 }

 func validateHardLimit(hard, allocated corev1.ResourceList) error {
-	for name, usage := range allocated {
-		if usage.Sign() <= 0 {
-			continue
-		}
-
-		limit, exists := hard[name]
-		if !exists {
-			return fmt.Errorf(
-				"spec.quota.hard[%q] cannot be removed while %s is allocated",
-				name,
-				usage.String(),
-			)
-		}
-
-		if limit.Cmp(usage) < 0 {
-			return fmt.Errorf(
-				"spec.quota.hard[%q] cannot be reduced to %s while %s is allocated",
-				name,
-				limit.String(),
-				usage.String(),
-			)
-		}
-	}
-
-	return nil
+	return runtimequota.ValidateHardLimit("spec.quota.hard", hard, allocated)
 }

 func isManagedResourceQuota(req admission.Request, object any) bool {
@@ -5,6 +5,7 @@ package globalresourcequota

 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"sync"
 	"sync/atomic"
@@ -272,6 +273,37 @@ func TestValidateHardLimitAgainstAllocatedUsage(t *testing.T) {
 	}
 }

+func TestRuleManagedGlobalResourceQuotaCannotBeReducedBelowUsage(t *testing.T) {
+	t.Parallel()
+
+	oldQuota := globalQuotaForTest("tenant-a-shared", corev1.ResourceList{
+		corev1.ResourceRequestsCPU: resource.MustParse("3"),
+	})
+	oldQuota.Status.Total.Used = corev1.ResourceList{
+		corev1.ResourceRequestsCPU: resource.MustParse("3"),
+	}
+	controller := true
+	oldQuota.Labels = map[string]string{
+		meta.NewManagedByCapsuleLabel: meta.ValueController,
+		meta.RuleQuotaLabel:           "shared",
+	}
+	oldQuota.OwnerReferences = []metav1.OwnerReference{{
+		APIVersion: capsulev1beta2.GroupVersion.String(),
+		Kind:       "Tenant",
+		Name:       "tenant-a",
+		UID:        types.UID("tenant-a-uid"),
+		Controller: &controller,
+	}}
+
+	newQuota := oldQuota.DeepCopy()
+	newQuota.Spec.Quota.Hard[corev1.ResourceRequestsCPU] = resource.MustParse("2")
+
+	request := globalResourceQuotaUpdateRequest(t, oldQuota, newQuota)
+	if response := validateGlobalResourceQuotaRequest(context.Background(), ledgerClient(t), request); response == nil || response.Allowed {
+		t.Fatalf("managed rule quota decrease below usage was accepted: %#v", response)
+	}
+}
+
 func TestFormatExceededResources(t *testing.T) {
 	t.Parallel()

@@ -367,6 +399,29 @@ func globalQuotaForTest(name string, hard corev1.ResourceList) *capsulev1beta2.G
 	}
 }

+func globalResourceQuotaUpdateRequest(
+	t *testing.T,
+	oldQuota *capsulev1beta2.GlobalResourceQuota,
+	newQuota *capsulev1beta2.GlobalResourceQuota,
+) admission.Request {
+	t.Helper()
+
+	oldRaw, err := json.Marshal(oldQuota)
+	if err != nil {
+		t.Fatal(err)
+	}
+	newRaw, err := json.Marshal(newQuota)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	return admission.Request{AdmissionRequest: admissionv1.AdmissionRequest{
+		Operation: admissionv1.Update,
+		Object:    runtime.RawExtension{Raw: newRaw},
+		OldObject: runtime.RawExtension{Raw: oldRaw},
+	}}
+}
+
 func reservationForTest(
 	id string,
 	delta corev1.ResourceList,
@@ -0,0 +1,135 @@
+// Copyright 2020-2026 Project Capsule Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package validation
+
+import (
+	"context"
+	"strings"
+	"testing"
+
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
+
+	capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2"
+	"github.com/projectcapsule/capsule/pkg/api/rules"
+	"github.com/projectcapsule/capsule/pkg/runtime/configuration"
+	tenantutils "github.com/projectcapsule/capsule/pkg/tenant"
+)
+
+func TestValidateRuleQuotaUpdatesRejectsHardBelowUsageOrAllocation(t *testing.T) {
+	t.Setenv(configuration.EnvironmentControllerNamespace, "capsule-system")
+
+	scheme := runtime.NewScheme()
+	if err := capsulev1beta2.AddToScheme(scheme); err != nil {
+		t.Fatal(err)
+	}
+
+	oldTenant := ruleQuotaTenant("5")
+	globalQuota := &capsulev1beta2.GlobalResourceQuota{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: tenantutils.RuleGlobalResourceQuotaName(oldTenant, "services"),
+			UID:  types.UID("global-quota-uid"),
+		},
+		Status: capsulev1beta2.GlobalResourceQuotaStatus{
+			Total: capsulev1beta2.GlobalResourceQuotaUsage{Used: corev1.ResourceList{
+				corev1.ResourceServices: resource.MustParse("4"),
+			}},
+		},
+	}
+
+	t.Run("observed usage", func(t *testing.T) {
+		reader := fake.NewClientBuilder().WithScheme(scheme).WithObjects(globalQuota).Build()
+		response := validateRuleQuotaUpdates(context.Background(), reader, ruleQuotaTenant("3"), oldTenant)
+		assertRuleQuotaDenied(t, response, "cannot be reduced to 3 while 4 is allocated")
+	})
+
+	t.Run("inflight allocation", func(t *testing.T) {
+		quota := globalQuota.DeepCopy()
+		quota.Status.Total.Used[corev1.ResourceServices] = resource.MustParse("2")
+		ledger := ruleQuotaLedger(quota, "4")
+		reader := fake.NewClientBuilder().WithScheme(scheme).WithObjects(quota, ledger).Build()
+
+		response := validateRuleQuotaUpdates(context.Background(), reader, ruleQuotaTenant("3"), oldTenant)
+		assertRuleQuotaDenied(t, response, "cannot be reduced to 3 while 4 is allocated")
+	})
+
+	t.Run("equal to allocated", func(t *testing.T) {
+		quota := globalQuota.DeepCopy()
+		quota.Status.Total.Used[corev1.ResourceServices] = resource.MustParse("2")
+		ledger := ruleQuotaLedger(quota, "4")
+		reader := fake.NewClientBuilder().WithScheme(scheme).WithObjects(quota, ledger).Build()
+
+		if response := validateRuleQuotaUpdates(context.Background(), reader, ruleQuotaTenant("4"), oldTenant); response != nil {
+			t.Fatalf("hard limit equal to allocated usage was rejected: %#v", response)
+		}
+	})
+
+	t.Run("unchanged", func(t *testing.T) {
+		reader := &failingReader{}
+		if response := validateRuleQuotaUpdates(context.Background(), reader, oldTenant.DeepCopy(), oldTenant); response != nil {
+			t.Fatalf("unchanged hard limit performed quota validation: %#v", response)
+		}
+	})
+}
+
+type failingReader struct{ client.Reader }
+
+func (f *failingReader) Get(context.Context, client.ObjectKey, client.Object, ...client.GetOption) error {
+	return context.Canceled
+}
+
+func ruleQuotaTenant(hard string) *capsulev1beta2.Tenant {
+	return &capsulev1beta2.Tenant{
+		ObjectMeta: metav1.ObjectMeta{Name: "tenant-a", UID: types.UID("tenant-a-uid")},
+		Spec: capsulev1beta2.TenantSpec{Rules: []*rules.NamespaceRuleBodyTenant{{
+			NamespaceRuleBodyNamespace: &rules.NamespaceRuleBodyNamespace{
+				Quota: []rules.ResourceQuotaRule{{
+					Name: "services",
+					ResourceQuotaSpec: corev1.ResourceQuotaSpec{Hard: corev1.ResourceList{
+						corev1.ResourceServices: resource.MustParse(hard),
+					}},
+				}},
+			},
+		}}},
+	}
+}
+
+func ruleQuotaLedger(
+	quota *capsulev1beta2.GlobalResourceQuota,
+	allocated string,
+) *capsulev1beta2.QuantityLedger {
+	return &capsulev1beta2.QuantityLedger{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: configuration.ControllerNamespace(),
+			Name:      quota.GetLedgerName(),
+		},
+		Spec: capsulev1beta2.QuantityLedgerSpec{
+			TargetRef: capsulev1beta2.QuantityLedgerTargetRef{UID: quota.UID},
+		},
+		Status: capsulev1beta2.QuantityLedgerStatus{
+			ResourceQuota: &capsulev1beta2.QuantityLedgerResourceQuotaStatus{
+				Allocated: corev1.ResourceList{
+					corev1.ResourceServices: resource.MustParse(allocated),
+				},
+			},
+		},
+	}
+}
+
+func assertRuleQuotaDenied(t *testing.T, response *admission.Response, message string) {
+	t.Helper()
+
+	if response == nil || response.Allowed || response.Result == nil {
+		t.Fatal("rule quota update was accepted")
+	}
+	if !strings.Contains(response.Result.Message, message) {
+		t.Fatalf("denial message = %q, want substring %q", response.Result.Message, message)
+	}
+}
@@ -7,6 +7,9 @@ import (
 	"context"
 	"fmt"

+	corev1 "k8s.io/api/core/v1"
+	apiequality "k8s.io/apimachinery/pkg/api/equality"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	k8smeta "k8s.io/apimachinery/pkg/api/meta"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
@@ -15,8 +18,10 @@ import (
 	"github.com/projectcapsule/capsule/pkg/api/rules"
 	"github.com/projectcapsule/capsule/pkg/ruleengine"
 	ad "github.com/projectcapsule/capsule/pkg/runtime/admission"
+	"github.com/projectcapsule/capsule/pkg/runtime/configuration"
 	"github.com/projectcapsule/capsule/pkg/runtime/events"
 	"github.com/projectcapsule/capsule/pkg/runtime/handlers"
+	runtimequota "github.com/projectcapsule/capsule/pkg/runtime/quota"
 	tenantutils "github.com/projectcapsule/capsule/pkg/tenant"
 )

@@ -59,20 +64,114 @@ func (h *RuleValidationHandler) OnDelete(
 }

 func (h *RuleValidationHandler) OnUpdate(
-	_ client.Client,
-	_ client.Reader,
+	c client.Client,
+	reader client.Reader,
 	tnt *capsulev1beta2.Tenant,
 	old *capsulev1beta2.Tenant,
 	decoder admission.Decoder,
 	_ events.EventRecorder,
 ) handlers.Func {
-	return func(_ context.Context, req admission.Request) *admission.Response {
+	return func(ctx context.Context, req admission.Request) *admission.Response {
 		if response := h.handle(tnt, req); response != nil {
 			return response
 		}

+		if reader == nil {
+			reader = c
+		}
+
+		return validateRuleQuotaUpdates(ctx, reader, tnt, old)
+	}
+}
+
+func validateRuleQuotaUpdates(
+	ctx context.Context,
+	reader client.Reader,
+	tnt *capsulev1beta2.Tenant,
+	old *capsulev1beta2.Tenant,
+) *admission.Response {
+	if reader == nil || tnt == nil || old == nil {
 		return nil
 	}
+
+	oldHard := make(map[string]corev1.ResourceList)
+
+	for _, rule := range old.Spec.Rules {
+		if rule == nil || rule.NamespaceRuleBodyNamespace == nil {
+			continue
+		}
+
+		for _, quota := range rule.Quota {
+			oldHard[quota.Name] = quota.Hard
+		}
+	}
+
+	for ruleIndex, rule := range tnt.Spec.Rules {
+		if rule == nil || rule.NamespaceRuleBodyNamespace == nil {
+			continue
+		}
+
+		for quotaIndex, quota := range rule.Quota {
+			previous, existed := oldHard[quota.Name]
+			if !existed || apiequality.Semantic.DeepEqual(previous, quota.Hard) {
+				continue
+			}
+
+			globalQuota := &capsulev1beta2.GlobalResourceQuota{}
+			if err := reader.Get(ctx, client.ObjectKey{
+				Name: tenantutils.RuleGlobalResourceQuotaName(tnt, quota.Name),
+			}, globalQuota); err != nil {
+				if apierrors.IsNotFound(err) {
+					continue
+				}
+
+				return ad.ErroredResponse(fmt.Errorf(
+					"cannot validate rules[%d].quota[%d] against its GlobalResourceQuota: %w",
+					ruleIndex,
+					quotaIndex,
+					err,
+				))
+			}
+
+			path := fmt.Sprintf("rules[%d].quota[%d].hard", ruleIndex, quotaIndex)
+			if err := runtimequota.ValidateHardLimit(path, quota.Hard, globalQuota.Status.Total.Used); err != nil {
+				return ad.Deny(err.Error())
+			}
+
+			ledger := &capsulev1beta2.QuantityLedger{}
+
+			err := reader.Get(ctx, client.ObjectKey{
+				Namespace: configuration.ControllerNamespace(),
+				Name:      globalQuota.GetLedgerName(),
+			}, ledger)
+			if apierrors.IsNotFound(err) {
+				continue
+			}
+
+			if err != nil {
+				return ad.ErroredResponse(fmt.Errorf(
+					"cannot validate rules[%d].quota[%d] against its QuantityLedger: %w",
+					ruleIndex,
+					quotaIndex,
+					err,
+				))
+			}
+
+			if ledger.Spec.TargetRef.UID != globalQuota.UID || ledger.Status.ResourceQuota == nil {
+				continue
+			}
+
+			if err := runtimequota.ValidateHardLimit(
+				path,
+				quota.Hard,
+				ledger.Status.ResourceQuota.Allocated,
+			); err != nil {
+				return ad.Deny(err.Error())
+			}
+		}
+	}
+
+	return nil
 }

 func (h *RuleValidationHandler) handle(