package e2e import ( "context" "fmt" "sort" "sync" "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "sigs.k8s.io/controller-runtime/pkg/client" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/selection" "k8s.io/apimachinery/pkg/types" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/client-go/kubernetes/scheme" "k8s.io/utils/ptr" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" "github.com/projectcapsule/capsule/pkg/api/meta" capmeta "github.com/projectcapsule/capsule/pkg/api/meta" "github.com/projectcapsule/capsule/pkg/api/runtime" "github.com/projectcapsule/capsule/pkg/runtime/quota" "github.com/projectcapsule/capsule/pkg/runtime/selectors" ) func expectLedgerSettled(ctx context.Context, namespace, name string) { Eventually(func(g Gomega) { obj := &capsulev1beta2.QuantityLedger{} g.Expect(k8sClient.Get(ctx, types.NamespacedName{ Name: name, Namespace: namespace, }, obj)).To(Succeed(), "failed to get QuantityLedger %s/%s", namespace, name, ) g.Expect(obj.Status.Reserved.IsZero()).To(BeTrue(), "ledger %s/%s still has reserved=%q reservations=%+v pendingDeletes=%+v", namespace, name, obj.Status.Reserved.String(), obj.Status.Reservations, obj.Status.PendingDeletes, ) g.Expect(obj.Status.PendingDeletes).To(BeEmpty(), "ledger %s/%s still has pendingDeletes=%+v reserved=%q reservations=%+v", namespace, name, obj.Status.PendingDeletes, obj.Status.Reserved.String(), obj.Status.Reservations, ) }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) } func expectGlobalQuotaUsedAndClaims(ctx context.Context, name string, used string, claims int) { expectedUsed := resource.MustParse(used) Eventually(func(g Gomega) { obj := &capsulev1beta2.GlobalCustomQuota{} g.Expect(k8sClient.Get(ctx, types.NamespacedName{Name: name}, obj)).To(Succeed(), "failed to get GlobalCustomQuota %s", name) g.Expect( obj.Status.Usage.Used.Cmp(expectedUsed), ).To(Equal(0), "unexpected used value for GlobalCustomQuota %s: used=%q expectedUsed=%q available=%q claims=%d expectedClaims=%d", name, obj.Status.Usage.Used.String(), used, obj.Status.Usage.Available.String(), len(obj.Status.Claims), claims, ) g.Expect(obj.Status.Usage.Used.Sign()).To(BeNumerically(">=", 0), "usage went negative for GlobalCustomQuota %s: used=%q", name, obj.Status.Usage.Used.String()) g.Expect(obj.Status.Usage.Available.Sign()).To(BeNumerically(">=", 0), "available went negative for GlobalCustomQuota %s: available=%q", name, obj.Status.Usage.Available.String()) g.Expect(len(obj.Status.Claims)).To(Equal(claims), "unexpected claims for GlobalCustomQuota %s: used=%q expectedUsed=%q available=%q claims=%d expectedClaims=%d", name, obj.Status.Usage.Used.String(), used, obj.Status.Usage.Available.String(), len(obj.Status.Claims), claims, ) }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) } func expectCustomQuotaUsedAndClaims(ctx context.Context, namespace, name string, used string, claims int) { expectedUsed := resource.MustParse(used) Eventually(func(g Gomega) { obj := &capsulev1beta2.CustomQuota{} g.Expect(k8sClient.Get(ctx, types.NamespacedName{ Name: name, Namespace: namespace, }, obj)).To(Succeed(), "failed to get CustomQuota %s/%s", namespace, name) g.Expect( obj.Status.Usage.Used.Cmp(expectedUsed), ).To(Equal(0), "unexpected used value for CustomQuota %s/%s: used=%q expectedUsed=%q available=%q claims=%d expectedClaims=%d", namespace, name, obj.Status.Usage.Used.String(), used, obj.Status.Usage.Available.String(), len(obj.Status.Claims), claims, ) g.Expect(obj.Status.Usage.Used.Sign()).To(BeNumerically(">=", 0), "usage went negative for CustomQuota %s/%s: used=%q", namespace, name, obj.Status.Usage.Used.String()) g.Expect(obj.Status.Usage.Available.Sign()).To(BeNumerically(">=", 0), "available went negative for CustomQuota %s/%s: available=%q", namespace, name, obj.Status.Usage.Available.String()) g.Expect(len(obj.Status.Claims)).To(Equal(claims), "unexpected claims for CustomQuota %s/%s: used=%q expectedUsed=%q available=%q claims=%d expectedClaims=%d", namespace, name, obj.Status.Usage.Used.String(), used, obj.Status.Usage.Available.String(), len(obj.Status.Claims), claims, ) }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) } func awaitGlobalQuotaReady(ctx context.Context, name string) { Eventually(func(g Gomega) { gq := &capsulev1beta2.GlobalCustomQuota{} g.Expect(k8sClient.Get(ctx, types.NamespacedName{Name: name}, gq)).To(Succeed()) g.Expect(gq.Status.Targets).NotTo(Equal(0)) // status should be initialized by the controller g.Expect(gq.Status.Usage.Used.String()).NotTo(BeEmpty()) g.Expect(gq.Status.Usage.Available.String()).NotTo(BeEmpty()) ledger := &capsulev1beta2.QuantityLedger{} g.Expect(k8sClient.Get(ctx, types.NamespacedName{ Name: name, Namespace: "capsule-system", }, ledger)).To(Succeed()) g.Expect(capmeta.IsStatusConditionTrue(gq.Status.Conditions, capmeta.ReadyCondition)).To(BeTrue()) g.Expect(ledger.Spec.TargetRef.Kind).To(Equal("GlobalCustomQuota")) g.Expect(ledger.Spec.TargetRef.Name).To(Equal(name)) // initial ledger should be settled before test objects are created g.Expect(ledger.Status.Reserved.IsZero()).To(BeTrue()) g.Expect(ledger.Status.PendingDeletes).To(BeEmpty()) }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) } func awaitCustomQuotaReady(ctx context.Context, namespace, name string) { Eventually(func(g Gomega) { cq := &capsulev1beta2.CustomQuota{} g.Expect(k8sClient.Get(ctx, types.NamespacedName{ Name: name, Namespace: namespace, }, cq)).To(Succeed()) g.Expect(cq.Status.Usage.Used.String()).NotTo(BeEmpty()) g.Expect(cq.Status.Usage.Available.String()).NotTo(BeEmpty()) ledger := &capsulev1beta2.QuantityLedger{} g.Expect(k8sClient.Get(ctx, types.NamespacedName{ Name: name, Namespace: namespace, }, ledger)).To(Succeed()) g.Expect(ledger.Spec.TargetRef.Kind).To(Equal("CustomQuota")) g.Expect(ledger.Spec.TargetRef.Name).To(Equal(name)) g.Expect(ledger.Spec.TargetRef.Namespace).To(Equal(namespace)) g.Expect(ledger.Status.Reserved.IsZero()).To(BeTrue()) g.Expect(ledger.Status.PendingDeletes).To(BeEmpty()) }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) } func getGlobalQuota(ctx context.Context, name string) *capsulev1beta2.GlobalCustomQuota { obj := &capsulev1beta2.GlobalCustomQuota{} Expect(k8sClient.Get(ctx, types.NamespacedName{Name: name}, obj)).To(Succeed()) return obj } func getLedger(ctx context.Context, namespace, name string) *capsulev1beta2.QuantityLedger { obj := &capsulev1beta2.QuantityLedger{} Expect(k8sClient.Get(ctx, types.NamespacedName{ Name: name, Namespace: namespace, }, obj)).To(Succeed()) return obj } var _ = Describe("when GlobalCustomQuota uses ledger-backed reconciliation", Ordered, Label("global", "globalcustomquota", "customquota", "ledger", "skip-on-openshift"), Ordered, func() { const ( testNamespace = "global-custom-quota-e2e-test" tenantLabel = "e2e.capsule.dev/test-suite" tenantValue = "global-custom-quota-e2e" ) var ( ctx context.Context ns *corev1.Namespace ) awaitAllGlobalQuotasReady := func(names ...string) { for _, name := range names { awaitGlobalQuotaReady(ctx, name) } } expectPodCreationDeniedContaining := func(build func(name string) *corev1.Pod, expected string) { Eventually(func() string { name := fmt.Sprintf("denied-%d", time.Now().UnixNano()) obj := build(name) err := k8sClient.Create(ctx, obj) if err == nil { _ = k8sClient.Delete(ctx, obj) return "" } return err.Error() }, defaultTimeoutInterval, defaultPollInterval).Should(ContainSubstring(expected)) } expectGlobalQuotaWildcardNamespaces := func(name string) { Eventually(func(g Gomega) { obj := &capsulev1beta2.GlobalCustomQuota{} g.Expect(k8sClient.Get(ctx, types.NamespacedName{Name: name}, obj)).To(Succeed()) g.Expect(obj.Status.Namespaces).To(ContainElement("*"), "expected wildcard namespace status for %s, got=%v", name, obj.Status.Namespaces, ) }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) } expectGlobalQuotaNamespaces := func(name string, expected ...string) { Eventually(func(g Gomega) { obj := &capsulev1beta2.GlobalCustomQuota{} g.Expect(k8sClient.Get(ctx, types.NamespacedName{Name: name}, obj)).To(Succeed()) actual := append([]string(nil), obj.Status.Namespaces...) sort.Strings(actual) want := append([]string(nil), expected...) sort.Strings(want) g.Expect(actual).To(Equal(want), "unexpected status.namespaces for %s: got=%v want=%v", name, actual, want, ) }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) } BeforeAll(func() { ctx = context.Background() utilruntime.Must(capsulev1beta2.AddToScheme(scheme.Scheme)) }) BeforeEach(func() { ns = &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ Name: testNamespace, Labels: map[string]string{ tenantLabel: tenantValue, "env": "e2e", }, }, } EventuallyCreation(func() error { ns.ResourceVersion = "" return k8sClient.Create(ctx, ns) }).Should(Succeed()) }) AfterEach(func() { req, err := labels.NewRequirement("e2e.capsule.dev/test-suite", selection.Equals, []string{"globalcustomquota-ledger"}) Expect(err).NotTo(HaveOccurred()) var customQuotaList capsulev1beta2.CustomQuotaList Expect(k8sClient.List( context.TODO(), &customQuotaList, client.MatchingLabelsSelector{ Selector: labels.NewSelector().Add(*req), }, )).Should(Succeed()) for i := range customQuotaList.Items { EventuallyDeletion(&customQuotaList.Items[i]) } var list capsulev1beta2.GlobalCustomQuotaList Expect(k8sClient.List( context.TODO(), &list, client.MatchingLabelsSelector{ Selector: labels.NewSelector().Add(*req), }, )).Should(Succeed()) for i := range list.Items { EventuallyDeletion(&list.Items[i]) } ForceDeleteNamespace(ctx, testNamespace) }) It("aggregates a custom pod quantity path and settles the corresponding ledger", Label("skip-on-openshift"), func() { q := &capsulev1beta2.GlobalCustomQuota{ ObjectMeta: metav1.ObjectMeta{ Name: "gq-pod-cpu-requests", Labels: map[string]string{ "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", }, }, Spec: capsulev1beta2.GlobalCustomQuotaSpec{ CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ Limit: resource.MustParse("500m"), Sources: []capsulev1beta2.CustomQuotaSpecSource{ { VersionKind: runtime.VersionKind{ APIVersion: "v1", Kind: "Pod", }, CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ Operation: quota.OpAdd, Path: ".spec.containers[*].resources.requests.cpu", Selectors: []selectors.SelectorWithFields{ { LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ "track": "yes", }, }, }, }, }, }, }, }, }, } EventuallyCreation(func() error { return k8sClient.Create(ctx, q) }).Should(Succeed()) awaitGlobalQuotaReady(ctx, q.GetName()) dep := MakeDeployment(testNamespace, "cpu-requests", 2, map[string]string{ "track": "yes", }, "100m") EventuallyCreation(func() error { dep.ResourceVersion = "" return k8sClient.Create(ctx, dep) }).Should(Succeed()) ExpectPodsForDeployment(ctx, testNamespace, "cpu-requests", 2) expectGlobalQuotaUsedAndClaims(ctx, q.GetName(), "200m", 2) expectLedgerSettled(ctx, ControllerNamespace, q.GetName()) ScaleDeployment(ctx, testNamespace, "cpu-requests", 4) ExpectPodsForDeployment(ctx, testNamespace, "cpu-requests", 4) expectGlobalQuotaUsedAndClaims(ctx, q.GetName(), "400m", 4) expectLedgerSettled(ctx, ControllerNamespace, q.GetName()) ledger := getLedger(ctx, ControllerNamespace, q.GetName()) Expect(ledger.Spec.TargetRef.Kind).To(Equal("GlobalCustomQuota")) Expect(ledger.Spec.TargetRef.Name).To(Equal(q.GetName())) gq := getGlobalQuota(ctx, q.GetName()) Expect(gq.Status.Usage.Used.Cmp(resource.MustParse("400m"))).To(Equal(0)) }) It("marks the quota not ready when an existing matching object has no value at the configured quantity path", func() { q := &capsulev1beta2.GlobalCustomQuota{ ObjectMeta: metav1.ObjectMeta{ Name: "gq-missing-path-not-ready", Labels: map[string]string{ "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", }, }, Spec: capsulev1beta2.GlobalCustomQuotaSpec{ CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ Limit: resource.MustParse("10Gi"), Sources: []capsulev1beta2.CustomQuotaSpecSource{ { VersionKind: runtime.VersionKind{ APIVersion: "v1", Kind: "Pod", }, CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ Operation: quota.OpAdd, Path: ".spec.volumes[*].emptyDir.sizeLimit", }, }, }, }, NamespaceSelectors: []selectors.NamespaceSelector{ { LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ tenantLabel: tenantValue, }, }, }, }, }, } pod := MakePod(testNamespace, "missing-emptydir-size", nil, nil, "nginx:1.27.0", "", "") EventuallyCreation(func() error { pod.ResourceVersion = "" return k8sClient.Create(ctx, pod) }).Should(Succeed()) EventuallyCreation(func() error { return k8sClient.Create(ctx, q) }).Should(Succeed()) Eventually(func(g Gomega) { obj := &capsulev1beta2.GlobalCustomQuota{} g.Expect(k8sClient.Get(ctx, types.NamespacedName{Name: q.GetName()}, obj)).To(Succeed()) cond := obj.Status.Conditions.GetConditionByType(meta.ReadyCondition) g.Expect(cond).NotTo(BeNil()) g.Expect(cond.Status).To(Equal(metav1.ConditionFalse)) g.Expect(cond.Message).To(ContainSubstring("did not resolve to any value")) }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) }) It("denies creating a matching object when the configured quantity path resolves to no value", Label("skip-on-openshift"), func() { q := &capsulev1beta2.GlobalCustomQuota{ ObjectMeta: metav1.ObjectMeta{ Name: "gq-missing-path-deny-create", Labels: map[string]string{ "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", }, }, Spec: capsulev1beta2.GlobalCustomQuotaSpec{ CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ Limit: resource.MustParse("10Gi"), Sources: []capsulev1beta2.CustomQuotaSpecSource{ { VersionKind: runtime.VersionKind{ APIVersion: "v1", Kind: "Pod", }, CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ Operation: quota.OpAdd, Path: ".spec.volumes[*].emptyDir.sizeLimit", }, }, }, }, NamespaceSelectors: []selectors.NamespaceSelector{ { LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ tenantLabel: tenantValue, }, }, }, }, }, } EventuallyCreation(func() error { return k8sClient.Create(ctx, q) }).Should(Succeed()) awaitGlobalQuotaReady(ctx, q.GetName()) Eventually(func() error { pod := MakePod(testNamespace, "missing-path-denied", nil, nil, "nginx:1.27.0", "", "") return k8sClient.Create(ctx, pod) }, defaultTimeoutInterval, defaultPollInterval).Should( MatchError(ContainSubstring("did not resolve to any value")), ) expectGlobalQuotaUsedAndClaims(ctx, q.GetName(), "0", 0) expectLedgerSettled(ctx, ControllerNamespace, q.GetName()) }) It("remains consistent under concurrent pod creations for a GlobalCustomQuota", Label("skip-on-openshift"), func() { q := &capsulev1beta2.GlobalCustomQuota{ ObjectMeta: metav1.ObjectMeta{ Name: "gq-concurrent-pod-count", Labels: map[string]string{ "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", }, }, Spec: capsulev1beta2.GlobalCustomQuotaSpec{ CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ Limit: resource.MustParse("10"), Sources: []capsulev1beta2.CustomQuotaSpecSource{ { VersionKind: runtime.VersionKind{ APIVersion: "v1", Kind: "Pod", }, CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ Operation: quota.OpCount, Selectors: []selectors.SelectorWithFields{ { LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ "track": "yes", }, }, }, }, }, }, }, }, NamespaceSelectors: []selectors.NamespaceSelector{ { LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ tenantLabel: tenantValue, }, }, }, }, }, } EventuallyCreation(func() error { return k8sClient.Create(ctx, q) }).Should(Succeed()) awaitGlobalQuotaReady(ctx, q.GetName()) const ( totalAttempts = 100 expectedSuccess = 10 ) type result struct { name string err error } results := make(chan result, totalAttempts) var ( mu sync.Mutex succeeded int successSeen = map[string]struct{}{} ) for i := 0; i < totalAttempts; i++ { i := i go func() { name := fmt.Sprintf("gq-concurrent-pod-%02d", i) pod := MakePod( testNamespace, name, map[string]string{"track": "yes"}, nil, "nginx:1.27.0", "", "", ) Eventually(func() error { mu.Lock() done := succeeded >= expectedSuccess mu.Unlock() if done { return nil } err := k8sClient.Create(ctx, pod) if err != nil { return err } mu.Lock() defer mu.Unlock() if _, exists := successSeen[name]; !exists { successSeen[name] = struct{}{} succeeded++ } return nil }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) mu.Lock() _, ok := successSeen[name] mu.Unlock() if ok { results <- result{name: name, err: nil} return } results <- result{name: name, err: fmt.Errorf("not created because expected success count was already reached")} }() } var failed int var failedMsgs []string for i := 0; i < totalAttempts; i++ { res := <-results if res.err != nil { failed++ failedMsgs = append(failedMsgs, fmt.Sprintf("%s: %v", res.name, res.err)) } } mu.Lock() successNames := make([]string, 0, len(successSeen)) for name := range successSeen { successNames = append(successNames, name) } succeeded = len(successNames) mu.Unlock() sort.Strings(successNames) gq := getGlobalQuota(ctx, q.GetName()) ledger := getLedger(ctx, ControllerNamespace, q.GetName()) Expect(succeeded).To(Equal(expectedSuccess), "unexpected number of successful concurrent pod creations\nsuccesses=%d\nfailures=%d\nsuccessNames=%v\nfailureDetails=%v\nquotaUsed=%q\nquotaAvailable=%q\nclaims=%d\nledgerReserved=%q\nledgerReservations=%+v\nledgerPendingDeletes=%+v", succeeded, failed, successNames, failedMsgs, gq.Status.Usage.Used.String(), gq.Status.Usage.Available.String(), len(gq.Status.Claims), ledger.Status.Reserved.String(), ledger.Status.Reservations, ledger.Status.PendingDeletes, ) expectGlobalQuotaUsedAndClaims(ctx, q.GetName(), "10", 10) expectLedgerSettled(ctx, ControllerNamespace, q.GetName()) }) It("uses the smallest matching quota as authoritative while accounting successful pod count in both global and namespaced quotas", func() { gq := &capsulev1beta2.GlobalCustomQuota{ ObjectMeta: metav1.ObjectMeta{ Name: "gq-mixed-pod-count", Labels: map[string]string{ "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", }, }, Spec: capsulev1beta2.GlobalCustomQuotaSpec{ CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ Limit: resource.MustParse("5"), Sources: []capsulev1beta2.CustomQuotaSpecSource{ { VersionKind: runtime.VersionKind{ APIVersion: "v1", Kind: "Pod", }, CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ Operation: quota.OpCount, }, }, }, }, NamespaceSelectors: []selectors.NamespaceSelector{ { LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ tenantLabel: tenantValue, }, }, }, }, }, } cq := &capsulev1beta2.CustomQuota{ ObjectMeta: metav1.ObjectMeta{ Name: "cq-mixed-pod-count", Namespace: testNamespace, Labels: map[string]string{ "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", }, }, Spec: capsulev1beta2.CustomQuotaSpec{ Limit: resource.MustParse("2"), Sources: []capsulev1beta2.CustomQuotaSpecSource{ { VersionKind: runtime.VersionKind{ APIVersion: "v1", Kind: "Pod", }, CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ Operation: quota.OpCount, }, }, }, }, } EventuallyCreation(func() error { return k8sClient.Create(ctx, gq) }).Should(Succeed()) EventuallyCreation(func() error { return k8sClient.Create(ctx, cq) }).Should(Succeed()) awaitGlobalQuotaReady(ctx, gq.GetName()) awaitCustomQuotaReady(ctx, testNamespace, cq.GetName()) pod1 := MakePod(testNamespace, "mixed-count-1", nil, nil, "nginx:1.27.0", "", "") pod2 := MakePod(testNamespace, "mixed-count-2", nil, nil, "nginx:1.27.0", "", "") EventuallyCreation(func() error { pod1.ResourceVersion = "" return k8sClient.Create(ctx, pod1) }).Should(Succeed()) EventuallyCreation(func() error { pod2.ResourceVersion = "" return k8sClient.Create(ctx, pod2) }).Should(Succeed()) expectGlobalQuotaUsedAndClaims(ctx, gq.GetName(), "2", 2) expectCustomQuotaUsedAndClaims(ctx, cq.GetNamespace(), cq.GetName(), "2", 2) expectLedgerSettled(ctx, ControllerNamespace, gq.GetName()) expectLedgerSettled(ctx, cq.GetNamespace(), cq.GetName()) Eventually(func() error { pod3 := MakePod(testNamespace, "mixed-count-3", nil, nil, "nginx:1.27.0", "", "") return k8sClient.Create(ctx, pod3) }, defaultTimeoutInterval, defaultPollInterval).Should( MatchError(ContainSubstring(`CustomQuota "cq-mixed-pod-count"`)), ) expectGlobalQuotaUsedAndClaims(ctx, gq.GetName(), "2", 2) expectCustomQuotaUsedAndClaims(ctx, cq.GetNamespace(), cq.GetName(), "2", 2) }) It("uses the smallest matching quota as authoritative while accounting successful cpu usage in both global and namespaced quotas", func() { gq := &capsulev1beta2.GlobalCustomQuota{ ObjectMeta: metav1.ObjectMeta{ Name: "gq-mixed-pod-cpu", Labels: map[string]string{ "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", }, }, Spec: capsulev1beta2.GlobalCustomQuotaSpec{ CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ Limit: resource.MustParse("500m"), Sources: []capsulev1beta2.CustomQuotaSpecSource{ { VersionKind: runtime.VersionKind{ APIVersion: "v1", Kind: "Pod", }, CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ Operation: quota.OpAdd, Path: ".spec.containers[*].resources.requests.cpu", }, }, }, }, NamespaceSelectors: []selectors.NamespaceSelector{ { LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ tenantLabel: tenantValue, }, }, }, }, }, } cq := &capsulev1beta2.CustomQuota{ ObjectMeta: metav1.ObjectMeta{ Name: "cq-mixed-pod-cpu", Namespace: testNamespace, Labels: map[string]string{ "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", }, }, Spec: capsulev1beta2.CustomQuotaSpec{ Limit: resource.MustParse("200m"), Sources: []capsulev1beta2.CustomQuotaSpecSource{ { VersionKind: runtime.VersionKind{ APIVersion: "v1", Kind: "Pod", }, CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ Operation: quota.OpAdd, Path: ".spec.containers[*].resources.requests.cpu", }, }, }, }, } EventuallyCreation(func() error { return k8sClient.Create(ctx, gq) }).Should(Succeed()) EventuallyCreation(func() error { return k8sClient.Create(ctx, cq) }).Should(Succeed()) awaitGlobalQuotaReady(ctx, gq.GetName()) awaitCustomQuotaReady(ctx, testNamespace, cq.GetName()) pod1 := MakePod(testNamespace, "mixed-cpu-1", nil, nil, "nginx:1.27.0", "100m", "") pod2 := MakePod(testNamespace, "mixed-cpu-2", nil, nil, "nginx:1.27.0", "100m", "") EventuallyCreation(func() error { pod1.ResourceVersion = "" return k8sClient.Create(ctx, pod1) }).Should(Succeed()) EventuallyCreation(func() error { pod2.ResourceVersion = "" return k8sClient.Create(ctx, pod2) }).Should(Succeed()) expectGlobalQuotaUsedAndClaims(ctx, gq.GetName(), "200m", 2) expectCustomQuotaUsedAndClaims(ctx, cq.GetNamespace(), cq.GetName(), "200m", 2) expectLedgerSettled(ctx, ControllerNamespace, gq.GetName()) expectLedgerSettled(ctx, cq.GetNamespace(), cq.GetName()) Eventually(func() error { pod3 := MakePod(testNamespace, "mixed-cpu-3", nil, nil, "nginx:1.27.0", "100m", "") return k8sClient.Create(ctx, pod3) }, defaultTimeoutInterval, defaultPollInterval).Should( MatchError(ContainSubstring(`CustomQuota "cq-mixed-pod-cpu"`)), ) }) It("accounts only the matching subset for overlapping global and namespaced quota selectors", func() { gq := &capsulev1beta2.GlobalCustomQuota{ ObjectMeta: metav1.ObjectMeta{ Name: "gq-mixed-overlap", Labels: map[string]string{ "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", }, }, Spec: capsulev1beta2.GlobalCustomQuotaSpec{ CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ Limit: resource.MustParse("10"), Sources: []capsulev1beta2.CustomQuotaSpecSource{ { VersionKind: runtime.VersionKind{ APIVersion: "v1", Kind: "Pod", }, CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ Operation: quota.OpCount, Selectors: []selectors.SelectorWithFields{ { LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ "track": "yes", }, }, }, }, }, }, }, }, NamespaceSelectors: []selectors.NamespaceSelector{ { LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ tenantLabel: tenantValue, }, }, }, }, }, } cq := &capsulev1beta2.CustomQuota{ ObjectMeta: metav1.ObjectMeta{ Name: "cq-mixed-overlap", Namespace: testNamespace, Labels: map[string]string{ "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", }, }, Spec: capsulev1beta2.CustomQuotaSpec{ Limit: resource.MustParse("2"), Sources: []capsulev1beta2.CustomQuotaSpecSource{ { VersionKind: runtime.VersionKind{ APIVersion: "v1", Kind: "Pod", }, CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ Operation: quota.OpCount, Selectors: []selectors.SelectorWithFields{ { LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ "track": "yes", "tier": "frontend", }, }, }, }, }, }, }, }, } EventuallyCreation(func() error { return k8sClient.Create(ctx, gq) }).Should(Succeed()) EventuallyCreation(func() error { return k8sClient.Create(ctx, cq) }).Should(Succeed()) awaitGlobalQuotaReady(ctx, gq.GetName()) awaitCustomQuotaReady(ctx, testNamespace, cq.GetName()) p1 := MakePod(testNamespace, "mixed-overlap-1", map[string]string{"track": "yes", "tier": "frontend"}, nil, "nginx:1.27.0", "", "") p2 := MakePod(testNamespace, "mixed-overlap-2", map[string]string{"track": "yes", "tier": "frontend"}, nil, "nginx:1.27.0", "", "") p3 := MakePod(testNamespace, "mixed-overlap-3", map[string]string{"track": "yes", "tier": "backend"}, nil, "nginx:1.27.0", "", "") EventuallyCreation(func() error { p1.ResourceVersion = ""; return k8sClient.Create(ctx, p1) }).Should(Succeed()) EventuallyCreation(func() error { p2.ResourceVersion = ""; return k8sClient.Create(ctx, p2) }).Should(Succeed()) EventuallyCreation(func() error { p3.ResourceVersion = ""; return k8sClient.Create(ctx, p3) }).Should(Succeed()) expectGlobalQuotaUsedAndClaims(ctx, gq.GetName(), "3", 3) expectCustomQuotaUsedAndClaims(ctx, cq.GetNamespace(), cq.GetName(), "2", 2) expectLedgerSettled(ctx, ControllerNamespace, gq.GetName()) expectLedgerSettled(ctx, cq.GetNamespace(), cq.GetName()) Eventually(func() error { p4 := MakePod(testNamespace, "mixed-overlap-4", map[string]string{"track": "yes", "tier": "frontend"}, nil, "nginx:1.27.0", "", "") return k8sClient.Create(ctx, p4) }, defaultTimeoutInterval, defaultPollInterval).Should( MatchError(ContainSubstring(`CustomQuota "cq-mixed-overlap"`)), ) }) It("tracks different paths independently when global and namespaced quotas match the same pod gvk", func() { gq := &capsulev1beta2.GlobalCustomQuota{ ObjectMeta: metav1.ObjectMeta{ Name: "gq-mixed-path-cpu", Labels: map[string]string{ "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", }, }, Spec: capsulev1beta2.GlobalCustomQuotaSpec{ CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ Limit: resource.MustParse("500m"), Sources: []capsulev1beta2.CustomQuotaSpecSource{ { VersionKind: runtime.VersionKind{ APIVersion: "v1", Kind: "Pod", }, CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ Operation: quota.OpAdd, Path: ".spec.containers[*].resources.requests.cpu", }, }, }, }, NamespaceSelectors: []selectors.NamespaceSelector{ { LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ tenantLabel: tenantValue, }, }, }, }, }, } cq := &capsulev1beta2.CustomQuota{ ObjectMeta: metav1.ObjectMeta{ Name: "cq-mixed-path-emptydir", Namespace: testNamespace, Labels: map[string]string{ "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", }, }, Spec: capsulev1beta2.CustomQuotaSpec{ Limit: resource.MustParse("2Gi"), Sources: []capsulev1beta2.CustomQuotaSpecSource{ { VersionKind: runtime.VersionKind{ APIVersion: "v1", Kind: "Pod", }, CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ Operation: quota.OpAdd, Path: ".spec.volumes[*].emptyDir.sizeLimit", }, }, }, }, } EventuallyCreation(func() error { return k8sClient.Create(ctx, gq) }).Should(Succeed()) EventuallyCreation(func() error { return k8sClient.Create(ctx, cq) }).Should(Succeed()) awaitGlobalQuotaReady(ctx, gq.GetName()) awaitCustomQuotaReady(ctx, testNamespace, cq.GetName()) p1 := MakePod(testNamespace, "mixed-path-1", nil, nil, "nginx:1.27.0", "100m", "1Gi") p2 := MakePod(testNamespace, "mixed-path-2", nil, nil, "nginx:1.27.0", "100m", "1Gi") EventuallyCreation(func() error { p1.ResourceVersion = ""; return k8sClient.Create(ctx, p1) }).Should(Succeed()) EventuallyCreation(func() error { p2.ResourceVersion = ""; return k8sClient.Create(ctx, p2) }).Should(Succeed()) expectGlobalQuotaUsedAndClaims(ctx, gq.GetName(), "200m", 2) expectCustomQuotaUsedAndClaims(ctx, cq.GetNamespace(), cq.GetName(), "2Gi", 2) expectLedgerSettled(ctx, ControllerNamespace, gq.GetName()) expectLedgerSettled(ctx, cq.GetNamespace(), cq.GetName()) Eventually(func() error { p3 := MakePod(testNamespace, "mixed-path-3", nil, nil, "nginx:1.27.0", "100m", "1Gi") return k8sClient.Create(ctx, p3) }, defaultTimeoutInterval, defaultPollInterval).Should( MatchError(ContainSubstring(`CustomQuota "cq-mixed-path-emptydir"`)), ) }) It("accounts deployment scaling in both global and namespaced quotas and denies when the smaller namespaced quota is exceeded", func() { gq := &capsulev1beta2.GlobalCustomQuota{ ObjectMeta: metav1.ObjectMeta{ Name: "gq-mixed-scale", Labels: map[string]string{ "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", }, }, Spec: capsulev1beta2.GlobalCustomQuotaSpec{ CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ Limit: resource.MustParse("10"), Sources: []capsulev1beta2.CustomQuotaSpecSource{ { VersionKind: runtime.VersionKind{ APIVersion: "v1", Kind: "Pod", }, CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ Operation: quota.OpCount, }, }, }, }, NamespaceSelectors: []selectors.NamespaceSelector{ { LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ tenantLabel: tenantValue, }, }, }, }, }, } cq := &capsulev1beta2.CustomQuota{ ObjectMeta: metav1.ObjectMeta{ Name: "cq-mixed-scale", Namespace: testNamespace, Labels: map[string]string{ "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", }, }, Spec: capsulev1beta2.CustomQuotaSpec{ Limit: resource.MustParse("3"), Sources: []capsulev1beta2.CustomQuotaSpecSource{ { VersionKind: runtime.VersionKind{ APIVersion: "v1", Kind: "Pod", }, CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ Operation: quota.OpCount, }, }, }, }, } EventuallyCreation(func() error { return k8sClient.Create(ctx, gq) }).Should(Succeed()) EventuallyCreation(func() error { return k8sClient.Create(ctx, cq) }).Should(Succeed()) awaitGlobalQuotaReady(ctx, gq.GetName()) awaitCustomQuotaReady(ctx, testNamespace, cq.GetName()) dep := MakeDeployment(testNamespace, "mixed-scale", 1, nil, "") EventuallyCreation(func() error { dep.ResourceVersion = "" return k8sClient.Create(ctx, dep) }).Should(Succeed()) ExpectPodsForDeployment(ctx, testNamespace, "mixed-scale", 1) expectGlobalQuotaUsedAndClaims(ctx, gq.GetName(), "1", 1) expectCustomQuotaUsedAndClaims(ctx, cq.GetNamespace(), cq.GetName(), "1", 1) ScaleDeployment(ctx, testNamespace, "mixed-scale", 3) ExpectPodsForDeployment(ctx, testNamespace, "mixed-scale", 3) expectGlobalQuotaUsedAndClaims(ctx, gq.GetName(), "3", 3) expectCustomQuotaUsedAndClaims(ctx, cq.GetNamespace(), cq.GetName(), "3", 3) ScaleDeployment(ctx, testNamespace, "mixed-scale", 4) ExpectPodsForDeployment(ctx, testNamespace, "mixed-scale", 3) Eventually(func(g Gomega) { obj := &capsulev1beta2.CustomQuota{} g.Expect(k8sClient.Get(ctx, types.NamespacedName{Name: cq.GetName(), Namespace: testNamespace}, obj)).To(Succeed()) g.Expect(obj.Status.Usage.Used.Cmp(resource.MustParse("3"))).To(Equal(0)) }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) }) It("clamps usage to zero for a pure subtraction source", func() { q := &capsulev1beta2.GlobalCustomQuota{ ObjectMeta: metav1.ObjectMeta{ Name: "gq-sub-only-clamps-zero", Labels: map[string]string{ "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", }, }, Spec: capsulev1beta2.GlobalCustomQuotaSpec{ CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ Limit: resource.MustParse("10Gi"), Sources: []capsulev1beta2.CustomQuotaSpecSource{ { VersionKind: runtime.VersionKind{ APIVersion: "v1", Kind: "PersistentVolumeClaim", }, CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ Operation: quota.OpSub, Path: ".spec.resources.requests.storage", }, }, }, }, NamespaceSelectors: []selectors.NamespaceSelector{ { LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ tenantLabel: tenantValue, }, }, }, }, }, } EventuallyCreation(func() error { return k8sClient.Create(ctx, q) }).Should(Succeed()) awaitGlobalQuotaReady(ctx, q.GetName()) pvc := MakePVC(testNamespace, "sub-only-pvc", "2Gi") EventuallyCreation(func() error { pvc.ResourceVersion = "" return k8sClient.Create(ctx, pvc) }).Should(Succeed()) expectGlobalQuotaUsedAndClaims(ctx, q.GetName(), "0", 1) expectLedgerSettled(ctx, ControllerNamespace, q.GetName()) }) It("subtracts matching pvc storage from added pod emptyDir storage", func() { q := &capsulev1beta2.GlobalCustomQuota{ ObjectMeta: metav1.ObjectMeta{ Name: "gq-add-sub-storage", Labels: map[string]string{ "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", }, }, Spec: capsulev1beta2.GlobalCustomQuotaSpec{ CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ Limit: resource.MustParse("10Gi"), Sources: []capsulev1beta2.CustomQuotaSpecSource{ { VersionKind: runtime.VersionKind{ APIVersion: "v1", Kind: "Pod", }, CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ Operation: quota.OpAdd, Path: ".spec.volumes[*].emptyDir.sizeLimit", }, }, { VersionKind: runtime.VersionKind{ APIVersion: "v1", Kind: "PersistentVolumeClaim", }, CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ Operation: quota.OpSub, Path: ".spec.resources.requests.storage", }, }, }, }, NamespaceSelectors: []selectors.NamespaceSelector{ { LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ tenantLabel: tenantValue, }, }, }, }, }, } EventuallyCreation(func() error { return k8sClient.Create(ctx, q) }).Should(Succeed()) awaitGlobalQuotaReady(ctx, q.GetName()) pod := MakePod(testNamespace, "add-sub-pod", nil, nil, "nginx:1.27.0", "", "3Gi") pvc := MakePVC(testNamespace, "add-sub-pvc", "1Gi") EventuallyCreation(func() error { pod.ResourceVersion = "" return k8sClient.Create(ctx, pod) }).Should(Succeed()) EventuallyCreation(func() error { pvc.ResourceVersion = "" return k8sClient.Create(ctx, pvc) }).Should(Succeed()) expectGlobalQuotaUsedAndClaims(ctx, q.GetName(), "2Gi", 2) expectLedgerSettled(ctx, ControllerNamespace, q.GetName()) }) It("clamps mixed add and subtraction result to zero when subtraction exceeds additions", func() { q := &capsulev1beta2.GlobalCustomQuota{ ObjectMeta: metav1.ObjectMeta{ Name: "gq-add-sub-clamp-zero", Labels: map[string]string{ "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", }, }, Spec: capsulev1beta2.GlobalCustomQuotaSpec{ CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ Limit: resource.MustParse("10Gi"), Sources: []capsulev1beta2.CustomQuotaSpecSource{ { VersionKind: runtime.VersionKind{ APIVersion: "v1", Kind: "Pod", }, CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ Operation: quota.OpAdd, Path: ".spec.volumes[*].emptyDir.sizeLimit", }, }, { VersionKind: runtime.VersionKind{ APIVersion: "v1", Kind: "PersistentVolumeClaim", }, CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ Operation: quota.OpSub, Path: ".spec.resources.requests.storage", }, }, }, }, NamespaceSelectors: []selectors.NamespaceSelector{ { LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ tenantLabel: tenantValue, }, }, }, }, }, } EventuallyCreation(func() error { return k8sClient.Create(ctx, q) }).Should(Succeed()) awaitGlobalQuotaReady(ctx, q.GetName()) pod := MakePod(testNamespace, "add-sub-clamp-pod", nil, nil, "nginx:1.27.0", "", "1Gi") pvc := MakePVC(testNamespace, "add-sub-clamp-pvc", "2Gi") EventuallyCreation(func() error { pod.ResourceVersion = "" return k8sClient.Create(ctx, pod) }).Should(Succeed()) EventuallyCreation(func() error { pvc.ResourceVersion = "" return k8sClient.Create(ctx, pvc) }).Should(Succeed()) expectGlobalQuotaUsedAndClaims(ctx, q.GetName(), "0", 2) expectLedgerSettled(ctx, ControllerNamespace, q.GetName()) }) It("supports subtraction with label selectors and removes the subtraction when the object no longer matches", func() { q := &capsulev1beta2.GlobalCustomQuota{ ObjectMeta: metav1.ObjectMeta{ Name: "gq-sub-label-selector", Labels: map[string]string{ "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", }, }, Spec: capsulev1beta2.GlobalCustomQuotaSpec{ CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ Limit: resource.MustParse("10Gi"), Sources: []capsulev1beta2.CustomQuotaSpecSource{ { VersionKind: runtime.VersionKind{ APIVersion: "v1", Kind: "Pod", }, CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ Operation: quota.OpAdd, Path: ".spec.volumes[*].emptyDir.sizeLimit", }, }, { VersionKind: runtime.VersionKind{ APIVersion: "v1", Kind: "PersistentVolumeClaim", }, CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ Operation: quota.OpSub, Path: ".spec.resources.requests.storage", Selectors: []selectors.SelectorWithFields{ { LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ "discount": "yes", }, }, }, }, }, }, }, }, NamespaceSelectors: []selectors.NamespaceSelector{ { LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ tenantLabel: tenantValue, }, }, }, }, }, } EventuallyCreation(func() error { return k8sClient.Create(ctx, q) }).Should(Succeed()) awaitGlobalQuotaReady(ctx, q.GetName()) pod := MakePod(testNamespace, "sub-label-pod", nil, nil, "nginx:1.27.0", "", "3Gi") pvc := MakePVC(testNamespace, "sub-label-pvc", "1Gi") pvc.Labels = map[string]string{"discount": "yes"} EventuallyCreation(func() error { pod.ResourceVersion = "" return k8sClient.Create(ctx, pod) }).Should(Succeed()) EventuallyCreation(func() error { pvc.ResourceVersion = "" return k8sClient.Create(ctx, pvc) }).Should(Succeed()) expectGlobalQuotaUsedAndClaims(ctx, q.GetName(), "2Gi", 2) expectLedgerSettled(ctx, ControllerNamespace, q.GetName()) Eventually(func() error { obj := &corev1.PersistentVolumeClaim{} if err := k8sClient.Get(ctx, types.NamespacedName{Name: pvc.Name, Namespace: pvc.Namespace}, obj); err != nil { return err } obj.Labels = map[string]string{"discount": "no"} return k8sClient.Update(ctx, obj) }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) expectGlobalQuotaUsedAndClaims(ctx, q.GetName(), "3Gi", 1) expectLedgerSettled(ctx, ControllerNamespace, q.GetName()) }) It("reconciles subtraction correctly when the subtracting resource is deleted", func() { q := &capsulev1beta2.GlobalCustomQuota{ ObjectMeta: metav1.ObjectMeta{ Name: "gq-sub-delete-reconcile", Labels: map[string]string{ "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", }, }, Spec: capsulev1beta2.GlobalCustomQuotaSpec{ CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ Limit: resource.MustParse("10Gi"), Sources: []capsulev1beta2.CustomQuotaSpecSource{ { VersionKind: runtime.VersionKind{ APIVersion: "v1", Kind: "Pod", }, CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ Operation: quota.OpAdd, Path: ".spec.volumes[*].emptyDir.sizeLimit", }, }, { VersionKind: runtime.VersionKind{ APIVersion: "v1", Kind: "PersistentVolumeClaim", }, CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ Operation: quota.OpSub, Path: ".spec.resources.requests.storage", }, }, }, }, NamespaceSelectors: []selectors.NamespaceSelector{ { LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ tenantLabel: tenantValue, }, }, }, }, }, } EventuallyCreation(func() error { return k8sClient.Create(ctx, q) }).Should(Succeed()) awaitGlobalQuotaReady(ctx, q.GetName()) pod := MakePod(testNamespace, "sub-delete-pod", nil, nil, "nginx:1.27.0", "", "3Gi") pvc := MakePVC(testNamespace, "sub-delete-pvc", "1Gi") EventuallyCreation(func() error { pod.ResourceVersion = "" return k8sClient.Create(ctx, pod) }).Should(Succeed()) EventuallyCreation(func() error { pvc.ResourceVersion = "" return k8sClient.Create(ctx, pvc) }).Should(Succeed()) expectGlobalQuotaUsedAndClaims(ctx, q.GetName(), "2Gi", 2) expectLedgerSettled(ctx, ControllerNamespace, q.GetName()) EventuallyDeletion(pvc) expectGlobalQuotaUsedAndClaims(ctx, q.GetName(), "3Gi", 1) expectLedgerSettled(ctx, ControllerNamespace, q.GetName()) }) It("subtracts cpu requests from counted pod usage across multiple matching pods and clamps at zero", func() { q := &capsulev1beta2.GlobalCustomQuota{ ObjectMeta: metav1.ObjectMeta{ Name: "gq-sub-cpu-clamp", Labels: map[string]string{ "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", }, }, Spec: capsulev1beta2.GlobalCustomQuotaSpec{ CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ Limit: resource.MustParse("10"), Sources: []capsulev1beta2.CustomQuotaSpecSource{ { VersionKind: runtime.VersionKind{ APIVersion: "v1", Kind: "Pod", }, CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ Operation: quota.OpCount, }, }, { VersionKind: runtime.VersionKind{ APIVersion: "v1", Kind: "Pod", }, CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ Operation: quota.OpSub, Path: ".spec.containers[*].resources.requests.cpu", }, }, }, }, NamespaceSelectors: []selectors.NamespaceSelector{ { LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ tenantLabel: tenantValue, }, }, }, }, }, } EventuallyCreation(func() error { return k8sClient.Create(ctx, q) }).Should(Succeed()) awaitGlobalQuotaReady(ctx, q.GetName()) pod1 := MakePod(testNamespace, "sub-cpu-pod-1", nil, nil, "nginx:1.27.0", "500m", "") pod2 := MakePod(testNamespace, "sub-cpu-pod-2", nil, nil, "nginx:1.27.0", "500m", "") EventuallyCreation(func() error { pod1.ResourceVersion = "" return k8sClient.Create(ctx, pod1) }).Should(Succeed()) EventuallyCreation(func() error { pod2.ResourceVersion = "" return k8sClient.Create(ctx, pod2) }).Should(Succeed()) // 2 - 1.0 = 1 expectGlobalQuotaUsedAndClaims(ctx, q.GetName(), "1", 2) expectLedgerSettled(ctx, ControllerNamespace, q.GetName()) }) It("applies subtraction while scaling a deployment", func() { q := &capsulev1beta2.GlobalCustomQuota{ ObjectMeta: metav1.ObjectMeta{ Name: "gq-sub-deployment-scale", Labels: map[string]string{ "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", }, }, Spec: capsulev1beta2.GlobalCustomQuotaSpec{ CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ Limit: resource.MustParse("10"), Sources: []capsulev1beta2.CustomQuotaSpecSource{ { VersionKind: runtime.VersionKind{ APIVersion: "v1", Kind: "Pod", }, CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ Operation: quota.OpCount, }, }, { VersionKind: runtime.VersionKind{ APIVersion: "v1", Kind: "Pod", }, CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ Operation: quota.OpSub, Path: ".spec.containers[*].resources.requests.cpu", }, }, }, }, NamespaceSelectors: []selectors.NamespaceSelector{ { LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ tenantLabel: tenantValue, }, }, }, }, }, } EventuallyCreation(func() error { return k8sClient.Create(ctx, q) }).Should(Succeed()) awaitGlobalQuotaReady(ctx, q.GetName()) dep := MakeDeployment(testNamespace, "sub-scale", 2, nil, "250m") EventuallyCreation(func() error { dep.ResourceVersion = "" return k8sClient.Create(ctx, dep) }).Should(Succeed()) ExpectPodsForDeployment(ctx, testNamespace, "sub-scale", 2) // 2 - 0.5 = 1.5 expectGlobalQuotaUsedAndClaims(ctx, q.GetName(), "1500m", 2) expectLedgerSettled(ctx, ControllerNamespace, q.GetName()) ScaleDeployment(ctx, testNamespace, "sub-scale", 4) ExpectPodsForDeployment(ctx, testNamespace, "sub-scale", 4) // 4 - 1.0 = 3 expectGlobalQuotaUsedAndClaims(ctx, q.GetName(), "3", 4) expectLedgerSettled(ctx, ControllerNamespace, q.GetName()) }) It("uses the smallest matching quota as authoritative even when both quotas use subtraction", func() { small := &capsulev1beta2.GlobalCustomQuota{ ObjectMeta: metav1.ObjectMeta{ Name: "gq-sub-small", Labels: map[string]string{ "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", }, }, Spec: capsulev1beta2.GlobalCustomQuotaSpec{ CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ Limit: resource.MustParse("2"), Sources: []capsulev1beta2.CustomQuotaSpecSource{ { VersionKind: runtime.VersionKind{ APIVersion: "v1", Kind: "Pod", }, CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ Operation: quota.OpCount, }, }, { VersionKind: runtime.VersionKind{ APIVersion: "v1", Kind: "Pod", }, CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ Operation: quota.OpSub, Path: ".spec.containers[*].resources.requests.cpu", }, }, }, }, NamespaceSelectors: []selectors.NamespaceSelector{ { LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ tenantLabel: tenantValue, }, }, }, }, }, } large := &capsulev1beta2.GlobalCustomQuota{ ObjectMeta: metav1.ObjectMeta{ Name: "gq-sub-large", Labels: map[string]string{ "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", }, }, Spec: capsulev1beta2.GlobalCustomQuotaSpec{ CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ Limit: resource.MustParse("5"), Sources: []capsulev1beta2.CustomQuotaSpecSource{ { VersionKind: runtime.VersionKind{ APIVersion: "v1", Kind: "Pod", }, CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ Operation: quota.OpCount, }, }, { VersionKind: runtime.VersionKind{ APIVersion: "v1", Kind: "Pod", }, CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ Operation: quota.OpSub, Path: ".spec.containers[*].resources.requests.cpu", }, }, }, }, NamespaceSelectors: []selectors.NamespaceSelector{ { LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ tenantLabel: tenantValue, }, }, }, }, }, } EventuallyCreation(func() error { return k8sClient.Create(ctx, small) }).Should(Succeed()) EventuallyCreation(func() error { return k8sClient.Create(ctx, large) }).Should(Succeed()) awaitGlobalQuotaReady(ctx, small.GetName()) awaitGlobalQuotaReady(ctx, large.GetName()) pod1 := MakePod(testNamespace, "sub-auth-1", nil, nil, "nginx:1.27.0", "500m", "") pod2 := MakePod(testNamespace, "sub-auth-2", nil, nil, "nginx:1.27.0", "500m", "") pod3 := MakePod(testNamespace, "sub-auth-3", nil, nil, "nginx:1.27.0", "500m", "") EventuallyCreation(func() error { pod1.ResourceVersion = "" return k8sClient.Create(ctx, pod1) }).Should(Succeed()) EventuallyCreation(func() error { pod2.ResourceVersion = "" return k8sClient.Create(ctx, pod2) }).Should(Succeed()) // 2 - 1.0 = 1 expectGlobalQuotaUsedAndClaims(ctx, small.GetName(), "1", 2) expectGlobalQuotaUsedAndClaims(ctx, large.GetName(), "1", 2) expectLedgerSettled(ctx, ControllerNamespace, small.GetName()) expectLedgerSettled(ctx, ControllerNamespace, large.GetName()) Eventually(func() error { pod3.ResourceVersion = "" return k8sClient.Create(ctx, pod3) }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) // 3 - 1.5 = 1.5 still fits into both limits, so one more to push smaller first expectGlobalQuotaUsedAndClaims(ctx, small.GetName(), "1500m", 3) expectGlobalQuotaUsedAndClaims(ctx, large.GetName(), "1500m", 3) Eventually(func() error { p := MakePod(testNamespace, "sub-auth-4", nil, nil, "nginx:1.27.0", "500m", "") return k8sClient.Create(ctx, p) }, defaultTimeoutInterval, defaultPollInterval).Should(MatchError(ContainSubstring(`GlobalCustomQuota "gq-sub-small"`))) }) It("posts wildcard namespace status when no namespaceSelectors are configured and keeps it stable as namespaces change", func() { quota := &capsulev1beta2.GlobalCustomQuota{ ObjectMeta: metav1.ObjectMeta{ Name: "gq-nsstatus-all", Labels: map[string]string{ "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", }, }, Spec: capsulev1beta2.GlobalCustomQuotaSpec{ CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ Limit: resource.MustParse("1000"), Sources: []capsulev1beta2.CustomQuotaSpecSource{ { VersionKind: runtime.VersionKind{ APIVersion: "v1", Kind: "Pod", }, CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ Operation: quota.OpCount, }, }, }, }, }, } extraA := NewNamespace("gq-nsstatus-all-a", map[string]string{"purpose": "e2e-global-custom-quota"}) extraB := NewNamespace("gq-nsstatus-all-b", map[string]string{"purpose": "e2e-global-custom-quota"}) EventuallyCreation(func() error { return k8sClient.Create(ctx, quota) }).Should(Succeed()) awaitGlobalQuotaReady(ctx, quota.GetName()) expectGlobalQuotaWildcardNamespaces(quota.GetName()) NamespaceDeletionAdmin(extraA, defaultTimeoutInterval) expectGlobalQuotaWildcardNamespaces(quota.GetName()) NamespaceDeletionAdmin(extraB, defaultTimeoutInterval) expectGlobalQuotaWildcardNamespaces(quota.GetName()) NamespaceDeletionAdmin(extraA, defaultTimeoutInterval) expectGlobalQuotaWildcardNamespaces(quota.GetName()) NamespaceDeletionAdmin(extraB, defaultTimeoutInterval) expectGlobalQuotaWildcardNamespaces(quota.GetName()) }) It("posts all namespaces matched by multiple namespaceSelectors and updates status on namespace create and delete", func() { nsA1 := NewNamespace("gq-nsstatus-a1", map[string]string{"team": "a"}) nsA2 := NewNamespace("gq-nsstatus-a2", map[string]string{"team": "a"}) nsB1 := NewNamespace("gq-nsstatus-b1", map[string]string{"team": "b"}) nsOther := NewNamespace("gq-nsstatus-other", map[string]string{"team": "other"}) NamespaceCreationAdmin(nsA1, defaultTimeoutInterval).Should(Succeed()) NamespaceCreationAdmin(nsB1, defaultTimeoutInterval).Should(Succeed()) NamespaceCreationAdmin(nsOther, defaultTimeoutInterval).Should(Succeed()) quota := &capsulev1beta2.GlobalCustomQuota{ ObjectMeta: metav1.ObjectMeta{ Name: "gq-nsstatus-multi-selectors", Labels: map[string]string{ "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", }, }, Spec: capsulev1beta2.GlobalCustomQuotaSpec{ CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ Limit: resource.MustParse("10"), Sources: []capsulev1beta2.CustomQuotaSpecSource{ { VersionKind: runtime.VersionKind{ APIVersion: "v1", Kind: "Pod", }, CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ Operation: quota.OpCount, }, }, }, }, NamespaceSelectors: []selectors.NamespaceSelector{ { LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ "team": "a", }, }, }, { LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ "team": "b", }, }, }, }, }, } EventuallyCreation(func() error { return k8sClient.Create(ctx, quota) }).Should(Succeed()) awaitGlobalQuotaReady(ctx, quota.GetName()) expectGlobalQuotaNamespaces(quota.GetName(), "gq-nsstatus-a1", "gq-nsstatus-b1", ) NamespaceCreationAdmin(nsA2, defaultTimeoutInterval).Should(Succeed()) expectGlobalQuotaNamespaces(quota.GetName(), "gq-nsstatus-a1", "gq-nsstatus-a2", "gq-nsstatus-b1", ) NamespaceDeletionAdmin(nsB1, defaultTimeoutInterval).Should(Succeed()) expectGlobalQuotaNamespaces(quota.GetName(), "gq-nsstatus-a1", "gq-nsstatus-a2", ) }) It("posts an empty namespace status when namespaceSelectors match no namespaces and updates when matches appear or disappear", func() { quota := &capsulev1beta2.GlobalCustomQuota{ ObjectMeta: metav1.ObjectMeta{ Name: "gq-nsstatus-empty", Labels: map[string]string{ "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", }, }, Spec: capsulev1beta2.GlobalCustomQuotaSpec{ CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ Limit: resource.MustParse("10"), Sources: []capsulev1beta2.CustomQuotaSpecSource{ { VersionKind: runtime.VersionKind{ APIVersion: "v1", Kind: "Pod", }, CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ Operation: quota.OpCount, }, }, }, }, NamespaceSelectors: []selectors.NamespaceSelector{ { LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ "team": "does-not-exist", }, }, }, }, }, } EventuallyCreation(func() error { return k8sClient.Create(ctx, quota) }).Should(Succeed()) awaitGlobalQuotaReady(ctx, quota.GetName()) expectGlobalQuotaNamespaces(quota.GetName()) ns := NewNamespace("gq-nsstatus-empty-match", map[string]string{"team": "does-not-exist"}) NamespaceCreationAdmin(ns, defaultTimeoutInterval).Should(Succeed()) expectGlobalQuotaNamespaces(quota.GetName(), "gq-nsstatus-empty-match") NamespaceDeletionAdmin(ns, defaultTimeoutInterval).Should(Succeed()) expectGlobalQuotaNamespaces(quota.GetName()) }) It("does not produce negative usage when a matching pod is relabeled to no longer match", func() { q := &capsulev1beta2.GlobalCustomQuota{ ObjectMeta: metav1.ObjectMeta{ Name: "gq-no-negative-on-relabel", Labels: map[string]string{ "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", }, }, Spec: capsulev1beta2.GlobalCustomQuotaSpec{ CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ Limit: resource.MustParse("10"), Sources: []capsulev1beta2.CustomQuotaSpecSource{ { VersionKind: runtime.VersionKind{ APIVersion: "v1", Kind: "Pod", }, CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ Operation: quota.OpCount, Selectors: []selectors.SelectorWithFields{ { LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ "track": "yes", }, }, }, }, }, }, }, }, NamespaceSelectors: []selectors.NamespaceSelector{ { LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ tenantLabel: tenantValue, }, }, }, }, }, } EventuallyCreation(func() error { return k8sClient.Create(ctx, q) }).Should(Succeed()) awaitGlobalQuotaReady(ctx, q.GetName()) pod := MakePod(testNamespace, "no-negative-on-relabel", map[string]string{"track": "yes"}, nil, "nginx:1.27.0", "", "") EventuallyCreation(func() error { pod.ResourceVersion = "" return k8sClient.Create(ctx, pod) }).Should(Succeed()) expectGlobalQuotaUsedAndClaims(ctx, q.GetName(), "1", 1) UpdatePodLabels(ctx, testNamespace, "no-negative-on-relabel", map[string]string{"track": "no"}) expectLedgerSettled(ctx, ControllerNamespace, q.GetName()) Eventually(func(g Gomega) { obj := &capsulev1beta2.GlobalCustomQuota{} g.Expect(k8sClient.Get(ctx, types.NamespacedName{Name: q.GetName()}, obj)).To(Succeed()) g.Expect(obj.Status.Usage.Used.Sign()).To(BeNumerically(">=", 0)) g.Expect(obj.Status.Usage.Used.Cmp(resource.MustParse("0"))).To(Equal(0)) g.Expect(len(obj.Status.Claims)).To(Equal(0)) }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) }) It("counts pods correctly while scaling a deployment", func() { q := &capsulev1beta2.GlobalCustomQuota{ ObjectMeta: metav1.ObjectMeta{ Name: "gq-pod-count", Labels: map[string]string{ "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", }, }, Spec: capsulev1beta2.GlobalCustomQuotaSpec{ CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ Limit: resource.MustParse("5"), Sources: []capsulev1beta2.CustomQuotaSpecSource{ { VersionKind: runtime.VersionKind{ APIVersion: "v1", Kind: "Pod", }, CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ Operation: quota.OpCount, }, }, }, }, NamespaceSelectors: []selectors.NamespaceSelector{ { LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ tenantLabel: tenantValue, }, }, }, }, }, } EventuallyCreation(func() error { return k8sClient.Create(ctx, q) }).Should(Succeed()) awaitGlobalQuotaReady(ctx, q.GetName()) dep := MakeDeployment(testNamespace, "counted", 1, nil, "") EventuallyCreation(func() error { dep.ResourceVersion = "" return k8sClient.Create(ctx, dep) }).Should(Succeed()) ExpectPodsForDeployment(ctx, testNamespace, "counted", 1) expectGlobalQuotaUsedAndClaims(ctx, q.GetName(), "1", 1) ScaleDeployment(ctx, testNamespace, "counted", 3) ExpectPodsForDeployment(ctx, testNamespace, "counted", 3) expectGlobalQuotaUsedAndClaims(ctx, q.GetName(), "3", 3) ScaleDeployment(ctx, testNamespace, "counted", 2) ExpectPodsForDeployment(ctx, testNamespace, "counted", 2) expectGlobalQuotaUsedAndClaims(ctx, q.GetName(), "2", 2) expectLedgerSettled(ctx, ControllerNamespace, q.GetName()) }) It("tracks count with a single MatchLabels selector and updates when the pod no longer matches", func() { q := &capsulev1beta2.GlobalCustomQuota{ ObjectMeta: metav1.ObjectMeta{ Name: "gq-count-single-matchlabel", Labels: map[string]string{ "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", }, }, Spec: capsulev1beta2.GlobalCustomQuotaSpec{ CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ Limit: resource.MustParse("10"), Sources: []capsulev1beta2.CustomQuotaSpecSource{ { VersionKind: runtime.VersionKind{ APIVersion: "v1", Kind: "Pod", }, CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ Operation: quota.OpCount, Selectors: []selectors.SelectorWithFields{ { LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ "track": "yes", }, }, }, }, }, }, }, }, NamespaceSelectors: []selectors.NamespaceSelector{ { LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ tenantLabel: tenantValue, }, }, }, }, }, } EventuallyCreation(func() error { return k8sClient.Create(ctx, q) }).Should(Succeed()) pod := MakePod(testNamespace, "single-matchlabel", map[string]string{"track": "yes"}, nil, "nginx:1.27.0", "", "") EventuallyCreation(func() error { pod.ResourceVersion = "" return k8sClient.Create(ctx, pod) }).Should(Succeed()) expectGlobalQuotaUsedAndClaims(ctx, q.GetName(), "1", 1) UpdatePodLabels(ctx, testNamespace, "single-matchlabel", map[string]string{"track": "no"}) expectGlobalQuotaUsedAndClaims(ctx, q.GetName(), "0", 0) expectLedgerSettled(ctx, ControllerNamespace, q.GetName()) }) It("tracks count with multiple MatchLabels and updates when the pod no longer matches", func() { q := &capsulev1beta2.GlobalCustomQuota{ ObjectMeta: metav1.ObjectMeta{ Name: "gq-count-multi-matchlabel", Labels: map[string]string{ "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", }, }, Spec: capsulev1beta2.GlobalCustomQuotaSpec{ CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ Limit: resource.MustParse("10"), Sources: []capsulev1beta2.CustomQuotaSpecSource{ { VersionKind: runtime.VersionKind{ APIVersion: "v1", Kind: "Pod", }, CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ Operation: quota.OpCount, Selectors: []selectors.SelectorWithFields{ { LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ "track": "yes", "tier": "frontend", }, }, }, }, }, }, }, }, NamespaceSelectors: []selectors.NamespaceSelector{ { LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ tenantLabel: tenantValue, }, }, }, }, }, } EventuallyCreation(func() error { return k8sClient.Create(ctx, q) }).Should(Succeed()) pod := MakePod(testNamespace, "multi-matchlabel", map[string]string{ "track": "yes", "tier": "frontend", }, nil, "nginx:1.27.0", "", "") EventuallyCreation(func() error { pod.ResourceVersion = "" return k8sClient.Create(ctx, pod) }).Should(Succeed()) expectGlobalQuotaUsedAndClaims(ctx, q.GetName(), "1", 1) UpdatePodLabels(ctx, testNamespace, "multi-matchlabel", map[string]string{ "track": "yes", "tier": "backend", }) expectGlobalQuotaUsedAndClaims(ctx, q.GetName(), "0", 0) expectLedgerSettled(ctx, ControllerNamespace, q.GetName()) }) It("tracks count with a single field selector and updates when the pod no longer matches", func() { q := &capsulev1beta2.GlobalCustomQuota{ ObjectMeta: metav1.ObjectMeta{ Name: "gq-count-single-fieldselector", Labels: map[string]string{ "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", }, }, Spec: capsulev1beta2.GlobalCustomQuotaSpec{ CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ Limit: resource.MustParse("10"), Sources: []capsulev1beta2.CustomQuotaSpecSource{ { VersionKind: runtime.VersionKind{ APIVersion: "v1", Kind: "Pod", }, CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ Operation: quota.OpCount, Selectors: []selectors.SelectorWithFields{ { FieldSelectors: []string{ `.spec.containers[?(@.image=="nginx:1.27.0")]`, }, }, }, }, }, }, }, NamespaceSelectors: []selectors.NamespaceSelector{ { LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ tenantLabel: tenantValue, }, }, }, }, }, } EventuallyCreation(func() error { return k8sClient.Create(ctx, q) }).Should(Succeed()) pod := MakePod(testNamespace, "single-fieldselector", nil, nil, "nginx:1.27.0", "", "") EventuallyCreation(func() error { pod.ResourceVersion = "" return k8sClient.Create(ctx, pod) }).Should(Succeed()) expectGlobalQuotaUsedAndClaims(ctx, q.GetName(), "1", 1) UpdatePodImage(ctx, testNamespace, "single-fieldselector", "nginx:1.26.0") expectGlobalQuotaUsedAndClaims(ctx, q.GetName(), "0", 0) expectLedgerSettled(ctx, ControllerNamespace, q.GetName()) }) It("tracks count with multiple field selectors and updates when the pod no longer matches", func() { q := &capsulev1beta2.GlobalCustomQuota{ ObjectMeta: metav1.ObjectMeta{ Name: "gq-count-multi-fieldselector", Labels: map[string]string{ "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", }, }, Spec: capsulev1beta2.GlobalCustomQuotaSpec{ CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ Limit: resource.MustParse("10"), Sources: []capsulev1beta2.CustomQuotaSpecSource{ { VersionKind: runtime.VersionKind{ APIVersion: "v1", Kind: "Pod", }, CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ Operation: quota.OpCount, Selectors: []selectors.SelectorWithFields{ { FieldSelectors: []string{ `.spec.containers[?(@.image=="nginx:1.27.0")]`, `.spec.containers[?(@.name=="main")]`, }, }, }, }, }, }, }, NamespaceSelectors: []selectors.NamespaceSelector{ { LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ tenantLabel: tenantValue, }, }, }, }, }, } EventuallyCreation(func() error { return k8sClient.Create(ctx, q) }).Should(Succeed()) pod := MakePod(testNamespace, "multi-fieldselector", nil, nil, "nginx:1.27.0", "", "") EventuallyCreation(func() error { pod.ResourceVersion = "" return k8sClient.Create(ctx, pod) }).Should(Succeed()) expectGlobalQuotaUsedAndClaims(ctx, q.GetName(), "1", 1) UpdatePodImage(ctx, testNamespace, "multi-fieldselector", "nginx:1.26.0") expectGlobalQuotaUsedAndClaims(ctx, q.GetName(), "0", 0) expectLedgerSettled(ctx, ControllerNamespace, q.GetName()) }) It("aggregates multiple sources across pod emptyDir size and pvc storage size", func() { q := &capsulev1beta2.GlobalCustomQuota{ ObjectMeta: metav1.ObjectMeta{ Name: "gq-multi-source-storage", Labels: map[string]string{ "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", }, }, Spec: capsulev1beta2.GlobalCustomQuotaSpec{ CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ Limit: resource.MustParse("3Gi"), Sources: []capsulev1beta2.CustomQuotaSpecSource{ { VersionKind: runtime.VersionKind{ APIVersion: "v1", Kind: "Pod", }, CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ Path: ".spec.volumes[*].emptyDir.sizeLimit", Operation: quota.OpAdd, }, }, { VersionKind: runtime.VersionKind{ APIVersion: "v1", Kind: "PersistentVolumeClaim", }, CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ Path: ".spec.resources.requests.storage", Operation: quota.OpAdd, }, }, }, }, NamespaceSelectors: []selectors.NamespaceSelector{ { LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ tenantLabel: tenantValue, }, }, }, }, }, } EventuallyCreation(func() error { return k8sClient.Create(ctx, q) }).Should(Succeed()) pod := MakePod(testNamespace, "multi-source-pod", nil, nil, "nginx:1.27.0", "", "1Gi") pvc := MakePVC(testNamespace, "multi-source-pvc", "2Gi") EventuallyCreation(func() error { pod.ResourceVersion = "" return k8sClient.Create(ctx, pod) }).Should(Succeed()) EventuallyCreation(func() error { pvc.ResourceVersion = "" return k8sClient.Create(ctx, pvc) }).Should(Succeed()) expectGlobalQuotaUsedAndClaims(ctx, q.GetName(), "3Gi", 2) expectLedgerSettled(ctx, ControllerNamespace, q.GetName()) }) It("tracks count with multiple field selectors and updates when the pod no longer matches", func() { q := &capsulev1beta2.GlobalCustomQuota{ ObjectMeta: metav1.ObjectMeta{ Name: "gq-count-multi-fieldselector", Labels: map[string]string{ "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", }, }, Spec: capsulev1beta2.GlobalCustomQuotaSpec{ CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ Limit: resource.MustParse("10"), Sources: []capsulev1beta2.CustomQuotaSpecSource{ { VersionKind: runtime.VersionKind{ APIVersion: "v1", Kind: "Pod", }, CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ Operation: quota.OpCount, Selectors: []selectors.SelectorWithFields{ { FieldSelectors: []string{ `.spec.containers[?(@.image=="nginx:1.27.0")]`, `.spec.containers[?(@.name=="main")]`, }, }, }, }, }, }, }, NamespaceSelectors: []selectors.NamespaceSelector{ { LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ tenantLabel: tenantValue, }, }, }, }, }, } EventuallyCreation(func() error { return k8sClient.Create(ctx, q) }).Should(Succeed()) pod := MakePod(testNamespace, "multi-fieldselector", nil, nil, "nginx:1.27.0", "", "") EventuallyCreation(func() error { pod.ResourceVersion = "" return k8sClient.Create(ctx, pod) }).Should(Succeed()) expectGlobalQuotaUsedAndClaims(ctx, q.GetName(), "1", 1) UpdatePodImage(ctx, testNamespace, "multi-fieldselector", "nginx:1.26.0") expectGlobalQuotaUsedAndClaims(ctx, q.GetName(), "0", 0) expectLedgerSettled(ctx, ControllerNamespace, q.GetName()) }) It("aggregates multiple sources with selectors across pod emptyDir and pvc storage", func() { q := &capsulev1beta2.GlobalCustomQuota{ ObjectMeta: metav1.ObjectMeta{ Name: "gq-multi-source-selectors-storage", Labels: map[string]string{ "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", }, }, Spec: capsulev1beta2.GlobalCustomQuotaSpec{ CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ Limit: resource.MustParse("10Gi"), Sources: []capsulev1beta2.CustomQuotaSpecSource{ { VersionKind: runtime.VersionKind{ APIVersion: "v1", Kind: "Pod", }, CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ Path: ".spec.volumes[*].emptyDir.sizeLimit", Operation: quota.OpAdd, Selectors: []selectors.SelectorWithFields{ { LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ "track": "yes", }, }, }, }, }, }, { VersionKind: runtime.VersionKind{ APIVersion: "v1", Kind: "PersistentVolumeClaim", }, CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ Path: ".spec.resources.requests.storage", Operation: quota.OpAdd, Selectors: []selectors.SelectorWithFields{ { FieldSelectors: []string{ `.spec.accessModes[?(@=="ReadWriteOnce")]`, }, }, }, }, }, }, }, NamespaceSelectors: []selectors.NamespaceSelector{ { LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ tenantLabel: tenantValue, }, }, }, }, }, } EventuallyCreation(func() error { return k8sClient.Create(ctx, q) }).Should(Succeed()) matchingPod := MakePod(testNamespace, "matching-emptydir", map[string]string{"track": "yes"}, nil, "nginx:1.27.0", "", "1Gi") nonMatchingPod := MakePod(testNamespace, "ignored-emptydir", map[string]string{"track": "no"}, nil, "nginx:1.27.0", "", "5Gi") matchingPVC := MakePVC(testNamespace, "matching-pvc", "2Gi") nonMatchingPVC := MakePVC(testNamespace, "ignored-pvc", "4Gi") nonMatchingPVC.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} EventuallyCreation(func() error { matchingPod.ResourceVersion = "" return k8sClient.Create(ctx, matchingPod) }).Should(Succeed()) EventuallyCreation(func() error { nonMatchingPod.ResourceVersion = "" return k8sClient.Create(ctx, nonMatchingPod) }).Should(Succeed()) EventuallyCreation(func() error { matchingPVC.ResourceVersion = "" return k8sClient.Create(ctx, matchingPVC) }).Should(Succeed()) EventuallyCreation(func() error { nonMatchingPVC.ResourceVersion = "" return k8sClient.Create(ctx, nonMatchingPVC) }).Should(Succeed()) expectGlobalQuotaUsedAndClaims(ctx, q.GetName(), "3Gi", 2) expectLedgerSettled(ctx, ControllerNamespace, q.GetName()) }) It("reconciles multiple sources when objects stop matching or are deleted", func() { q := &capsulev1beta2.GlobalCustomQuota{ ObjectMeta: metav1.ObjectMeta{ Name: "gq-multi-source-reconcile", Labels: map[string]string{ "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", }, }, Spec: capsulev1beta2.GlobalCustomQuotaSpec{ CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ Limit: resource.MustParse("10Gi"), Sources: []capsulev1beta2.CustomQuotaSpecSource{ { VersionKind: runtime.VersionKind{ APIVersion: "v1", Kind: "Pod", }, CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ Path: ".spec.volumes[*].emptyDir.sizeLimit", Operation: quota.OpAdd, Selectors: []selectors.SelectorWithFields{ { LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ "track": "yes", }, }, }, }, }, }, { VersionKind: runtime.VersionKind{ APIVersion: "v1", Kind: "PersistentVolumeClaim", }, CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ Path: ".spec.resources.requests.storage", Operation: quota.OpAdd, }, }, }, }, NamespaceSelectors: []selectors.NamespaceSelector{ { LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ tenantLabel: tenantValue, }, }, }, }, }, } EventuallyCreation(func() error { return k8sClient.Create(ctx, q) }).Should(Succeed()) pod := MakePod(testNamespace, "reconcile-emptydir", map[string]string{"track": "yes"}, nil, "nginx:1.27.0", "", "1Gi") pvc := MakePVC(testNamespace, "reconcile-pvc", "2Gi") EventuallyCreation(func() error { pod.ResourceVersion = "" return k8sClient.Create(ctx, pod) }).Should(Succeed()) EventuallyCreation(func() error { pvc.ResourceVersion = "" return k8sClient.Create(ctx, pvc) }).Should(Succeed()) expectGlobalQuotaUsedAndClaims(ctx, q.GetName(), "3Gi", 2) UpdatePodLabels(ctx, testNamespace, "reconcile-emptydir", map[string]string{"track": "no"}) expectGlobalQuotaUsedAndClaims(ctx, q.GetName(), "2Gi", 1) EventuallyDeletion(pvc) expectGlobalQuotaUsedAndClaims(ctx, q.GetName(), "0", 0) expectLedgerSettled(ctx, ControllerNamespace, q.GetName()) }) It("rejects admission when a field selector uses an invalid jsonpath filter on a scalar", func() { q := &capsulev1beta2.GlobalCustomQuota{ ObjectMeta: metav1.ObjectMeta{ Name: "gq-invalid-fieldselector", Labels: map[string]string{ "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", }, }, Spec: capsulev1beta2.GlobalCustomQuotaSpec{ CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ Limit: resource.MustParse("10"), Sources: []capsulev1beta2.CustomQuotaSpecSource{ { VersionKind: runtime.VersionKind{ APIVersion: "v1", Kind: "Pod", }, CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ Operation: quota.OpCount, Selectors: []selectors.SelectorWithFields{ { FieldSelectors: []string{ `.spec.restartPolicy[?(@=="Always")]`, }, }, }, }, }, }, }, NamespaceSelectors: []selectors.NamespaceSelector{ { LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ tenantLabel: tenantValue, }, }, }, }, }, } EventuallyCreation(func() error { return k8sClient.Create(ctx, q) }).Should(Succeed()) pod := MakePod(testNamespace, "invalid-selector-pod", nil, nil, "nginx:1.27.0", "", "") Eventually(func() error { pod.ResourceVersion = "" return k8sClient.Create(ctx, pod) }, defaultTimeoutInterval, defaultPollInterval).Should( MatchError(ContainSubstring("is not array or slice and cannot be filtered")), ) }) It("uses the smallest matching global quota as authoritative while accounting usage in all matching quotas for pod count", func() { small := &capsulev1beta2.GlobalCustomQuota{ ObjectMeta: metav1.ObjectMeta{ Name: "gq-pod-count-small", Labels: map[string]string{ "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", }, }, Spec: capsulev1beta2.GlobalCustomQuotaSpec{ CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ Limit: resource.MustParse("2"), Sources: []capsulev1beta2.CustomQuotaSpecSource{ { VersionKind: runtime.VersionKind{ APIVersion: "v1", Kind: "Pod", }, CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ Operation: quota.OpCount, }, }, }, }, NamespaceSelectors: []selectors.NamespaceSelector{ { LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ tenantLabel: tenantValue, }, }, }, }, }, } large := &capsulev1beta2.GlobalCustomQuota{ ObjectMeta: metav1.ObjectMeta{ Name: "gq-pod-count-large", Labels: map[string]string{ "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", }, }, Spec: capsulev1beta2.GlobalCustomQuotaSpec{ CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ Limit: resource.MustParse("5"), Sources: []capsulev1beta2.CustomQuotaSpecSource{ { VersionKind: runtime.VersionKind{ APIVersion: "v1", Kind: "Pod", }, CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ Operation: quota.OpCount, }, }, }, }, NamespaceSelectors: []selectors.NamespaceSelector{ { LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ tenantLabel: tenantValue, }, }, }, }, }, } EventuallyCreation(func() error { return k8sClient.Create(ctx, small) }).Should(Succeed()) EventuallyCreation(func() error { return k8sClient.Create(ctx, large) }).Should(Succeed()) awaitAllGlobalQuotasReady(small.GetName(), large.GetName()) pod1 := MakePod(testNamespace, "multi-gq-count-1", nil, nil, "nginx:1.27.0", "", "") pod2 := MakePod(testNamespace, "multi-gq-count-2", nil, nil, "nginx:1.27.0", "", "") EventuallyCreation(func() error { pod1.ResourceVersion = "" return k8sClient.Create(ctx, pod1) }).Should(Succeed()) EventuallyCreation(func() error { pod2.ResourceVersion = "" return k8sClient.Create(ctx, pod2) }).Should(Succeed()) expectLedgerSettled(ctx, ControllerNamespace, small.GetName()) expectLedgerSettled(ctx, ControllerNamespace, large.GetName()) expectGlobalQuotaUsedAndClaims(ctx, small.GetName(), "2", 2) expectGlobalQuotaUsedAndClaims(ctx, large.GetName(), "2", 2) expectPodCreationDeniedContaining(func(name string) *corev1.Pod { return MakePod(testNamespace, name, nil, nil, "nginx:1.27.0", "", "") }, `GlobalCustomQuota "gq-pod-count-small"`) expectGlobalQuotaUsedAndClaims(ctx, small.GetName(), "2", 2) expectGlobalQuotaUsedAndClaims(ctx, large.GetName(), "2", 2) }) It("uses the smallest matching global quota as authoritative while accounting usage in all matching quotas for pod cpu requests", func() { small := &capsulev1beta2.GlobalCustomQuota{ ObjectMeta: metav1.ObjectMeta{ Name: "gq-pod-cpu-small", Labels: map[string]string{ "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", }, }, Spec: capsulev1beta2.GlobalCustomQuotaSpec{ CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ Limit: resource.MustParse("200m"), Sources: []capsulev1beta2.CustomQuotaSpecSource{ { VersionKind: runtime.VersionKind{ APIVersion: "v1", Kind: "Pod", }, CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ Operation: quota.OpAdd, Path: ".spec.containers[*].resources.requests.cpu", }, }, }, }, NamespaceSelectors: []selectors.NamespaceSelector{ { LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ tenantLabel: tenantValue, }, }, }, }, }, } large := &capsulev1beta2.GlobalCustomQuota{ ObjectMeta: metav1.ObjectMeta{ Name: "gq-pod-cpu-large", Labels: map[string]string{ "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", }, }, Spec: capsulev1beta2.GlobalCustomQuotaSpec{ CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ Limit: resource.MustParse("500m"), Sources: []capsulev1beta2.CustomQuotaSpecSource{ { VersionKind: runtime.VersionKind{ APIVersion: "v1", Kind: "Pod", }, CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ Operation: quota.OpAdd, Path: ".spec.containers[*].resources.requests.cpu", }, }, }, }, NamespaceSelectors: []selectors.NamespaceSelector{ { LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ tenantLabel: tenantValue, }, }, }, }, }, } EventuallyCreation(func() error { return k8sClient.Create(ctx, small) }).Should(Succeed()) EventuallyCreation(func() error { return k8sClient.Create(ctx, large) }).Should(Succeed()) awaitAllGlobalQuotasReady(small.GetName(), large.GetName()) pod1 := MakePod(testNamespace, "multi-gq-cpu-1", nil, nil, "nginx:1.27.0", "100m", "") pod2 := MakePod(testNamespace, "multi-gq-cpu-2", nil, nil, "nginx:1.27.0", "100m", "") EventuallyCreation(func() error { pod1.ResourceVersion = "" return k8sClient.Create(ctx, pod1) }).Should(Succeed()) EventuallyCreation(func() error { pod2.ResourceVersion = "" return k8sClient.Create(ctx, pod2) }).Should(Succeed()) expectLedgerSettled(ctx, ControllerNamespace, small.GetName()) expectLedgerSettled(ctx, ControllerNamespace, large.GetName()) expectGlobalQuotaUsedAndClaims(ctx, small.GetName(), "200m", 2) expectGlobalQuotaUsedAndClaims(ctx, large.GetName(), "200m", 2) expectPodCreationDeniedContaining(func(name string) *corev1.Pod { return MakePod(testNamespace, name, nil, nil, "nginx:1.27.0", "100m", "") }, `GlobalCustomQuota "gq-pod-cpu-small"`) expectGlobalQuotaUsedAndClaims(ctx, small.GetName(), "200m", 2) expectGlobalQuotaUsedAndClaims(ctx, large.GetName(), "200m", 2) }) It("accounts only the matching subset for overlapping selectors on the same pod gvk", func() { broad := &capsulev1beta2.GlobalCustomQuota{ ObjectMeta: metav1.ObjectMeta{ Name: "gq-pod-track-broad", Labels: map[string]string{ "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", }, }, Spec: capsulev1beta2.GlobalCustomQuotaSpec{ CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ Limit: resource.MustParse("5"), Sources: []capsulev1beta2.CustomQuotaSpecSource{ { VersionKind: runtime.VersionKind{ APIVersion: "v1", Kind: "Pod", }, CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ Operation: quota.OpCount, Selectors: []selectors.SelectorWithFields{ { LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ "track": "yes", }, }, }, }, }, }, }, }, NamespaceSelectors: []selectors.NamespaceSelector{ { LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ tenantLabel: tenantValue, }, }, }, }, }, } narrow := &capsulev1beta2.GlobalCustomQuota{ ObjectMeta: metav1.ObjectMeta{ Name: "gq-pod-track-frontend", Labels: map[string]string{ "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", }, }, Spec: capsulev1beta2.GlobalCustomQuotaSpec{ CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ Limit: resource.MustParse("2"), Sources: []capsulev1beta2.CustomQuotaSpecSource{ { VersionKind: runtime.VersionKind{ APIVersion: "v1", Kind: "Pod", }, CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ Operation: quota.OpCount, Selectors: []selectors.SelectorWithFields{ { LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ "track": "yes", "tier": "frontend", }, }, }, }, }, }, }, }, NamespaceSelectors: []selectors.NamespaceSelector{ { LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ tenantLabel: tenantValue, }, }, }, }, }, } EventuallyCreation(func() error { return k8sClient.Create(ctx, broad) }).Should(Succeed()) EventuallyCreation(func() error { return k8sClient.Create(ctx, narrow) }).Should(Succeed()) awaitAllGlobalQuotasReady(narrow.GetName(), broad.GetName()) pod1 := MakePod(testNamespace, "track-frontend-1", map[string]string{"track": "yes", "tier": "frontend"}, nil, "nginx:1.27.0", "", "") pod2 := MakePod(testNamespace, "track-frontend-2", map[string]string{"track": "yes", "tier": "frontend"}, nil, "nginx:1.27.0", "", "") pod3 := MakePod(testNamespace, "track-backend-1", map[string]string{"track": "yes", "tier": "backend"}, nil, "nginx:1.27.0", "", "") pod4 := MakePod(testNamespace, "track-backend-2", map[string]string{"track": "yes", "tier": "backend"}, nil, "nginx:1.27.0", "", "") EventuallyCreation(func() error { pod1.ResourceVersion = ""; return k8sClient.Create(ctx, pod1) }).Should(Succeed()) EventuallyCreation(func() error { pod2.ResourceVersion = ""; return k8sClient.Create(ctx, pod2) }).Should(Succeed()) EventuallyCreation(func() error { pod3.ResourceVersion = ""; return k8sClient.Create(ctx, pod3) }).Should(Succeed()) EventuallyCreation(func() error { pod4.ResourceVersion = ""; return k8sClient.Create(ctx, pod4) }).Should(Succeed()) expectLedgerSettled(ctx, ControllerNamespace, broad.GetName()) expectLedgerSettled(ctx, ControllerNamespace, narrow.GetName()) expectGlobalQuotaUsedAndClaims(ctx, broad.GetName(), "4", 4) expectGlobalQuotaUsedAndClaims(ctx, narrow.GetName(), "2", 2) expectPodCreationDeniedContaining(func(name string) *corev1.Pod { return MakePod(testNamespace, name, map[string]string{"track": "yes", "tier": "frontend"}, nil, "nginx:1.27.0", "", "") }, `GlobalCustomQuota "gq-pod-track-frontend"`) expectGlobalQuotaUsedAndClaims(ctx, broad.GetName(), "4", 4) expectGlobalQuotaUsedAndClaims(ctx, narrow.GetName(), "2", 2) EventuallyCreation(func() error { pod := MakePod(testNamespace, "track-backend-3", map[string]string{"track": "yes", "tier": "backend"}, nil, "nginx:1.27.0", "", "") pod.ResourceVersion = "" return k8sClient.Create(ctx, pod) }).Should(Succeed()) expectLedgerSettled(ctx, ControllerNamespace, broad.GetName()) expectLedgerSettled(ctx, ControllerNamespace, narrow.GetName()) expectGlobalQuotaUsedAndClaims(ctx, broad.GetName(), "5", 5) expectGlobalQuotaUsedAndClaims(ctx, narrow.GetName(), "2", 2) }) It("tracks different paths independently when multiple global quotas match the same pod gvk", func() { cpuQuota := &capsulev1beta2.GlobalCustomQuota{ ObjectMeta: metav1.ObjectMeta{ Name: "gq-pod-path-cpu", Labels: map[string]string{ "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", }, }, Spec: capsulev1beta2.GlobalCustomQuotaSpec{ CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ Limit: resource.MustParse("400m"), Sources: []capsulev1beta2.CustomQuotaSpecSource{ { VersionKind: runtime.VersionKind{ APIVersion: "v1", Kind: "Pod", }, CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ Operation: quota.OpAdd, Path: ".spec.containers[*].resources.requests.cpu", }, }, }, }, NamespaceSelectors: []selectors.NamespaceSelector{ { LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ tenantLabel: tenantValue, }, }, }, }, }, } emptyDirQuota := &capsulev1beta2.GlobalCustomQuota{ ObjectMeta: metav1.ObjectMeta{ Name: "gq-pod-path-emptydir", Labels: map[string]string{ "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", }, }, Spec: capsulev1beta2.GlobalCustomQuotaSpec{ CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ Limit: resource.MustParse("2Gi"), Sources: []capsulev1beta2.CustomQuotaSpecSource{ { VersionKind: runtime.VersionKind{ APIVersion: "v1", Kind: "Pod", }, CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ Operation: quota.OpAdd, Path: ".spec.volumes[*].emptyDir.sizeLimit", }, }, }, }, NamespaceSelectors: []selectors.NamespaceSelector{ { LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ tenantLabel: tenantValue, }, }, }, }, }, } EventuallyCreation(func() error { return k8sClient.Create(ctx, cpuQuota) }).Should(Succeed()) EventuallyCreation(func() error { return k8sClient.Create(ctx, emptyDirQuota) }).Should(Succeed()) awaitAllGlobalQuotasReady(cpuQuota.GetName(), emptyDirQuota.GetName()) pod1 := MakePod(testNamespace, "path-pod-1", nil, nil, "nginx:1.27.0", "100m", "1Gi") pod2 := MakePod(testNamespace, "path-pod-2", nil, nil, "nginx:1.27.0", "100m", "1Gi") EventuallyCreation(func() error { pod1.ResourceVersion = ""; return k8sClient.Create(ctx, pod1) }).Should(Succeed()) EventuallyCreation(func() error { pod2.ResourceVersion = ""; return k8sClient.Create(ctx, pod2) }).Should(Succeed()) expectLedgerSettled(ctx, ControllerNamespace, cpuQuota.GetName()) expectLedgerSettled(ctx, ControllerNamespace, emptyDirQuota.GetName()) expectGlobalQuotaUsedAndClaims(ctx, cpuQuota.GetName(), "200m", 2) expectGlobalQuotaUsedAndClaims(ctx, emptyDirQuota.GetName(), "2Gi", 2) expectPodCreationDeniedContaining(func(name string) *corev1.Pod { return MakePod(testNamespace, name, nil, nil, "nginx:1.27.0", "100m", "1Gi") }, `GlobalCustomQuota "gq-pod-path-emptydir"`) expectGlobalQuotaUsedAndClaims(ctx, cpuQuota.GetName(), "200m", 2) expectGlobalQuotaUsedAndClaims(ctx, emptyDirQuota.GetName(), "2Gi", 2) }) It("accounts only the quotas that actually match when multiple global quotas share the same gvk", func() { labelQuota := &capsulev1beta2.GlobalCustomQuota{ ObjectMeta: metav1.ObjectMeta{ Name: "gq-pod-track-only", Labels: map[string]string{ "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", }, }, Spec: capsulev1beta2.GlobalCustomQuotaSpec{ CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ Limit: resource.MustParse("10"), Sources: []capsulev1beta2.CustomQuotaSpecSource{ { VersionKind: runtime.VersionKind{ APIVersion: "v1", Kind: "Pod", }, CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ Operation: quota.OpCount, Selectors: []selectors.SelectorWithFields{ { LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ "track": "yes", }, }, }, }, }, }, }, }, NamespaceSelectors: []selectors.NamespaceSelector{ { LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ tenantLabel: tenantValue, }, }, }, }, }, } fieldQuota := &capsulev1beta2.GlobalCustomQuota{ ObjectMeta: metav1.ObjectMeta{ Name: "gq-pod-nginx-only", Labels: map[string]string{ "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", }, }, Spec: capsulev1beta2.GlobalCustomQuotaSpec{ CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ Limit: resource.MustParse("10"), Sources: []capsulev1beta2.CustomQuotaSpecSource{ { VersionKind: runtime.VersionKind{ APIVersion: "v1", Kind: "Pod", }, CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ Operation: quota.OpCount, Selectors: []selectors.SelectorWithFields{ { FieldSelectors: []string{ `.spec.containers[?(@.image=="nginx:1.27.0")]`, }, }, }, }, }, }, }, NamespaceSelectors: []selectors.NamespaceSelector{ { LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ tenantLabel: tenantValue, }, }, }, }, }, } EventuallyCreation(func() error { return k8sClient.Create(ctx, labelQuota) }).Should(Succeed()) EventuallyCreation(func() error { return k8sClient.Create(ctx, fieldQuota) }).Should(Succeed()) awaitAllGlobalQuotasReady(labelQuota.GetName(), fieldQuota.GetName()) matchBoth := MakePod(testNamespace, "subset-both", map[string]string{"track": "yes"}, nil, "nginx:1.27.0", "", "") matchLabelOnly := MakePod(testNamespace, "subset-label-only", map[string]string{"track": "yes"}, nil, "busybox:1.36.1", "", "") matchFieldOnly := MakePod(testNamespace, "subset-field-only", nil, nil, "nginx:1.27.0", "", "") EventuallyCreation(func() error { matchBoth.ResourceVersion = ""; return k8sClient.Create(ctx, matchBoth) }).Should(Succeed()) EventuallyCreation(func() error { matchLabelOnly.ResourceVersion = ""; return k8sClient.Create(ctx, matchLabelOnly) }).Should(Succeed()) EventuallyCreation(func() error { matchFieldOnly.ResourceVersion = ""; return k8sClient.Create(ctx, matchFieldOnly) }).Should(Succeed()) expectLedgerSettled(ctx, ControllerNamespace, labelQuota.GetName()) expectLedgerSettled(ctx, ControllerNamespace, fieldQuota.GetName()) expectGlobalQuotaUsedAndClaims(ctx, labelQuota.GetName(), "2", 2) expectGlobalQuotaUsedAndClaims(ctx, fieldQuota.GetName(), "2", 2) }) It("uses deterministic tie-breaking when multiple global quotas have the same remaining availability", func() { quotaA := &capsulev1beta2.GlobalCustomQuota{ ObjectMeta: metav1.ObjectMeta{ Name: "gq-tie-a", Labels: map[string]string{ "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", }, }, Spec: capsulev1beta2.GlobalCustomQuotaSpec{ CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ Limit: resource.MustParse("3"), Sources: []capsulev1beta2.CustomQuotaSpecSource{ { VersionKind: runtime.VersionKind{ APIVersion: "v1", Kind: "Pod", }, CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ Operation: quota.OpCount, }, }, }, }, NamespaceSelectors: []selectors.NamespaceSelector{ { LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ tenantLabel: tenantValue, }, }, }, }, }, } quotaB := &capsulev1beta2.GlobalCustomQuota{ ObjectMeta: metav1.ObjectMeta{ Name: "gq-tie-b", Labels: map[string]string{ "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", }, }, Spec: capsulev1beta2.GlobalCustomQuotaSpec{ CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ Limit: resource.MustParse("4"), Sources: []capsulev1beta2.CustomQuotaSpecSource{ { VersionKind: runtime.VersionKind{ APIVersion: "v1", Kind: "Pod", }, CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ Operation: quota.OpCount, }, }, }, }, NamespaceSelectors: []selectors.NamespaceSelector{ { LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ tenantLabel: tenantValue, }, }, }, }, }, } EventuallyCreation(func() error { return k8sClient.Create(ctx, quotaA) }).Should(Succeed()) EventuallyCreation(func() error { return k8sClient.Create(ctx, quotaB) }).Should(Succeed()) awaitAllGlobalQuotasReady(quotaA.GetName(), quotaB.GetName()) // Drive them to equal remaining availability: // gq-tie-a limit 3, used 1 => available 2 // gq-tie-b limit 4, used 2 => available 2 pod1 := MakePod(testNamespace, "tie-1", nil, nil, "nginx:1.27.0", "", "") pod2 := MakePod(testNamespace, "tie-2", nil, nil, "nginx:1.27.0", "", "") EventuallyCreation(func() error { pod1.ResourceVersion = ""; return k8sClient.Create(ctx, pod1) }).Should(Succeed()) EventuallyCreation(func() error { pod2.ResourceVersion = ""; return k8sClient.Create(ctx, pod2) }).Should(Succeed()) expectLedgerSettled(ctx, ControllerNamespace, quotaA.GetName()) expectLedgerSettled(ctx, ControllerNamespace, quotaB.GetName()) expectGlobalQuotaUsedAndClaims(ctx, quotaA.GetName(), "2", 2) expectGlobalQuotaUsedAndClaims(ctx, quotaB.GetName(), "2", 2) // Next pod is still allowed, because both have 1 and 2 available respectively after pod3. pod3 := MakePod(testNamespace, "tie-3", nil, nil, "nginx:1.27.0", "", "") EventuallyCreation(func() error { pod3.ResourceVersion = ""; return k8sClient.Create(ctx, pod3) }).Should(Succeed()) expectLedgerSettled(ctx, ControllerNamespace, quotaA.GetName()) expectLedgerSettled(ctx, ControllerNamespace, quotaB.GetName()) expectGlobalQuotaUsedAndClaims(ctx, quotaA.GetName(), "3", 3) expectGlobalQuotaUsedAndClaims(ctx, quotaB.GetName(), "3", 3) // Now gq-tie-a is exhausted first and should be authoritative. expectPodCreationDeniedContaining(func(name string) *corev1.Pod { return MakePod(testNamespace, name, nil, nil, "nginx:1.27.0", "", "") }, `GlobalCustomQuota "gq-tie-a"`) expectGlobalQuotaUsedAndClaims(ctx, quotaA.GetName(), "3", 3) expectGlobalQuotaUsedAndClaims(ctx, quotaB.GetName(), "3", 3) }) It("aggregates the same successful pod into multiple quotas with different paths on the same gvk", func() { cpuQuota := &capsulev1beta2.GlobalCustomQuota{ ObjectMeta: metav1.ObjectMeta{ Name: "gq-multi-path-cpu", Labels: map[string]string{ "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", }, }, Spec: capsulev1beta2.GlobalCustomQuotaSpec{ CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ Limit: resource.MustParse("300m"), Sources: []capsulev1beta2.CustomQuotaSpecSource{ { VersionKind: runtime.VersionKind{ APIVersion: "v1", Kind: "Pod", }, CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ Operation: quota.OpAdd, Path: ".spec.containers[*].resources.requests.cpu", }, }, }, }, NamespaceSelectors: []selectors.NamespaceSelector{ { LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ tenantLabel: tenantValue, }, }, }, }, }, } emptyDirQuota := &capsulev1beta2.GlobalCustomQuota{ ObjectMeta: metav1.ObjectMeta{ Name: "gq-multi-path-emptydir", Labels: map[string]string{ "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", }, }, Spec: capsulev1beta2.GlobalCustomQuotaSpec{ CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ Limit: resource.MustParse("3Gi"), Sources: []capsulev1beta2.CustomQuotaSpecSource{ { VersionKind: runtime.VersionKind{ APIVersion: "v1", Kind: "Pod", }, CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ Operation: quota.OpAdd, Path: ".spec.volumes[*].emptyDir.sizeLimit", }, }, }, }, NamespaceSelectors: []selectors.NamespaceSelector{ { LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ tenantLabel: tenantValue, }, }, }, }, }, } EventuallyCreation(func() error { return k8sClient.Create(ctx, cpuQuota) }).Should(Succeed()) EventuallyCreation(func() error { return k8sClient.Create(ctx, emptyDirQuota) }).Should(Succeed()) awaitAllGlobalQuotasReady(cpuQuota.GetName(), emptyDirQuota.GetName()) pod := MakePod(testNamespace, "multi-path-shared-1", nil, nil, "nginx:1.27.0", "100m", "1Gi") EventuallyCreation(func() error { pod.ResourceVersion = "" return k8sClient.Create(ctx, pod) }).Should(Succeed()) expectLedgerSettled(ctx, ControllerNamespace, cpuQuota.GetName()) expectLedgerSettled(ctx, ControllerNamespace, emptyDirQuota.GetName()) expectGlobalQuotaUsedAndClaims(ctx, cpuQuota.GetName(), "100m", 1) expectGlobalQuotaUsedAndClaims(ctx, emptyDirQuota.GetName(), "1Gi", 1) pod2 := MakePod(testNamespace, "multi-path-shared-2", nil, nil, "nginx:1.27.0", "100m", "1Gi") EventuallyCreation(func() error { pod2.ResourceVersion = "" return k8sClient.Create(ctx, pod2) }).Should(Succeed()) expectGlobalQuotaUsedAndClaims(ctx, cpuQuota.GetName(), "200m", 2) expectGlobalQuotaUsedAndClaims(ctx, emptyDirQuota.GetName(), "2Gi", 2) }) It("excludes succeeded pods from cpu limit quota using not-equals field selector", func() { q := &capsulev1beta2.GlobalCustomQuota{ ObjectMeta: metav1.ObjectMeta{ Name: "gq-not-equals-succeeded", Labels: map[string]string{ "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", }, }, Spec: capsulev1beta2.GlobalCustomQuotaSpec{ CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ Limit: resource.MustParse("2"), Sources: []capsulev1beta2.CustomQuotaSpecSource{ { VersionKind: runtime.VersionKind{ APIVersion: "v1", Kind: "Pod", }, CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ Operation: quota.OpAdd, Path: ".spec.containers[*].resources.limits.cpu", Selectors: []selectors.SelectorWithFields{ { FieldSelectors: []string{ ".status.phase!=Succeeded", ".status.phase!=Failed", ".status.phase!=Unknown", }, }, }, }, }, }, }, NamespaceSelectors: []selectors.NamespaceSelector{ { LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ tenantLabel: tenantValue, }, }, }, }, }, } EventuallyCreation(func() error { return k8sClient.Create(ctx, q) }).Should(Succeed()) awaitGlobalQuotaReady(ctx, q.GetName()) // Running pod with 1 CPU limit — should be counted. runningPod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "ne-running", Namespace: testNamespace, }, Spec: corev1.PodSpec{ SecurityContext: nobodyPodSecurityContext(), Containers: []corev1.Container{ { Name: "main", Image: "nginx:1.27.0", SecurityContext: restrictedContainerSecurityContext(), Resources: corev1.ResourceRequirements{ Limits: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("1"), corev1.ResourceMemory: resource.MustParse("64Mi"), }, }, }, }, RestartPolicy: corev1.RestartPolicyAlways, }, } EventuallyCreation(func() error { runningPod.ResourceVersion = "" return k8sClient.Create(ctx, runningPod) }).Should(Succeed()) expectGlobalQuotaUsedAndClaims(ctx, q.GetName(), "1", 1) // Succeeded pod with 1 CPU limit — must be excluded after completion. succeededPod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "ne-succeeded", Namespace: testNamespace, }, Spec: corev1.PodSpec{ SecurityContext: nobodyPodSecurityContext(), Containers: []corev1.Container{ { Name: "main", Image: "busybox:1.36", Command: []string{"sh", "-c", "exit 0"}, SecurityContext: restrictedContainerSecurityContext(), Resources: corev1.ResourceRequirements{ Limits: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("1"), corev1.ResourceMemory: resource.MustParse("64Mi"), }, }, }, }, RestartPolicy: corev1.RestartPolicyNever, TerminationGracePeriodSeconds: ptr.To(int64(0)), }, } EventuallyCreation(func() error { succeededPod.ResourceVersion = "" return k8sClient.Create(ctx, succeededPod) }).Should(Succeed()) // Wait for the pod to complete. Eventually(func(g Gomega) { obj := &corev1.Pod{} g.Expect(k8sClient.Get(ctx, types.NamespacedName{ Name: succeededPod.Name, Namespace: testNamespace, }, obj)).To(Succeed()) g.Expect(obj.Status.Phase).To(Equal(corev1.PodSucceeded)) }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) // Succeeded pod must be excluded; only the running pod should count. expectGlobalQuotaUsedAndClaims(ctx, q.GetName(), "1", 1) expectLedgerSettled(ctx, ControllerNamespace, q.GetName()) // Admission: 1 existing CPU + 2 requested > limit 2 — must be denied. expectPodCreationDeniedContaining(func(name string) *corev1.Pod { return &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: testNamespace}, Spec: corev1.PodSpec{ SecurityContext: nobodyPodSecurityContext(), Containers: []corev1.Container{ { Name: "main", Image: "nginx:1.27.0", SecurityContext: restrictedContainerSecurityContext(), Resources: corev1.ResourceRequirements{ Limits: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("2"), corev1.ResourceMemory: resource.MustParse("64Mi"), }, }, }, }, RestartPolicy: corev1.RestartPolicyAlways, }, } }, q.GetName()) }) It("excludes failed pods from cpu limit quota using not-equals field selector", func() { q := &capsulev1beta2.GlobalCustomQuota{ ObjectMeta: metav1.ObjectMeta{ Name: "gq-not-equals-failed", Labels: map[string]string{ "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", }, }, Spec: capsulev1beta2.GlobalCustomQuotaSpec{ CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ Limit: resource.MustParse("2"), Sources: []capsulev1beta2.CustomQuotaSpecSource{ { VersionKind: runtime.VersionKind{ APIVersion: "v1", Kind: "Pod", }, CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ Operation: quota.OpAdd, Path: ".spec.containers[*].resources.limits.cpu", Selectors: []selectors.SelectorWithFields{ { FieldSelectors: []string{ ".status.phase!=Succeeded", ".status.phase!=Failed", ".status.phase!=Unknown", }, }, }, }, }, }, }, NamespaceSelectors: []selectors.NamespaceSelector{ { LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ tenantLabel: tenantValue, }, }, }, }, }, } EventuallyCreation(func() error { return k8sClient.Create(ctx, q) }).Should(Succeed()) awaitGlobalQuotaReady(ctx, q.GetName()) // Running pod with 1 CPU limit — should be counted. runningPod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "ne-failed-running", Namespace: testNamespace, }, Spec: corev1.PodSpec{ SecurityContext: nobodyPodSecurityContext(), Containers: []corev1.Container{ { Name: "main", Image: "nginx:1.27.0", SecurityContext: restrictedContainerSecurityContext(), Resources: corev1.ResourceRequirements{ Limits: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("1"), corev1.ResourceMemory: resource.MustParse("64Mi"), }, }, }, }, RestartPolicy: corev1.RestartPolicyAlways, }, } EventuallyCreation(func() error { runningPod.ResourceVersion = "" return k8sClient.Create(ctx, runningPod) }).Should(Succeed()) expectGlobalQuotaUsedAndClaims(ctx, q.GetName(), "1", 1) // Failed pod with 1 CPU limit — must be excluded after failure. failedPod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "ne-failed-terminal", Namespace: testNamespace, }, Spec: corev1.PodSpec{ SecurityContext: nobodyPodSecurityContext(), Containers: []corev1.Container{ { Name: "main", Image: "busybox:1.36", Command: []string{"sh", "-c", "exit 1"}, SecurityContext: restrictedContainerSecurityContext(), Resources: corev1.ResourceRequirements{ Limits: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("1"), corev1.ResourceMemory: resource.MustParse("64Mi"), }, }, }, }, RestartPolicy: corev1.RestartPolicyNever, TerminationGracePeriodSeconds: ptr.To(int64(0)), }, } EventuallyCreation(func() error { failedPod.ResourceVersion = "" return k8sClient.Create(ctx, failedPod) }).Should(Succeed()) // Wait for the pod to fail. Eventually(func(g Gomega) { obj := &corev1.Pod{} g.Expect(k8sClient.Get(ctx, types.NamespacedName{ Name: failedPod.Name, Namespace: testNamespace, }, obj)).To(Succeed()) g.Expect(obj.Status.Phase).To(Equal(corev1.PodFailed)) }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) // Failed pod must be excluded; only the running pod should count. expectGlobalQuotaUsedAndClaims(ctx, q.GetName(), "1", 1) expectLedgerSettled(ctx, ControllerNamespace, q.GetName()) }) })