// Copyright 2020-2026 Project Capsule Authors // SPDX-License-Identifier: Apache-2.0 package e2e import ( "context" "fmt" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" "github.com/projectcapsule/capsule/pkg/api/meta" "github.com/projectcapsule/capsule/pkg/api/rbac" "github.com/projectcapsule/capsule/pkg/api/rules" tenantutils "github.com/projectcapsule/capsule/pkg/tenant" ) var _ = Describe("rule-generated GlobalResourceQuota", Ordered, Label("resourcequota", "rules", "ledger", "skip-on-openshift"), func() { const ( tenantName = "e2e-rule-quota" sharedCPUA = "e2e-rule-quota-cpu-a" sharedCPUB = "e2e-rule-quota-cpu-b" podCountA = "e2e-rule-quota-pods-a" podCountB = "e2e-rule-quota-pods-b" hpaCountA = "e2e-rule-quota-hpa-a" hpaCountB = "e2e-rule-quota-hpa-b" podLevelA = "e2e-rule-quota-pod-level-a" podLevelB = "e2e-rule-quota-pod-level-b" serviceA = "e2e-rule-quota-services-a" serviceB = "e2e-rule-quota-services-b" unselected = "e2e-rule-quota-unselected" ) ctx := context.Background() owner := rbac.UserSpec{Name: tenantName, Kind: "User"} quotaScopes := []struct { quotaName string selectorKey string namespaces []string hard corev1.ResourceList }{ { quotaName: "shared-cpu", selectorKey: "e2e.projectcapsule.dev/shared-cpu", namespaces: []string{sharedCPUA, sharedCPUB}, hard: corev1.ResourceList{ corev1.ResourceRequestsCPU: resource.MustParse("1"), }, }, { quotaName: "pod-count", selectorKey: "e2e.projectcapsule.dev/pod-count", namespaces: []string{podCountA, podCountB}, hard: corev1.ResourceList{ corev1.ResourcePods: resource.MustParse("1"), }, }, { quotaName: "hpa-count", selectorKey: "e2e.projectcapsule.dev/hpa-count", namespaces: []string{hpaCountA, hpaCountB}, hard: corev1.ResourceList{ corev1.ResourceName("count/horizontalpodautoscalers.autoscaling"): resource.MustParse("1"), }, }, { quotaName: "pod-level-cpu", selectorKey: "e2e.projectcapsule.dev/pod-level-cpu", namespaces: []string{podLevelA, podLevelB}, hard: corev1.ResourceList{ corev1.ResourceRequestsCPU: resource.MustParse("1"), }, }, { quotaName: "service-count", selectorKey: "e2e.projectcapsule.dev/service-count", namespaces: []string{serviceA, serviceB}, hard: corev1.ResourceList{ corev1.ResourceServices: resource.MustParse("5"), }, }, } ruleBodies := make([]*rules.NamespaceRuleBodyTenant, 0, len(quotaScopes)) for _, scope := range quotaScopes { ruleBodies = append(ruleBodies, &rules.NamespaceRuleBodyTenant{ NamespaceRuleBodyNamespace: &rules.NamespaceRuleBodyNamespace{ Quota: []rules.ResourceQuotaRule{{ Name: scope.quotaName, ResourceQuotaSpec: corev1.ResourceQuotaSpec{Hard: scope.hard}, }}, }, NamespaceSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{scope.selectorKey: "true"}, }, }) } tnt := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ Name: tenantName, Labels: map[string]string{"env": "e2e"}, }, Spec: capsulev1beta2.TenantSpec{ Owners: rbac.OwnerListSpec{{CoreOwnerSpec: rbac.CoreOwnerSpec{UserSpec: owner}}}, Rules: ruleBodies, }, } BeforeAll(func() { EventuallyCreation(func() error { tnt.ResourceVersion = "" return k8sClient.Create(ctx, tnt) }).Should(Succeed()) TenantReadyTrue(tnt) for _, scope := range quotaScopes { for _, namespace := range scope.namespaces { ns := NewNamespace(namespace, map[string]string{ meta.TenantLabel: tenantName, scope.selectorKey: "true", }) NamespaceCreation(ns, owner, defaultTimeoutInterval).Should(Succeed()) NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) } } By("creating a namespace outside every quota selector", func() { ns := NewNamespace(unselected, map[string]string{meta.TenantLabel: tenantName}) NamespaceCreation(ns, owner, defaultTimeoutInterval).Should(Succeed()) NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) }) current := &capsulev1beta2.Tenant{} Expect(k8sClient.Get(ctx, clientKey("", tenantName), current)).To(Succeed()) tnt.UID = current.UID for _, scope := range quotaScopes { globalQuota := &capsulev1beta2.GlobalResourceQuota{} Eventually(func() error { return k8sClient.Get( ctx, clientKey("", tenantutils.RuleGlobalResourceQuotaName(tnt, scope.quotaName)), globalQuota, ) }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) quotaName := globalQuota.GetResourceQuotaName() for _, namespace := range scope.namespaces { Eventually(func(g Gomega) { quota := &corev1.ResourceQuota{} g.Expect(k8sClient.Get(ctx, types.NamespacedName{ Namespace: namespace, Name: quotaName, }, quota)).To(Succeed()) g.Expect(quota.Spec.Hard).To(HaveLen(len(scope.hard))) for name, expected := range scope.hard { actual, found := quota.Spec.Hard[name] g.Expect(found).To(BeTrue(), "missing hard quota resource %s", name) g.Expect(actual.Cmp(expected)).To(Equal(0), "hard quota resource %s", name) } }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) } ledgerName := globalQuota.GetLedgerName() Eventually(func(g Gomega) { ledger := &capsulev1beta2.QuantityLedger{} g.Expect(k8sClient.Get(ctx, types.NamespacedName{ Namespace: ControllerNamespace, Name: ledgerName, }, ledger)).To(Succeed()) g.Expect(ledger.Status.ResourceQuota).NotTo(BeNil()) g.Expect(ledger.Status.ResourceQuota.Initialized).To(BeTrue()) g.Expect(ledger.Status.ResourceQuota.ObservedGeneration).To(Equal(globalQuota.Generation)) g.Expect(ledger.Status.ResourceQuota.Namespaces).To(ConsistOf(scope.namespaces)) }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) } }) AfterAll(func() { EventuallyDeletion(tnt) }) It("enforces one shared limit during concurrent admissions across selected namespaces", func() { const total = 20 cs := ownerClient(owner) results := make(chan error, total) for i := 0; i < total; i++ { go func(index int) { namespace := sharedCPUA if index%2 == 1 { namespace = sharedCPUB } pod := MakePod( namespace, fmt.Sprintf("rule-quota-concurrent-%02d", index), nil, nil, "registry.k8s.io/pause:3.10", "100m", "", ) _, err := cs.CoreV1().Pods(namespace).Create(ctx, pod, metav1.CreateOptions{}) results <- err }(i) } var succeeded, failed int for i := 0; i < total; i++ { if err := <-results; err == nil { succeeded++ } else { failed++ Expect(err.Error()).To(ContainSubstring("exceeds GlobalResourceQuota")) } } Expect(succeeded).To(Equal(10)) Expect(failed).To(Equal(10)) By("not applying the rule quota outside the selected namespace set", func() { pod := MakePod( unselected, "rule-quota-unselected", nil, nil, "registry.k8s.io/pause:3.10", "2", "", ) _, err := cs.CoreV1().Pods(unselected).Create(ctx, pod, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) }) }) It("counts Pods without compute resources and intercepts the shared object limit", func() { cs := ownerClient(owner) first := MakePod( podCountA, "rule-quota-pod-count-first", nil, nil, "registry.k8s.io/pause:3.10", "", "", ) _, err := cs.CoreV1().Pods(podCountA).Create(ctx, first, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) second := MakePod( podCountB, "rule-quota-pod-count-second", nil, nil, "registry.k8s.io/pause:3.10", "", "", ) _, err = cs.CoreV1().Pods(podCountB).Create(ctx, second, metav1.CreateOptions{}) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("exceeds GlobalResourceQuota")) Expect(err.Error()).To(ContainSubstring( "pods (requested=1, current=1, projected=2, hard=1, exceededBy=1)", )) }) It("counts HorizontalPodAutoscalers and intercepts the shared object limit", func() { cs := ownerClient(owner) makeHPA := func(namespace, name string) *autoscalingv2.HorizontalPodAutoscaler { return &autoscalingv2.HorizontalPodAutoscaler{ ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, Spec: autoscalingv2.HorizontalPodAutoscalerSpec{ ScaleTargetRef: autoscalingv2.CrossVersionObjectReference{ APIVersion: "apps/v1", Kind: "Deployment", Name: "quota-target", }, MaxReplicas: 3, }, } } _, err := cs.AutoscalingV2().HorizontalPodAutoscalers(hpaCountA).Create( ctx, makeHPA(hpaCountA, "rule-quota-hpa-first"), metav1.CreateOptions{}, ) Expect(err).NotTo(HaveOccurred()) _, err = cs.AutoscalingV2().HorizontalPodAutoscalers(hpaCountB).Create( ctx, makeHPA(hpaCountB, "rule-quota-hpa-second"), metav1.CreateOptions{}, ) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("exceeds GlobalResourceQuota")) }) It("calculates and intercepts Pod-level resources on controller-generated Pods", func() { cs := ownerClient(owner) makeDeployment := func(namespace, name string) *appsv1.Deployment { deployment := MakeDeployment(namespace, name, 1, nil, "") deployment.Spec.Template.Spec.Resources = &corev1.ResourceRequirements{ Requests: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("600m"), }, } return deployment } firstName := "rule-quota-pod-level-first" _, err := cs.AppsV1().Deployments(podLevelA).Create( ctx, makeDeployment(podLevelA, firstName), metav1.CreateOptions{}, ) Expect(err).NotTo(HaveOccurred()) ExpectPodsForDeployment(ctx, podLevelA, firstName, 1) secondName := "rule-quota-pod-level-second" _, err = cs.AppsV1().Deployments(podLevelB).Create( ctx, makeDeployment(podLevelB, secondName), metav1.CreateOptions{}, ) Expect(err).NotTo(HaveOccurred()) Eventually(func(g Gomega) { failure, getErr := replicaSetFailureForDeployment(ctx, podLevelB, secondName) g.Expect(getErr).NotTo(HaveOccurred()) g.Expect(failure).NotTo(BeNil()) g.Expect(failure.Status).To(Equal(corev1.ConditionTrue)) g.Expect(failure.Message).To(ContainSubstring("exceeds GlobalResourceQuota")) }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) ExpectPodsForDeployment(ctx, podLevelB, secondName, 0) }) It("atomically stalls concurrent Service admissions at the shared limit", func() { const total = 10 cs := ownerClient(owner) results := make(chan error, total) for i := 0; i < total; i++ { go func(index int) { namespace := serviceA if index%2 == 1 { namespace = serviceB } service := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Namespace: namespace, Name: fmt.Sprintf("rule-quota-service-%02d", index), }, Spec: corev1.ServiceSpec{ Ports: []corev1.ServicePort{{Port: 80}}, }, } _, err := cs.CoreV1().Services(namespace).Create(ctx, service, metav1.CreateOptions{}) results <- err }(i) } var succeeded, failed int for i := 0; i < total; i++ { if err := <-results; err == nil { succeeded++ } else { failed++ Expect(err.Error()).To(ContainSubstring("exceeds GlobalResourceQuota")) } } Expect(succeeded).To(Equal(5)) Expect(failed).To(Equal(5)) }) It("rejects invalid rule limits and projects valid changes to every namespace", func() { quotaKey := clientKey("", tenantutils.RuleGlobalResourceQuotaName(tnt, "service-count")) marker := "e2e.projectcapsule.dev/stable-identity" resourceName := corev1.ResourceServices setServiceLimit := func(limit string) error { current := &capsulev1beta2.Tenant{} if err := k8sClient.Get(ctx, clientKey("", tenantName), current); err != nil { return err } var serviceRule *rules.NamespaceRuleBodyTenant remaining := make([]*rules.NamespaceRuleBodyTenant, 0, len(current.Spec.Rules)-1) for _, rule := range current.Spec.Rules { if rule != nil && len(rule.Quota) == 1 && rule.Quota[0].Name == "service-count" { serviceRule = rule continue } remaining = append(remaining, rule) } if serviceRule == nil { return fmt.Errorf("service-count quota rule was not found") } serviceRule.Quota[0].Hard[resourceName] = resource.MustParse(limit) // Moving the updated rule also verifies that its name, rather than its // position, remains the generated quota's durable identity. current.Spec.Rules = append([]*rules.NamespaceRuleBodyTenant{serviceRule}, remaining...) return k8sClient.Update(ctx, current) } expectGlobalQuota := func(hard, used, available string) { Eventually(func(g Gomega) { current := &capsulev1beta2.GlobalResourceQuota{} g.Expect(k8sClient.Get(ctx, quotaKey, current)).To(Succeed()) g.Expect(current.Annotations).To(HaveKeyWithValue(marker, "preserved")) g.Expect(current.Labels).To(HaveKeyWithValue(meta.RuleQuotaLabel, "service-count")) g.Expect(current.Status.ObservedGeneration).To(Equal(current.Generation)) actualHard := current.Spec.Quota.Hard[resourceName] g.Expect(actualHard.Cmp(resource.MustParse(hard))).To(Equal(0)) statusHard := current.Status.Total.Hard[resourceName] g.Expect(statusHard.Cmp(resource.MustParse(hard))).To(Equal(0)) actualUsed := current.Status.Total.Used[resourceName] g.Expect(actualUsed.Cmp(resource.MustParse(used))).To(Equal(0)) actualAvailable := current.Status.Total.Available[resourceName] g.Expect(actualAvailable.Cmp(resource.MustParse(available))).To(Equal(0)) }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) } expectNamespaceRemaining := func(remaining string) { current := &capsulev1beta2.GlobalResourceQuota{} Expect(k8sClient.Get(ctx, quotaKey, current)).To(Succeed()) for _, namespace := range []string{serviceA, serviceB} { Eventually(func(g Gomega) { quota := &corev1.ResourceQuota{} g.Expect(k8sClient.Get(ctx, types.NamespacedName{ Namespace: namespace, Name: current.GetResourceQuotaName(), }, quota)).To(Succeed()) hard := quota.Spec.Hard[resourceName] hard.Sub(quota.Status.Used[resourceName]) g.Expect(hard.Cmp(resource.MustParse(remaining))).To(Equal(0)) }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) } } Eventually(func() error { current := &capsulev1beta2.GlobalResourceQuota{} if err := k8sClient.Get(ctx, quotaKey, current); err != nil { return err } if current.Annotations == nil { current.Annotations = map[string]string{} } current.Annotations[marker] = "preserved" return k8sClient.Update(ctx, current) }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) // The concurrent admission scenario immediately before this test creates // exactly five Services. Wait for native quota accounting before changing // the shared limit. expectGlobalQuota("5", "5", "0") By("rejecting a Tenant rule decrease below current usage", func() { err := setServiceLimit("3") Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring(`hard["services"] cannot be reduced to 3 while 5 is allocated`)) // Admission rejects the Tenant update before the generated quota can // become stale or diverge from its source rule. expectGlobalQuota("5", "5", "0") expectNamespaceRemaining("0") }) Eventually(func() error { return setServiceLimit("7") }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) By("exposing the newly available capacity in every namespaced ResourceQuota", func() { expectGlobalQuota("7", "5", "2") expectNamespaceRemaining("2") }) Eventually(func() error { return setServiceLimit("5") }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) By("allowing a decrease exactly to current usage", func() { expectGlobalQuota("5", "5", "0") expectNamespaceRemaining("0") }) }) }) func clientKey(namespace, name string) types.NamespacedName { return types.NamespacedName{Namespace: namespace, Name: name} }