diff --git a/api/v1beta2/globalresourcequota_func.go b/api/v1beta2/globalresourcequota_func.go new file mode 100644 index 00000000..440581a7 --- /dev/null +++ b/api/v1beta2/globalresourcequota_func.go @@ -0,0 +1,64 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package v1beta2 + +import ( + "crypto/sha256" + "fmt" + "sort" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + + runtimequota "github.com/projectcapsule/capsule/pkg/runtime/quota" +) + +func (q *GlobalResourceQuota) GetResourceQuotaName() string { + sum := sha256.Sum256([]byte(q.Name)) + + return fmt.Sprintf("capsule-global-quota-%x", sum[:10]) +} + +func (q *GlobalResourceQuota) GetLedgerName() string { + sum := sha256.Sum256(fmt.Appendf(nil, "%s/%s", q.Name, q.UID)) + + return fmt.Sprintf("global-resource-quota-%x", sum[:16]) +} + +func (q *GlobalResourceQuota) AssignNamespaces(namespaces []corev1.Namespace) { + names := make([]string, 0, len(namespaces)) + + for i := range namespaces { + ns := &namespaces[i] + if ns.Status.Phase == corev1.NamespaceActive && ns.DeletionTimestamp == nil { + names = append(names, ns.Name) + } + } + + sort.Strings(names) + q.Status.Namespaces = names + q.Status.NamespaceSize = uint(len(names)) +} + +func (q *GlobalResourceQuota) CalculateAvailable() { + available := make(corev1.ResourceList, len(q.Status.Total.Hard)) + + for name, hard := range q.Status.Total.Hard { + value := hard.DeepCopy() + value.Sub(q.Status.Total.Used[name]) + runtimequota.ClampQuantityToZero(&value) + available[name] = value + } + + q.Status.Total.Available = available +} + +func ZeroResourceList(resources corev1.ResourceList) corev1.ResourceList { + out := make(corev1.ResourceList, len(resources)) + for name := range resources { + out[name] = *resource.NewQuantity(0, resource.DecimalSI) + } + + return out +} diff --git a/api/v1beta2/globalresourcequota_status.go b/api/v1beta2/globalresourcequota_status.go new file mode 100644 index 00000000..f3344440 --- /dev/null +++ b/api/v1beta2/globalresourcequota_status.go @@ -0,0 +1,50 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package v1beta2 + +import ( + corev1 "k8s.io/api/core/v1" + + "github.com/projectcapsule/capsule/pkg/api/meta" +) + +type GlobalResourceQuotaStatus struct { + // ObservedGeneration is the most recent generation observed by the controller. + // +optional + ObservedGeneration int64 `json:"observedGeneration,omitempty"` + + // NamespaceSize is the number of selected namespaces. + // +kubebuilder:default=0 + NamespaceSize uint `json:"namespaceCount,omitempty"` + + // Namespaces is the ordered set of selected namespace names. + Namespaces []string `json:"namespaces,omitempty"` + + // Total contains aggregate quota usage across all selected namespaces. + Total GlobalResourceQuotaUsage `json:"total,omitzero"` + + // NamespaceUsage contains observed quota usage per selected namespace. + NamespaceUsage GlobalResourceQuotaNamespaceUsage `json:"namespaceUsage,omitempty"` + + // Conditions report reconciliation and admission readiness. + Conditions meta.ConditionList `json:"conditions,omitzero"` +} + +type GlobalResourceQuotaUsage struct { + // Hard is the configured shared limit. + Hard corev1.ResourceList `json:"hard,omitempty"` + + // Used is the usage observed across the relevant namespace set. + Used corev1.ResourceList `json:"used,omitempty"` + + // Available is max(Hard-Used, 0). + Available corev1.ResourceList `json:"available,omitempty"` +} + +type GlobalResourceQuotaNamespaceUsage map[string]GlobalResourceQuotaNamespaceStatus + +type GlobalResourceQuotaNamespaceStatus struct { + // Used is the usage observed in this namespace. + Used corev1.ResourceList `json:"used,omitempty"` +} diff --git a/api/v1beta2/globalresourcequota_types.go b/api/v1beta2/globalresourcequota_types.go new file mode 100644 index 00000000..3681cb93 --- /dev/null +++ b/api/v1beta2/globalresourcequota_types.go @@ -0,0 +1,53 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package v1beta2 + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/projectcapsule/capsule/pkg/runtime/selectors" +) + +// GlobalResourceQuotaSpec defines a native ResourceQuota shared by every +// namespace matched by any namespace selector. +type GlobalResourceQuotaSpec struct { + // NamespaceSelectors select the namespaces that share this quota. + // Selectors are ORed; requirements within one selector are ANDed. An empty + // label selector matches all namespaces. + NamespaceSelectors []selectors.NamespaceSelector `json:"namespaceSelectors,omitempty"` + + // Quota is the native Kubernetes ResourceQuota specification enforced + // across the selected namespaces. + Quota corev1.ResourceQuotaSpec `json:"quota"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:resource:scope=Cluster,shortName=globalquota;grq +// +kubebuilder:printcolumn:name="Namespaces",type="integer",JSONPath=".status.namespaceCount",description="Selected namespaces" +// +kubebuilder:printcolumn:name="Ready",type="string",JSONPath=".status.conditions[?(@.type==\"Ready\")].status",description="Reconcile status" +// +kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.conditions[?(@.type==\"Ready\")].message",description="Reconcile message" +// +kubebuilder:printcolumn:name="Age",type=date,JSONPath=".metadata.creationTimestamp" +type GlobalResourceQuota struct { + metav1.TypeMeta `json:",inline"` + + // +optional + metav1.ObjectMeta `json:"metadata,omitzero"` + + Spec GlobalResourceQuotaSpec `json:"spec"` + + // +optional + Status GlobalResourceQuotaStatus `json:"status,omitzero"` +} + +// +kubebuilder:object:root=true +type GlobalResourceQuotaList struct { + metav1.TypeMeta `json:",inline"` + + // +optional + metav1.ListMeta `json:"metadata,omitzero"` + + Items []GlobalResourceQuota `json:"items"` +} diff --git a/api/v1beta2/groupversion_info.go b/api/v1beta2/groupversion_info.go index 5a3c4f94..d9e04077 100644 --- a/api/v1beta2/groupversion_info.go +++ b/api/v1beta2/groupversion_info.go @@ -31,6 +31,8 @@ func addKnownTypes(scheme *runtime.Scheme) error { &CustomQuotaList{}, &GlobalCustomQuota{}, &GlobalCustomQuotaList{}, + &GlobalResourceQuota{}, + &GlobalResourceQuotaList{}, &GlobalTenantResource{}, &GlobalTenantResourceList{}, &QuantityLedger{}, diff --git a/api/v1beta2/quantityledgers_status.go b/api/v1beta2/quantityledgers_status.go index d36026c0..4e70161c 100644 --- a/api/v1beta2/quantityledgers_status.go +++ b/api/v1beta2/quantityledgers_status.go @@ -4,6 +4,7 @@ package v1beta2 import ( + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -57,6 +58,55 @@ type QuantityLedgerPendingDelete struct { CreatedAt metav1.Time `json:"createdAt"` } +// QuantityLedgerResourceQuotaReservation is an atomic reservation against all +// resources tracked by a GlobalResourceQuota. +type QuantityLedgerResourceQuotaReservation struct { + // Unique reservation identifier. + // +kubebuilder:validation:MinLength=1 + ID string `json:"id"` + + // Usage is the calculated usage of the admitted object. + Usage corev1.ResourceList `json:"usage,omitempty"` + + // Delta is the positive amount held while ResourceQuota status catches up. + Delta corev1.ResourceList `json:"delta,omitempty"` + + // Object that this reservation is intended to create/update. + ObjectRef QuantityLedgerObjectRef `json:"objectRef"` + + CreatedAt metav1.Time `json:"createdAt"` + UpdatedAt metav1.Time `json:"updatedAt"` + ExpiresAt *metav1.Time `json:"expiresAt,omitempty"` +} + +// QuantityLedgerResourceQuotaStatus is the coordination state for one +// GlobalResourceQuota. +type QuantityLedgerResourceQuotaStatus struct { + // ObservedGeneration is the GlobalResourceQuota generation represented by + // this ledger state. + ObservedGeneration int64 `json:"observedGeneration,omitempty"` + + // Initialized is true after every selected namespace has reported + // ResourceQuota status for this quota. + Initialized bool `json:"initialized,omitempty"` + + // Namespaces is the selected namespace set represented by Used. + Namespaces []string `json:"namespaces,omitempty"` + + // Used is the usage observed from ResourceQuota status across all selected + // namespaces. + Used corev1.ResourceList `json:"used,omitempty"` + + // Reserved is derived from Reservations. + Reserved corev1.ResourceList `json:"reserved,omitempty"` + + // Allocated is Used plus all active reservations. + Allocated corev1.ResourceList `json:"allocated,omitempty"` + + // Reservations contains inflight admission operations. + Reservations []QuantityLedgerResourceQuotaReservation `json:"reservations,omitempty"` +} + // QuantityLedgerStatus contains the mutable coordination state used by admission // and quota controllers. type QuantityLedgerStatus struct { @@ -80,4 +130,9 @@ type QuantityLedgerStatus struct { // Allocated is the admission-owned total that has been accepted by the webhook. // It must be updated only through optimistic concurrency on QuantityLedger. Allocated resource.Quantity `json:"allocated,omitempty"` + + // ResourceQuota contains coordination state for GlobalResourceQuota. + // It is unset for CustomQuota and GlobalCustomQuota ledgers. + // +optional + ResourceQuota *QuantityLedgerResourceQuotaStatus `json:"resourceQuota,omitempty"` } diff --git a/api/v1beta2/zz_generated.deepcopy.go b/api/v1beta2/zz_generated.deepcopy.go index 2bcb3901..a39eb647 100644 --- a/api/v1beta2/zz_generated.deepcopy.go +++ b/api/v1beta2/zz_generated.deepcopy.go @@ -656,6 +656,202 @@ func (in *GlobalCustomQuotaStatus) DeepCopy() *GlobalCustomQuotaStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GlobalResourceQuota) DeepCopyInto(out *GlobalResourceQuota) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GlobalResourceQuota. +func (in *GlobalResourceQuota) DeepCopy() *GlobalResourceQuota { + if in == nil { + return nil + } + out := new(GlobalResourceQuota) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *GlobalResourceQuota) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GlobalResourceQuotaList) DeepCopyInto(out *GlobalResourceQuotaList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]GlobalResourceQuota, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GlobalResourceQuotaList. +func (in *GlobalResourceQuotaList) DeepCopy() *GlobalResourceQuotaList { + if in == nil { + return nil + } + out := new(GlobalResourceQuotaList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *GlobalResourceQuotaList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GlobalResourceQuotaNamespaceStatus) DeepCopyInto(out *GlobalResourceQuotaNamespaceStatus) { + *out = *in + if in.Used != nil { + in, out := &in.Used, &out.Used + *out = make(corev1.ResourceList, len(*in)) + for key, val := range *in { + (*out)[key] = val.DeepCopy() + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GlobalResourceQuotaNamespaceStatus. +func (in *GlobalResourceQuotaNamespaceStatus) DeepCopy() *GlobalResourceQuotaNamespaceStatus { + if in == nil { + return nil + } + out := new(GlobalResourceQuotaNamespaceStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in GlobalResourceQuotaNamespaceUsage) DeepCopyInto(out *GlobalResourceQuotaNamespaceUsage) { + { + in := &in + *out = make(GlobalResourceQuotaNamespaceUsage, len(*in)) + for key, val := range *in { + (*out)[key] = *val.DeepCopy() + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GlobalResourceQuotaNamespaceUsage. +func (in GlobalResourceQuotaNamespaceUsage) DeepCopy() GlobalResourceQuotaNamespaceUsage { + if in == nil { + return nil + } + out := new(GlobalResourceQuotaNamespaceUsage) + in.DeepCopyInto(out) + return *out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GlobalResourceQuotaSpec) DeepCopyInto(out *GlobalResourceQuotaSpec) { + *out = *in + if in.NamespaceSelectors != nil { + in, out := &in.NamespaceSelectors, &out.NamespaceSelectors + *out = make([]selectors.NamespaceSelector, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + in.Quota.DeepCopyInto(&out.Quota) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GlobalResourceQuotaSpec. +func (in *GlobalResourceQuotaSpec) DeepCopy() *GlobalResourceQuotaSpec { + if in == nil { + return nil + } + out := new(GlobalResourceQuotaSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GlobalResourceQuotaStatus) DeepCopyInto(out *GlobalResourceQuotaStatus) { + *out = *in + if in.Namespaces != nil { + in, out := &in.Namespaces, &out.Namespaces + *out = make([]string, len(*in)) + copy(*out, *in) + } + in.Total.DeepCopyInto(&out.Total) + if in.NamespaceUsage != nil { + in, out := &in.NamespaceUsage, &out.NamespaceUsage + *out = make(GlobalResourceQuotaNamespaceUsage, len(*in)) + for key, val := range *in { + (*out)[key] = *val.DeepCopy() + } + } + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make(meta.ConditionList, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GlobalResourceQuotaStatus. +func (in *GlobalResourceQuotaStatus) DeepCopy() *GlobalResourceQuotaStatus { + if in == nil { + return nil + } + out := new(GlobalResourceQuotaStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GlobalResourceQuotaUsage) DeepCopyInto(out *GlobalResourceQuotaUsage) { + *out = *in + if in.Hard != nil { + in, out := &in.Hard, &out.Hard + *out = make(corev1.ResourceList, len(*in)) + for key, val := range *in { + (*out)[key] = val.DeepCopy() + } + } + if in.Used != nil { + in, out := &in.Used, &out.Used + *out = make(corev1.ResourceList, len(*in)) + for key, val := range *in { + (*out)[key] = val.DeepCopy() + } + } + if in.Available != nil { + in, out := &in.Available, &out.Available + *out = make(corev1.ResourceList, len(*in)) + for key, val := range *in { + (*out)[key] = val.DeepCopy() + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GlobalResourceQuotaUsage. +func (in *GlobalResourceQuotaUsage) DeepCopy() *GlobalResourceQuotaUsage { + if in == nil { + return nil + } + out := new(GlobalResourceQuotaUsage) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GlobalTenantResource) DeepCopyInto(out *GlobalTenantResource) { *out = *in @@ -999,6 +1195,90 @@ func (in *QuantityLedgerReservation) DeepCopy() *QuantityLedgerReservation { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *QuantityLedgerResourceQuotaReservation) DeepCopyInto(out *QuantityLedgerResourceQuotaReservation) { + *out = *in + if in.Usage != nil { + in, out := &in.Usage, &out.Usage + *out = make(corev1.ResourceList, len(*in)) + for key, val := range *in { + (*out)[key] = val.DeepCopy() + } + } + if in.Delta != nil { + in, out := &in.Delta, &out.Delta + *out = make(corev1.ResourceList, len(*in)) + for key, val := range *in { + (*out)[key] = val.DeepCopy() + } + } + out.ObjectRef = in.ObjectRef + in.CreatedAt.DeepCopyInto(&out.CreatedAt) + in.UpdatedAt.DeepCopyInto(&out.UpdatedAt) + if in.ExpiresAt != nil { + in, out := &in.ExpiresAt, &out.ExpiresAt + *out = (*in).DeepCopy() + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new QuantityLedgerResourceQuotaReservation. +func (in *QuantityLedgerResourceQuotaReservation) DeepCopy() *QuantityLedgerResourceQuotaReservation { + if in == nil { + return nil + } + out := new(QuantityLedgerResourceQuotaReservation) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *QuantityLedgerResourceQuotaStatus) DeepCopyInto(out *QuantityLedgerResourceQuotaStatus) { + *out = *in + if in.Namespaces != nil { + in, out := &in.Namespaces, &out.Namespaces + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Used != nil { + in, out := &in.Used, &out.Used + *out = make(corev1.ResourceList, len(*in)) + for key, val := range *in { + (*out)[key] = val.DeepCopy() + } + } + if in.Reserved != nil { + in, out := &in.Reserved, &out.Reserved + *out = make(corev1.ResourceList, len(*in)) + for key, val := range *in { + (*out)[key] = val.DeepCopy() + } + } + if in.Allocated != nil { + in, out := &in.Allocated, &out.Allocated + *out = make(corev1.ResourceList, len(*in)) + for key, val := range *in { + (*out)[key] = val.DeepCopy() + } + } + if in.Reservations != nil { + in, out := &in.Reservations, &out.Reservations + *out = make([]QuantityLedgerResourceQuotaReservation, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new QuantityLedgerResourceQuotaStatus. +func (in *QuantityLedgerResourceQuotaStatus) DeepCopy() *QuantityLedgerResourceQuotaStatus { + if in == nil { + return nil + } + out := new(QuantityLedgerResourceQuotaStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *QuantityLedgerSpec) DeepCopyInto(out *QuantityLedgerSpec) { *out = *in @@ -1041,6 +1321,11 @@ func (in *QuantityLedgerStatus) DeepCopyInto(out *QuantityLedgerStatus) { } } out.Allocated = in.Allocated.DeepCopy() + if in.ResourceQuota != nil { + in, out := &in.ResourceQuota, &out.ResourceQuota + *out = new(QuantityLedgerResourceQuotaStatus) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new QuantityLedgerStatus. diff --git a/charts/capsule/README.md b/charts/capsule/README.md index cbc8a956..f4f1a0c9 100644 --- a/charts/capsule/README.md +++ b/charts/capsule/README.md @@ -298,6 +298,15 @@ The following Values have changed key or Value: | webhooks.hooks.globalcustomquotas.namespaceSelector | object | `{}` | [NamespaceSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-namespaceselector) | | webhooks.hooks.globalcustomquotas.objectSelector | object | `{}` | [ObjectSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-objectselector) | | webhooks.hooks.globalcustomquotas.rules | list | `[{"apiGroups":["capsule.clastix.io"],"apiVersions":["v1beta2"],"operations":["CREATE","UPDATE","DELETE"],"resources":["globalcustomquotas"],"scope":"Cluster"}]` | [Rules](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-rules) | +| webhooks.hooks.globalresourcequotas | object | `{"enabled":true,"failurePolicy":"Fail","matchConditions":[],"matchPolicy":"Equivalent","namespaceSelector":{"matchExpressions":[{"key":"capsule.clastix.io/tenant","operator":"Exists"}]},"objectSelector":{},"opts":{},"rules":[{"apiGroups":["*"],"apiVersions":["*"],"operations":["CREATE","UPDATE"],"resources":["*/*"],"scope":"Namespaced"},{"apiGroups":["capsule.clastix.io"],"apiVersions":["v1beta2"],"operations":["CREATE","UPDATE"],"resources":["globalresourcequotas"],"scope":"Cluster"}]}` | Webhook for admission-time GlobalResourceQuota calculations | +| webhooks.hooks.globalresourcequotas.enabled | bool | `true` | Enable the Hook | +| webhooks.hooks.globalresourcequotas.failurePolicy | string | `"Fail"` | [FailurePolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#failure-policy) | +| webhooks.hooks.globalresourcequotas.matchConditions | list | `[]` | [MatchConditions](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) | +| webhooks.hooks.globalresourcequotas.matchPolicy | string | `"Equivalent"` | [MatchPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) | +| webhooks.hooks.globalresourcequotas.namespaceSelector | object | `{"matchExpressions":[{"key":"capsule.clastix.io/tenant","operator":"Exists"}]}` | [NamespaceSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-namespaceselector) | +| webhooks.hooks.globalresourcequotas.objectSelector | object | `{}` | [ObjectSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-objectselector) | +| webhooks.hooks.globalresourcequotas.opts | object | `{}` | Capsule Hook Options | +| webhooks.hooks.globalresourcequotas.rules | list | `[{"apiGroups":["*"],"apiVersions":["*"],"operations":["CREATE","UPDATE"],"resources":["*/*"],"scope":"Namespaced"},{"apiGroups":["capsule.clastix.io"],"apiVersions":["v1beta2"],"operations":["CREATE","UPDATE"],"resources":["globalresourcequotas"],"scope":"Cluster"}]` | [Rules](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-rules) | | webhooks.hooks.ingresses.enabled | bool | `true` | Enable the Hook | | webhooks.hooks.ingresses.failurePolicy | string | `"Fail"` | [FailurePolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#failure-policy) | | webhooks.hooks.ingresses.matchConditions | list | `[]` | [MatchConditions](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) | diff --git a/charts/capsule/crds/capsule.clastix.io_globalresourcequotas.yaml b/charts/capsule/crds/capsule.clastix.io_globalresourcequotas.yaml new file mode 100644 index 00000000..93b18c8d --- /dev/null +++ b/charts/capsule/crds/capsule.clastix.io_globalresourcequotas.yaml @@ -0,0 +1,318 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.21.0 + name: globalresourcequotas.capsule.clastix.io +spec: + group: capsule.clastix.io + names: + kind: GlobalResourceQuota + listKind: GlobalResourceQuotaList + plural: globalresourcequotas + shortNames: + - globalquota + - grq + singular: globalresourcequota + scope: Cluster + versions: + - additionalPrinterColumns: + - description: Selected namespaces + jsonPath: .status.namespaceCount + name: Namespaces + type: integer + - description: Reconcile status + jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - description: Reconcile message + jsonPath: .status.conditions[?(@.type=="Ready")].message + name: Status + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + GlobalResourceQuotaSpec defines a native ResourceQuota shared by every + namespace matched by any namespace selector. + properties: + namespaceSelectors: + description: |- + NamespaceSelectors select the namespaces that share this quota. + Selectors are ORed; requirements within one selector are ANDed. An empty + label selector matches all namespaces. + items: + description: Selector for resources and their labels or selecting + origin namespaces + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + type: array + quota: + description: |- + Quota is the native Kubernetes ResourceQuota specification enforced + across the selected namespaces. + properties: + hard: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + hard is the set of desired hard limits for each named resource. + More info: https://kubernetes.io/docs/concepts/policy/resource-quotas/ + type: object + scopeSelector: + description: |- + scopeSelector is also a collection of filters like scopes that must match each object tracked by a quota + but expressed using ScopeSelectorOperator in combination with possible values. + For a resource to match, both scopes AND scopeSelector (if specified in spec), must be matched. + properties: + matchExpressions: + description: A list of scope selector requirements by scope + of the resources. + items: + description: |- + A scoped-resource selector requirement is a selector that contains values, a scope name, and an operator + that relates the scope name and values. + properties: + operator: + description: |- + Represents a scope's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. + type: string + scopeName: + description: The name of the scope that the selector + applies to. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - operator + - scopeName + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + scopes: + description: |- + A collection of filters that must match each object tracked by a quota. + If not specified, the quota matches all objects. + items: + description: A ResourceQuotaScope defines a filter that must + match each object tracked by a quota + type: string + type: array + x-kubernetes-list-type: atomic + type: object + required: + - quota + type: object + status: + properties: + conditions: + description: Conditions report reconciliation and admission readiness. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + namespaceCount: + default: 0 + description: NamespaceSize is the number of selected namespaces. + type: integer + namespaceUsage: + additionalProperties: + properties: + used: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Used is the usage observed in this namespace. + type: object + type: object + description: NamespaceUsage contains observed quota usage per selected + namespace. + type: object + namespaces: + description: Namespaces is the ordered set of selected namespace names. + items: + type: string + type: array + observedGeneration: + description: ObservedGeneration is the most recent generation observed + by the controller. + format: int64 + type: integer + total: + description: Total contains aggregate quota usage across all selected + namespaces. + properties: + available: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Available is max(Hard-Used, 0). + type: object + hard: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Hard is the configured shared limit. + type: object + used: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Used is the usage observed across the relevant namespace + set. + type: object + type: object + required: + - conditions + - total + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/capsule/crds/capsule.clastix.io_quantityledgers.yaml b/charts/capsule/crds/capsule.clastix.io_quantityledgers.yaml index b3c6963b..852df888 100644 --- a/charts/capsule/crds/capsule.clastix.io_quantityledgers.yaml +++ b/charts/capsule/crds/capsule.clastix.io_quantityledgers.yaml @@ -303,6 +303,135 @@ spec: Controllers/webhooks should treat this as derived data from Reservations. pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true + resourceQuota: + description: |- + ResourceQuota contains coordination state for GlobalResourceQuota. + It is unset for CustomQuota and GlobalCustomQuota ledgers. + properties: + allocated: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Allocated is Used plus all active reservations. + type: object + initialized: + description: |- + Initialized is true after every selected namespace has reported + ResourceQuota status for this quota. + type: boolean + namespaces: + description: Namespaces is the selected namespace set represented + by Used. + items: + type: string + type: array + observedGeneration: + description: |- + ObservedGeneration is the GlobalResourceQuota generation represented by + this ledger state. + format: int64 + type: integer + reservations: + description: Reservations contains inflight admission operations. + items: + description: |- + QuantityLedgerResourceQuotaReservation is an atomic reservation against all + resources tracked by a GlobalResourceQuota. + properties: + createdAt: + format: date-time + type: string + delta: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Delta is the positive amount held while ResourceQuota + status catches up. + type: object + expiresAt: + format: date-time + type: string + id: + description: Unique reservation identifier. + minLength: 1 + type: string + objectRef: + description: Object that this reservation is intended to + create/update. + properties: + apiGroup: + description: APIGroup of the tracked object. + type: string + apiVersion: + description: APIVersion of the tracked object, for example + "v1". + minLength: 1 + type: string + kind: + description: Kind of the tracked object, for example + "Pod". + minLength: 1 + type: string + name: + description: Name of the tracked object. + type: string + namespace: + description: Namespace of the tracked object. + type: string + uid: + description: UID of the tracked object. + type: string + required: + - apiVersion + - kind + type: object + updatedAt: + format: date-time + type: string + usage: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Usage is the calculated usage of the admitted + object. + type: object + required: + - createdAt + - id + - objectRef + - updatedAt + type: object + type: array + reserved: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Reserved is derived from Reservations. + type: object + used: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Used is the usage observed from ResourceQuota status across all selected + namespaces. + type: object + type: object type: object type: object served: true diff --git a/charts/capsule/crds/capsule.clastix.io_rulestatuses.yaml b/charts/capsule/crds/capsule.clastix.io_rulestatuses.yaml index 5fa37e96..c05b8a0a 100644 --- a/charts/capsule/crds/capsule.clastix.io_rulestatuses.yaml +++ b/charts/capsule/crds/capsule.clastix.io_rulestatuses.yaml @@ -476,6 +476,95 @@ spec: type: array type: object type: object + quota: + description: |- + Quota contains native Kubernetes ResourceQuota specifications shared by + all namespaces selected by this rule. Unlike Enforce, quota accounting is + independent of the request audience. + items: + description: |- + ResourceQuotaRule defines a named ResourceQuota specification generated by a + Tenant rule. Name is the durable identity of the generated + GlobalResourceQuota and must be unique across all rules of a Tenant. + properties: + hard: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + hard is the set of desired hard limits for each named resource. + More info: https://kubernetes.io/docs/concepts/policy/resource-quotas/ + type: object + name: + description: |- + Name is the stable identity of this quota within the Tenant. Changing the + name replaces the generated GlobalResourceQuota; changing the quota or its + namespace selector updates the existing object. + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + scopeSelector: + description: |- + scopeSelector is also a collection of filters like scopes that must match each object tracked by a quota + but expressed using ScopeSelectorOperator in combination with possible values. + For a resource to match, both scopes AND scopeSelector (if specified in spec), must be matched. + properties: + matchExpressions: + description: A list of scope selector requirements by + scope of the resources. + items: + description: |- + A scoped-resource selector requirement is a selector that contains values, a scope name, and an operator + that relates the scope name and values. + properties: + operator: + description: |- + Represents a scope's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. + type: string + scopeName: + description: The name of the scope that the selector + applies to. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - operator + - scopeName + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + scopes: + description: |- + A collection of filters that must match each object tracked by a quota. + If not specified, the quota matches all objects. + items: + description: A ResourceQuotaScope defines a filter that + must match each object tracked by a quota + type: string + type: array + x-kubernetes-list-type: atomic + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map type: object type: array status: @@ -975,6 +1064,95 @@ spec: type: array type: object type: object + quota: + description: |- + Quota contains native Kubernetes ResourceQuota specifications shared by + all namespaces selected by this rule. Unlike Enforce, quota accounting is + independent of the request audience. + items: + description: |- + ResourceQuotaRule defines a named ResourceQuota specification generated by a + Tenant rule. Name is the durable identity of the generated + GlobalResourceQuota and must be unique across all rules of a Tenant. + properties: + hard: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + hard is the set of desired hard limits for each named resource. + More info: https://kubernetes.io/docs/concepts/policy/resource-quotas/ + type: object + name: + description: |- + Name is the stable identity of this quota within the Tenant. Changing the + name replaces the generated GlobalResourceQuota; changing the quota or its + namespace selector updates the existing object. + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + scopeSelector: + description: |- + scopeSelector is also a collection of filters like scopes that must match each object tracked by a quota + but expressed using ScopeSelectorOperator in combination with possible values. + For a resource to match, both scopes AND scopeSelector (if specified in spec), must be matched. + properties: + matchExpressions: + description: A list of scope selector requirements by + scope of the resources. + items: + description: |- + A scoped-resource selector requirement is a selector that contains values, a scope name, and an operator + that relates the scope name and values. + properties: + operator: + description: |- + Represents a scope's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. + type: string + scopeName: + description: The name of the scope that the selector + applies to. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - operator + - scopeName + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + scopes: + description: |- + A collection of filters that must match each object tracked by a quota. + If not specified, the quota matches all objects. + items: + description: A ResourceQuotaScope defines a filter that + must match each object tracked by a quota + type: string + type: array + x-kubernetes-list-type: atomic + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map type: object rules: description: |- @@ -1410,6 +1588,95 @@ spec: type: array type: object type: object + quota: + description: |- + Quota contains native Kubernetes ResourceQuota specifications shared by + all namespaces selected by this rule. Unlike Enforce, quota accounting is + independent of the request audience. + items: + description: |- + ResourceQuotaRule defines a named ResourceQuota specification generated by a + Tenant rule. Name is the durable identity of the generated + GlobalResourceQuota and must be unique across all rules of a Tenant. + properties: + hard: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + hard is the set of desired hard limits for each named resource. + More info: https://kubernetes.io/docs/concepts/policy/resource-quotas/ + type: object + name: + description: |- + Name is the stable identity of this quota within the Tenant. Changing the + name replaces the generated GlobalResourceQuota; changing the quota or its + namespace selector updates the existing object. + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + scopeSelector: + description: |- + scopeSelector is also a collection of filters like scopes that must match each object tracked by a quota + but expressed using ScopeSelectorOperator in combination with possible values. + For a resource to match, both scopes AND scopeSelector (if specified in spec), must be matched. + properties: + matchExpressions: + description: A list of scope selector requirements + by scope of the resources. + items: + description: |- + A scoped-resource selector requirement is a selector that contains values, a scope name, and an operator + that relates the scope name and values. + properties: + operator: + description: |- + Represents a scope's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. + type: string + scopeName: + description: The name of the scope that the + selector applies to. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - operator + - scopeName + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + scopes: + description: |- + A collection of filters that must match each object tracked by a quota. + If not specified, the quota matches all objects. + items: + description: A ResourceQuotaScope defines a filter that + must match each object tracked by a quota + type: string + type: array + x-kubernetes-list-type: atomic + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map type: object type: array required: diff --git a/charts/capsule/crds/capsule.clastix.io_tenants.yaml b/charts/capsule/crds/capsule.clastix.io_tenants.yaml index f5fc5405..e43528cd 100644 --- a/charts/capsule/crds/capsule.clastix.io_tenants.yaml +++ b/charts/capsule/crds/capsule.clastix.io_tenants.yaml @@ -3097,6 +3097,95 @@ spec: type: object type: array type: object + quota: + description: |- + Quota contains native Kubernetes ResourceQuota specifications shared by + all namespaces selected by this rule. Unlike Enforce, quota accounting is + independent of the request audience. + items: + description: |- + ResourceQuotaRule defines a named ResourceQuota specification generated by a + Tenant rule. Name is the durable identity of the generated + GlobalResourceQuota and must be unique across all rules of a Tenant. + properties: + hard: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + hard is the set of desired hard limits for each named resource. + More info: https://kubernetes.io/docs/concepts/policy/resource-quotas/ + type: object + name: + description: |- + Name is the stable identity of this quota within the Tenant. Changing the + name replaces the generated GlobalResourceQuota; changing the quota or its + namespace selector updates the existing object. + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + scopeSelector: + description: |- + scopeSelector is also a collection of filters like scopes that must match each object tracked by a quota + but expressed using ScopeSelectorOperator in combination with possible values. + For a resource to match, both scopes AND scopeSelector (if specified in spec), must be matched. + properties: + matchExpressions: + description: A list of scope selector requirements + by scope of the resources. + items: + description: |- + A scoped-resource selector requirement is a selector that contains values, a scope name, and an operator + that relates the scope name and values. + properties: + operator: + description: |- + Represents a scope's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. + type: string + scopeName: + description: The name of the scope that the + selector applies to. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - operator + - scopeName + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + scopes: + description: |- + A collection of filters that must match each object tracked by a quota. + If not specified, the quota matches all objects. + items: + description: A ResourceQuotaScope defines a filter that + must match each object tracked by a quota + type: string + type: array + x-kubernetes-list-type: atomic + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map type: object type: array runtimeClasses: diff --git a/charts/capsule/templates/configuration.yaml b/charts/capsule/templates/configuration.yaml index bf9dcdd5..297cc28d 100644 --- a/charts/capsule/templates/configuration.yaml +++ b/charts/capsule/templates/configuration.yaml @@ -1033,6 +1033,43 @@ spec: {{- toYaml .rules | nindent 10 }} sideEffects: None timeoutSeconds: {{ $.Values.webhooks.validatingWebhooksTimeoutSeconds }} + {{- end }} + {{- end }} + {{- with .Values.webhooks.hooks.globalresourcequotas }} + {{- if .enabled }} + {{- $any = true }} + - name: calculation.global-resource-quotas.validating.projectcapsule.dev + {{- with .opts }} + opts: + {{- toYaml . | nindent 10 }} + {{- end }} + admissionReviewVersions: + - v1 + - v1beta1 + path: "/global-resource-quotas/calculations" + failurePolicy: {{ .failurePolicy }} + matchPolicy: {{ .matchPolicy }} + {{- with .namespaceSelector }} + namespaceSelector: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .objectSelector }} + objectSelector: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- if or .matchConditions $.Values.webhooks.matchConditions }} + matchConditions: + {{- end }} + {{- with .matchConditions }} + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with $.Values.webhooks.matchConditions }} + {{- toYaml . | nindent 10 }} + {{- end }} + rules: + {{- toYaml .rules | nindent 10 }} + sideEffects: NoneOnDryRun + timeoutSeconds: {{ $.Values.webhooks.validatingWebhooksTimeoutSeconds }} {{- end }} {{- end }} {{- with .Values.webhooks.hooks.calculations }} diff --git a/charts/capsule/templates/hooks/crd-lifecycle/rbac.yaml b/charts/capsule/templates/hooks/crd-lifecycle/rbac.yaml index 524f964c..ea8d72e6 100644 --- a/charts/capsule/templates/hooks/crd-lifecycle/rbac.yaml +++ b/charts/capsule/templates/hooks/crd-lifecycle/rbac.yaml @@ -37,6 +37,7 @@ rules: - rulestatuses.capsule.clastix.io - customquotas.capsule.clastix.io - globalcustomquotas.capsule.clastix.io + - globalresourcequotas.capsule.clastix.io - quantityledgers.capsule.clastix.io verbs: - create diff --git a/charts/capsule/templates/rbac.yaml b/charts/capsule/templates/rbac.yaml index 7d0845f9..a7b1d436 100644 --- a/charts/capsule/templates/rbac.yaml +++ b/charts/capsule/templates/rbac.yaml @@ -174,6 +174,8 @@ rules: - customquotas/status - globalcustomquotas - globalcustomquotas/status + - globalresourcequotas + - globalresourcequotas/status - quantityledgers - quantityledgers/status verbs: diff --git a/charts/capsule/values.schema.json b/charts/capsule/values.schema.json index 0613f0db..6e5b32a6 100644 --- a/charts/capsule/values.schema.json +++ b/charts/capsule/values.schema.json @@ -1727,6 +1727,94 @@ } } }, + "globalresourcequotas": { + "description": "Webhook for admission-time GlobalResourceQuota calculations", + "type": "object", + "properties": { + "enabled": { + "description": "Enable the Hook", + "type": "boolean" + }, + "failurePolicy": { + "description": "[FailurePolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#failure-policy)", + "type": "string" + }, + "matchConditions": { + "description": "[MatchConditions](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy)", + "type": "array" + }, + "matchPolicy": { + "description": "[MatchPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy)", + "type": "string" + }, + "namespaceSelector": { + "description": "[NamespaceSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-namespaceselector)", + "type": "object", + "properties": { + "matchExpressions": { + "type": "array", + "items": { + "type": "object", + "properties": { + "key": { + "type": "string" + }, + "operator": { + "type": "string" + } + } + } + } + }, + "additionalProperties": true + }, + "objectSelector": { + "description": "[ObjectSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-objectselector)", + "type": "object", + "additionalProperties": true + }, + "opts": { + "description": "Capsule Hook Options", + "type": "object" + }, + "rules": { + "description": "[Rules](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-rules)", + "type": "array", + "items": { + "type": "object", + "properties": { + "apiGroups": { + "type": "array", + "items": { + "type": "string" + } + }, + "apiVersions": { + "type": "array", + "items": { + "type": "string" + } + }, + "operations": { + "type": "array", + "items": { + "type": "string" + } + }, + "resources": { + "type": "array", + "items": { + "type": "string" + } + }, + "scope": { + "type": "string" + } + } + } + } + } + }, "ingresses": { "type": "object", "properties": { diff --git a/charts/capsule/values.yaml b/charts/capsule/values.yaml index 1270fdee..816523b8 100644 --- a/charts/capsule/values.yaml +++ b/charts/capsule/values.yaml @@ -687,6 +687,42 @@ monitoring: summary: "Critical resource usage in Resourcepool {{ $labels.pool }}" description: "Resource {{ $labels.resource }} in pool {{ $labels.pool }} has exceeded 95% usage for the last 10 minutes." + - name: capsule.globalresourcequotas + rules: + - alert: CapsuleGlobalResourceQuotaNotReady + expr: | + capsule_global_resource_quota_condition{condition="Ready"} != 1 + for: 10m + labels: + severity: warning + app: capsule + component: globalresourcequotas + annotations: + summary: "Capsule GlobalResourceQuota {{ $labels.global_resource_quota }} is not ready" + description: "GlobalResourceQuota {{ $labels.global_resource_quota }} is not in Ready state for the last 10 minutes." + - alert: CapsuleGlobalResourceQuotaHighUsage + expr: | + capsule_global_resource_quota_usage_percentage > 90 + for: 10m + labels: + severity: warning + app: capsule + component: globalresourcequotas + annotations: + summary: "High resource usage in GlobalResourceQuota {{ $labels.global_resource_quota }}" + description: "Resource {{ $labels.resource }} in GlobalResourceQuota {{ $labels.global_resource_quota }} is at {{ $value }}% usage for the last 10 minutes." + - alert: CapsuleGlobalResourceQuotaCriticalUsage + expr: | + capsule_global_resource_quota_usage_percentage > 95 + for: 10m + labels: + severity: critical + app: capsule + component: globalresourcequotas + annotations: + summary: "Critical resource usage in GlobalResourceQuota {{ $labels.global_resource_quota }}" + description: "Resource {{ $labels.resource }} in GlobalResourceQuota {{ $labels.global_resource_quota }} has exceeded 95% usage for the last 10 minutes." + - name: capsule.replications rules: - alert: CapsuleGlobalTenantResourceNotReady @@ -934,6 +970,52 @@ webhooks: - globalcustomquotas scope: 'Cluster' + # -- Webhook for admission-time GlobalResourceQuota calculations + globalresourcequotas: + # -- Enable the Hook + enabled: true + # -- Capsule Hook Options + opts: {} + # -- [FailurePolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#failure-policy) + failurePolicy: Fail + # -- [MatchPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) + matchPolicy: Equivalent + # @schema type: object + # @schema additionalProperties: true + # -- [ObjectSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-objectselector) + objectSelector: {} + # @schema type: object + # @schema additionalProperties: true + # -- [NamespaceSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-namespaceselector) + namespaceSelector: + matchExpressions: + - key: capsule.clastix.io/tenant + operator: Exists + # -- [MatchConditions](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) + matchConditions: [] + # -- [Rules](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-rules) + rules: + - apiGroups: + - '*' + apiVersions: + - '*' + operations: + - CREATE + - UPDATE + resources: + - '*/*' + scope: Namespaced + - apiGroups: + - capsule.clastix.io + apiVersions: + - v1beta2 + operations: + - CREATE + - UPDATE + resources: + - globalresourcequotas + scope: Cluster + # -- Webhook for Custom Quota Calculations ([Read More](https://projectcapsule.dev/docs/resource-management/customquotas/#admission)) calculations: # -- Enable the Hook diff --git a/cmd/controller/main.go b/cmd/controller/main.go index adfb972c..6cdc8dc4 100644 --- a/cmd/controller/main.go +++ b/cmd/controller/main.go @@ -46,6 +46,7 @@ import ( cacheinvalidator "github.com/projectcapsule/capsule/internal/controllers/cfg/invalidator" configcontroller "github.com/projectcapsule/capsule/internal/controllers/cfg/status" customquotacontroller "github.com/projectcapsule/capsule/internal/controllers/customquotas" + globalresourcequotacontroller "github.com/projectcapsule/capsule/internal/controllers/globalresourcequotas" podlabelscontroller "github.com/projectcapsule/capsule/internal/controllers/pod" "github.com/projectcapsule/capsule/internal/controllers/pv" rbaccontroller "github.com/projectcapsule/capsule/internal/controllers/rbac" @@ -66,6 +67,7 @@ import ( "github.com/projectcapsule/capsule/internal/webhook/dra" "github.com/projectcapsule/capsule/internal/webhook/gateway" "github.com/projectcapsule/capsule/internal/webhook/generic" + globalresourcequotavalidation "github.com/projectcapsule/capsule/internal/webhook/globalresourcequota" "github.com/projectcapsule/capsule/internal/webhook/ingress" namespacemutation "github.com/projectcapsule/capsule/internal/webhook/namespace/mutation" namespacevalidation "github.com/projectcapsule/capsule/internal/webhook/namespace/validation" @@ -804,6 +806,7 @@ func main() { jsonPathCache, celCache, )), + route.GlobalResourceQuotaCalculation(globalresourcequotavalidation.Handler()), route.CalculationCustomQuotas( customquotavalidation.ObjectCalculationHandler( targetsCache, @@ -969,6 +972,16 @@ func main() { os.Exit(1) } + if err := globalresourcequotacontroller.Add( + ctrl.Log.WithName("capsule.ctrl").WithName("globalresourcequotas"), + manager, + manager.GetEventRecorder("globalresourcequotas-ctrl"), + controllerConfig, + ); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "globalresourcequotas") + os.Exit(1) + } + if err = customquotacontroller.Add(ctrl.Log.WithName("controllers").WithName("CustomQuotas"), manager, manager.GetEventRecorder("customquotas-ctrl"), diff --git a/e2e/global_resource_quota_test.go b/e2e/global_resource_quota_test.go new file mode 100644 index 00000000..629e9a05 --- /dev/null +++ b/e2e/global_resource_quota_test.go @@ -0,0 +1,573 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package e2e + +import ( + "context" + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" + "github.com/projectcapsule/capsule/pkg/runtime/selectors" +) + +var _ = Describe("GlobalResourceQuota", Ordered, Label("globalresourcequota", "resourcequota", "ledger", "skip-on-openshift"), func() { + const ( + tenantAName = "e2e-global-resource-quota-a" + tenantBName = "e2e-global-resource-quota-b" + + computeQuotaName = "e2e-global-resource-quota-compute" + serviceQuotaName = "e2e-global-resource-quota-services" + countQuotaName = "e2e-global-resource-quota-counts" + ephemeralQuotaName = "e2e-global-resource-quota-ephemeral" + + computeA = "e2e-global-quota-compute-a" + computeB = "e2e-global-quota-compute-b" + serviceA = "e2e-global-quota-services-a" + serviceB = "e2e-global-quota-services-b" + countA = "e2e-global-quota-counts-a" + countB = "e2e-global-quota-counts-b" + ephemeralA = "e2e-global-quota-ephemeral-a" + ephemeralB = "e2e-global-quota-ephemeral-b" + + computeSelector = "e2e.projectcapsule.dev/global-quota-compute" + serviceSelector = "e2e.projectcapsule.dev/global-quota-services" + countSelector = "e2e.projectcapsule.dev/global-quota-counts" + ephemeralSelector = "e2e.projectcapsule.dev/global-quota-ephemeral" + ) + + ctx := context.Background() + tenantAOwner := rbac.UserSpec{Name: tenantAName, Kind: rbac.OwnerKind("User")} + tenantBOwner := rbac.UserSpec{Name: tenantBName, Kind: rbac.OwnerKind("User")} + tenantA := &capsulev1beta2.Tenant{ + ObjectMeta: metav1.ObjectMeta{ + Name: tenantAName, + Labels: map[string]string{"env": "e2e"}, + }, + Spec: capsulev1beta2.TenantSpec{ + Owners: rbac.OwnerListSpec{{ + CoreOwnerSpec: rbac.CoreOwnerSpec{UserSpec: tenantAOwner}, + }}, + }, + } + tenantB := &capsulev1beta2.Tenant{ + ObjectMeta: metav1.ObjectMeta{ + Name: tenantBName, + Labels: map[string]string{"env": "e2e"}, + }, + Spec: capsulev1beta2.TenantSpec{ + Owners: rbac.OwnerListSpec{{ + CoreOwnerSpec: rbac.CoreOwnerSpec{UserSpec: tenantBOwner}, + }}, + }, + } + tenants := []*capsulev1beta2.Tenant{tenantA, tenantB} + computeHard := corev1.ResourceList{ + corev1.ResourceRequestsCPU: resource.MustParse("1"), + corev1.ResourceRequestsMemory: resource.MustParse("1Gi"), + corev1.ResourceLimitsCPU: resource.MustParse("2"), + corev1.ResourceLimitsMemory: resource.MustParse("2Gi"), + } + serviceHard := corev1.ResourceList{ + corev1.ResourceServices: resource.MustParse("5"), + } + countHard := corev1.ResourceList{ + corev1.ResourceSecrets: resource.MustParse("2"), + corev1.ResourceName("count/deployments.apps"): resource.MustParse("2"), + corev1.ResourceName("count/horizontalpodautoscalers.autoscaling"): resource.MustParse("1"), + } + ephemeralHard := corev1.ResourceList{ + corev1.ResourceEphemeralStorage: resource.MustParse("1Gi"), + corev1.ResourceRequestsEphemeralStorage: resource.MustParse("1Gi"), + corev1.ResourceLimitsEphemeralStorage: resource.MustParse("2Gi"), + } + computeQuota := &capsulev1beta2.GlobalResourceQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: computeQuotaName, + Labels: map[string]string{"env": "e2e"}, + }, + Spec: capsulev1beta2.GlobalResourceQuotaSpec{ + NamespaceSelectors: []selectors.NamespaceSelector{{ + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{computeSelector: "true"}, + }, + }}, + Quota: corev1.ResourceQuotaSpec{Hard: computeHard}, + }, + } + serviceQuota := &capsulev1beta2.GlobalResourceQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: serviceQuotaName, + Labels: map[string]string{"env": "e2e"}, + }, + Spec: capsulev1beta2.GlobalResourceQuotaSpec{ + NamespaceSelectors: []selectors.NamespaceSelector{{ + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{serviceSelector: "true"}, + }, + }}, + Quota: corev1.ResourceQuotaSpec{Hard: serviceHard}, + }, + } + countQuota := &capsulev1beta2.GlobalResourceQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: countQuotaName, + Labels: map[string]string{"env": "e2e"}, + }, + Spec: capsulev1beta2.GlobalResourceQuotaSpec{ + NamespaceSelectors: []selectors.NamespaceSelector{{ + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{countSelector: "true"}, + }, + }}, + Quota: corev1.ResourceQuotaSpec{Hard: countHard}, + }, + } + ephemeralQuota := &capsulev1beta2.GlobalResourceQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: ephemeralQuotaName, + Labels: map[string]string{"env": "e2e"}, + }, + Spec: capsulev1beta2.GlobalResourceQuotaSpec{ + NamespaceSelectors: []selectors.NamespaceSelector{{ + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ephemeralSelector: "true"}, + }, + }}, + Quota: corev1.ResourceQuotaSpec{Hard: ephemeralHard}, + }, + } + quotaCases := []struct { + quota *capsulev1beta2.GlobalResourceQuota + namespaces []string + hard corev1.ResourceList + }{ + {quota: computeQuota, namespaces: []string{computeA, computeB}, hard: computeHard}, + {quota: serviceQuota, namespaces: []string{serviceA, serviceB}, hard: serviceHard}, + {quota: countQuota, namespaces: []string{countA, countB}, hard: countHard}, + {quota: ephemeralQuota, namespaces: []string{ephemeralA, ephemeralB}, hard: ephemeralHard}, + } + namespaceCases := []struct { + name string + labelKey string + tenant *capsulev1beta2.Tenant + owner rbac.UserSpec + }{ + {name: computeA, labelKey: computeSelector, tenant: tenantA, owner: tenantAOwner}, + {name: computeB, labelKey: computeSelector, tenant: tenantB, owner: tenantBOwner}, + {name: serviceA, labelKey: serviceSelector, tenant: tenantA, owner: tenantAOwner}, + {name: serviceB, labelKey: serviceSelector, tenant: tenantB, owner: tenantBOwner}, + {name: countA, labelKey: countSelector, tenant: tenantA, owner: tenantAOwner}, + {name: countB, labelKey: countSelector, tenant: tenantB, owner: tenantBOwner}, + {name: ephemeralA, labelKey: ephemeralSelector, tenant: tenantA, owner: tenantAOwner}, + {name: ephemeralB, labelKey: ephemeralSelector, tenant: tenantB, owner: tenantBOwner}, + } + + BeforeAll(func() { + for _, tenant := range tenants { + EventuallyCreation(func() error { + tenant.ResourceVersion = "" + + return k8sClient.Create(ctx, tenant) + }).Should(Succeed()) + TenantReadyTrue(tenant) + } + + for _, namespace := range namespaceCases { + ns := NewNamespace(namespace.name, map[string]string{ + meta.TenantLabel: namespace.tenant.Name, + namespace.labelKey: "true", + }) + NamespaceCreation(ns, namespace.owner, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(namespace.tenant, ns).Should(Succeed()) + } + + for quotaIndex := range quotaCases { + quotaCase := "aCases[quotaIndex] + EventuallyCreation(func() error { + quotaCase.quota.ResourceVersion = "" + + return k8sClient.Create(ctx, quotaCase.quota) + }).Should(Succeed()) + + Eventually(func(g Gomega) { + current := &capsulev1beta2.GlobalResourceQuota{} + g.Expect(k8sClient.Get( + ctx, + types.NamespacedName{Name: quotaCase.quota.Name}, + current, + )).To(Succeed()) + g.Expect(current.Status.ObservedGeneration).To(Equal(current.Generation)) + g.Expect(current.Status.Namespaces).To(ConsistOf(quotaCase.namespaces)) + g.Expect(current.Status.NamespaceSize).To(Equal(uint(len(quotaCase.namespaces)))) + + ready := current.Status.Conditions.GetConditionByType(meta.ReadyCondition) + g.Expect(ready).NotTo(BeNil()) + g.Expect(ready.Status).To(Equal(metav1.ConditionTrue)) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + + current := &capsulev1beta2.GlobalResourceQuota{} + Expect(k8sClient.Get( + ctx, + types.NamespacedName{Name: quotaCase.quota.Name}, + current, + )).To(Succeed()) + quotaCase.quota = current + + for _, namespace := range quotaCase.namespaces { + Eventually(func(g Gomega) { + nativeQuota := &corev1.ResourceQuota{} + g.Expect(k8sClient.Get(ctx, types.NamespacedName{ + Namespace: namespace, + Name: current.GetResourceQuotaName(), + }, nativeQuota)).To(Succeed()) + expectResourceListEqual(g, nativeQuota.Spec.Hard, quotaCase.hard) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + } + + Eventually(func(g Gomega) { + ledger := &capsulev1beta2.QuantityLedger{} + g.Expect(k8sClient.Get(ctx, types.NamespacedName{ + Namespace: ControllerNamespace, + Name: current.GetLedgerName(), + }, ledger)).To(Succeed()) + g.Expect(ledger.Status.ResourceQuota).NotTo(BeNil()) + g.Expect(ledger.Status.ResourceQuota.Initialized).To(BeTrue()) + g.Expect(ledger.Status.ResourceQuota.ObservedGeneration).To(Equal(current.Generation)) + g.Expect(ledger.Status.ResourceQuota.Namespaces).To(ConsistOf(quotaCase.namespaces)) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + } + }) + + AfterAll(func() { + for _, quotaCase := range quotaCases { + EventuallyDeletion(quotaCase.quota) + } + for _, namespace := range namespaceCases { + ForceDeleteNamespace(ctx, namespace.name) + } + for _, tenant := range tenants { + EventuallyDeletion(tenant) + } + }) + + It("reports aggregate and per-namespace quota state", func() { + for _, quotaCase := range quotaCases { + current := &capsulev1beta2.GlobalResourceQuota{} + Expect(k8sClient.Get( + ctx, + types.NamespacedName{Name: quotaCase.quota.Name}, + current, + )).To(Succeed()) + + expectResourceListEqual(Default, current.Status.Total.Hard, quotaCase.hard) + Expect(current.Status.NamespaceUsage).To(HaveLen(len(quotaCase.namespaces))) + for _, namespace := range quotaCase.namespaces { + Expect(current.Status.NamespaceUsage).To(HaveKey(namespace)) + } + } + }) + + It("accounts Pod-level resources on Pods generated by Deployments", func() { + cs := clusterAdminClient() + makeDeployment := func(namespace, name string) *appsv1.Deployment { + deployment := MakeDeployment(namespace, name, 1, nil, "") + deployment.Spec.Template.Spec.Resources = &corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("600m"), + corev1.ResourceMemory: resource.MustParse("512Mi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1"), + corev1.ResourceMemory: resource.MustParse("1Gi"), + }, + } + + return deployment + } + + firstName := "global-quota-pod-level-first" + _, err := cs.AppsV1().Deployments(computeA).Create( + ctx, + makeDeployment(computeA, firstName), + metav1.CreateOptions{}, + ) + Expect(err).NotTo(HaveOccurred()) + ExpectPodsForDeployment(ctx, computeA, firstName, 1) + + secondName := "global-quota-pod-level-second" + _, err = cs.AppsV1().Deployments(computeB).Create( + ctx, + makeDeployment(computeB, secondName), + metav1.CreateOptions{}, + ) + Expect(err).NotTo(HaveOccurred()) + + Eventually(func(g Gomega) { + failure, getErr := replicaSetFailureForDeployment(ctx, computeB, secondName) + g.Expect(getErr).NotTo(HaveOccurred()) + g.Expect(failure).NotTo(BeNil()) + g.Expect(failure.Status).To(Equal(corev1.ConditionTrue)) + g.Expect(failure.Message).To(ContainSubstring("exceeds GlobalResourceQuota")) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + ExpectPodsForDeployment(ctx, computeB, secondName, 0) + }) + + It("atomically admits only five of ten concurrent Services", func() { + const total = 10 + + cs := clusterAdminClient() + results := make(chan error, total) + + for index := 0; index < total; index++ { + go func(index int) { + namespace := serviceA + if index%2 == 1 { + namespace = serviceB + } + + service := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: fmt.Sprintf("global-quota-service-%02d", index), + }, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{{Port: 80}}, + }, + } + _, createErr := cs.CoreV1().Services(namespace).Create( + ctx, + service, + metav1.CreateOptions{}, + ) + results <- createErr + }(index) + } + + var succeeded, failed int + for index := 0; index < total; index++ { + if createErr := <-results; createErr == nil { + succeeded++ + } else { + failed++ + Expect(createErr.Error()).To(ContainSubstring("exceeds GlobalResourceQuota")) + } + } + + Expect(succeeded).To(Equal(5)) + Expect(failed).To(Equal(5)) + }) + + It("enforces legacy core object counts across namespaces", func() { + const total = 4 + + cs := clusterAdminClient() + results := make(chan error, total) + + for index := 0; index < total; index++ { + go func(index int) { + namespace := countA + if index%2 == 1 { + namespace = countB + } + + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: fmt.Sprintf("global-quota-secret-%02d", index), + }, + Type: corev1.SecretTypeOpaque, + } + _, createErr := cs.CoreV1().Secrets(namespace).Create( + ctx, + secret, + metav1.CreateOptions{}, + ) + results <- createErr + }(index) + } + + expectConcurrentAdmissions(results, total, 2) + }) + + It("enforces qualified generic object counts across namespaces", func() { + const total = 4 + + cs := clusterAdminClient() + results := make(chan error, total) + + for index := 0; index < total; index++ { + go func(index int) { + namespace := countA + if index%2 == 1 { + namespace = countB + } + + deployment := MakeDeployment( + namespace, + fmt.Sprintf("global-quota-deployment-%02d", index), + 0, + nil, + "", + ) + _, createErr := cs.AppsV1().Deployments(namespace).Create( + ctx, + deployment, + metav1.CreateOptions{}, + ) + results <- createErr + }(index) + } + + expectConcurrentAdmissions(results, total, 2) + }) + + It("enforces API-group-qualified HorizontalPodAutoscaler counts", func() { + cs := clusterAdminClient() + makeHPA := func(namespace, name string) *autoscalingv2.HorizontalPodAutoscaler { + return &autoscalingv2.HorizontalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, + Spec: autoscalingv2.HorizontalPodAutoscalerSpec{ + ScaleTargetRef: autoscalingv2.CrossVersionObjectReference{ + APIVersion: "apps/v1", + Kind: "Deployment", + Name: "unused-target", + }, + MaxReplicas: 3, + }, + } + } + + _, err := cs.AutoscalingV2().HorizontalPodAutoscalers(countA).Create( + ctx, + makeHPA(countA, "global-quota-hpa-first"), + metav1.CreateOptions{}, + ) + Expect(err).NotTo(HaveOccurred()) + + _, err = cs.AutoscalingV2().HorizontalPodAutoscalers(countB).Create( + ctx, + makeHPA(countB, "global-quota-hpa-second"), + metav1.CreateOptions{}, + ) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("exceeds GlobalResourceQuota")) + Expect(err.Error()).To(ContainSubstring( + "count/horizontalpodautoscalers.autoscaling (requested=1, current=1, projected=2, hard=1, exceededBy=1)", + )) + }) + + It("accounts ephemeral-storage requests and limits across namespaces", func() { + cs := clusterAdminClient() + makePod := func(namespace, name string) *corev1.Pod { + pod := MakePod( + namespace, + name, + nil, + nil, + "registry.k8s.io/pause:3.10", + "", + "4Gi", + ) + pod.Spec.Containers[0].Resources = corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceEphemeralStorage: resource.MustParse("600Mi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceEphemeralStorage: resource.MustParse("1Gi"), + }, + } + + return pod + } + + _, err := cs.CoreV1().Pods(ephemeralA).Create( + ctx, + makePod(ephemeralA, "global-quota-ephemeral-first"), + metav1.CreateOptions{}, + ) + Expect(err).NotTo(HaveOccurred()) + + _, err = cs.CoreV1().Pods(ephemeralB).Create( + ctx, + makePod(ephemeralB, "global-quota-ephemeral-second"), + metav1.CreateOptions{}, + ) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("exceeds GlobalResourceQuota")) + Expect(err.Error()).To(ContainSubstring( + "requests.ephemeral-storage (requested=600Mi, current=600Mi, projected=1200Mi, hard=1Gi, exceededBy=176Mi)", + )) + }) +}) + +func expectResourceListEqual(g Gomega, actual, expected corev1.ResourceList) { + g.Expect(actual).To(HaveLen(len(expected))) + for name, expectedQuantity := range expected { + actualQuantity, found := actual[name] + g.Expect(found).To(BeTrue(), "missing resource %s", name) + g.Expect(actualQuantity.Cmp(expectedQuantity)).To(Equal(0), "resource %s", name) + } +} + +func replicaSetFailureForDeployment( + ctx context.Context, + namespace string, + deploymentName string, +) (*appsv1.ReplicaSetCondition, error) { + deployment := &appsv1.Deployment{} + if err := k8sClient.Get(ctx, types.NamespacedName{ + Namespace: namespace, + Name: deploymentName, + }, deployment); err != nil { + return nil, err + } + + replicaSets := &appsv1.ReplicaSetList{} + if err := k8sClient.List(ctx, replicaSets, client.InNamespace(namespace)); err != nil { + return nil, err + } + + for replicaSetIndex := range replicaSets.Items { + replicaSet := &replicaSets.Items[replicaSetIndex] + if !metav1.IsControlledBy(replicaSet, deployment) { + continue + } + + for conditionIndex := range replicaSet.Status.Conditions { + condition := &replicaSet.Status.Conditions[conditionIndex] + if condition.Type == appsv1.ReplicaSetReplicaFailure { + return condition, nil + } + } + } + + return nil, nil +} + +func expectConcurrentAdmissions(results <-chan error, total, expectedSuccess int) { + var succeeded, failed int + for index := 0; index < total; index++ { + if createErr := <-results; createErr == nil { + succeeded++ + } else { + failed++ + Expect(createErr.Error()).To(ContainSubstring("exceeds GlobalResourceQuota")) + } + } + + Expect(succeeded).To(Equal(expectedSuccess)) + Expect(failed).To(Equal(total - expectedSuccess)) +} diff --git a/e2e/replications_tenantresource_test.go b/e2e/replications_tenantresource_test.go index 9a96fd19..118bbebc 100644 --- a/e2e/replications_tenantresource_test.go +++ b/e2e/replications_tenantresource_test.go @@ -30,6 +30,8 @@ var ( resyncPeriod = metav1.Duration{Duration: 10 * time.Second} ) +const tenantResourceTargetLabel = "e2e.projectcapsule.dev/tenantresource-target" + var _ = Describe("TenantResource SSA", Ordered, Label("replications", "namespace", "tenantresource"), Ordered, func() { var ( ctx context.Context @@ -113,10 +115,20 @@ var _ = Describe("TenantResource SSA", Ordered, Label("replications", "namespace TenantReady(tnt, metav1.ConditionTrue, defaultTimeoutInterval) for _, ns := range append(append([]string{}, targetNamespaces...), baseNamespace) { - namespace := NewNamespace(ns, map[string]string{apimeta.TenantLabel: tnt.GetName()}) + labels := map[string]string{apimeta.TenantLabel: tnt.GetName()} + if ns != baseNamespace { + labels[tenantResourceTargetLabel] = "true" + } + + namespace := NewNamespace(ns, labels) NamespaceCreation(namespace, tenantOwner, defaultTimeoutInterval).Should(Succeed()) NamespaceIsPartOfTenant(tnt, namespace).Should(Succeed()) } + + ensureServiceAccount(baseNamespace, "default") + for _, ns := range append(append([]string{}, targetNamespaces...), baseNamespace) { + bindServiceAccountToTenantResourceManager(baseNamespace, "default", ns) + } }) AfterEach(func() { @@ -261,8 +273,6 @@ rules: It("skips applying resources to terminating namespaces and removes them from processedItems", func() { terminatingNamespace := targetNamespaces[2] - releaseNamespace := holdNamespaceTerminating(ctx, terminatingNamespace) - defer releaseNamespace() tr := &capsulev1beta2.TenantResource{ ObjectMeta: metav1.ObjectMeta{ @@ -274,6 +284,9 @@ rules: PruningOnDelete: ptr.To(true), ResyncPeriod: metav1.Duration{Duration: 5 * time.Second}, Resources: []capsulev1beta2.ResourceSpec{{ + NamespaceSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{tenantResourceTargetLabel: "true"}, + }, RawItems: []capsulev1beta2.RawExtension{{ RawExtension: runtime.RawExtension{ Object: &corev1.ConfigMap{ @@ -299,14 +312,64 @@ rules: return k8sClient.Create(ctx, tr) }).Should(Succeed()) - By("verifying non-terminating namespaces still receive the resource") - for _, ns := range targetNamespaces[:2] { + By("waiting for the initial replication to complete") + expectTenantResourceProcessedNamespaces( + baseNamespace, + tr.Name, + "tr-skip-terminating", + targetNamespaces, + ) + + By("establishing the resource in every active namespace") + for _, ns := range targetNamespaces { expectConfigMapData(ns, "tr-skip-terminating", map[string]string{ "mode": "active", }) } + releaseNamespace := holdNamespaceTerminating(ctx, terminatingNamespace) + defer releaseNamespace() + + By("updating the resource after one target namespace starts terminating") + Eventually(func() error { + current := &capsulev1beta2.TenantResource{} + if err := k8sClient.Get(ctx, types.NamespacedName{ + Name: tr.Name, + Namespace: tr.Namespace, + }, current); err != nil { + return err + } + + current.Spec.Resources[0].RawItems[0] = capsulev1beta2.RawExtension{ + RawExtension: runtime.RawExtension{ + Object: &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "ConfigMap", + }, + ObjectMeta: metav1.ObjectMeta{Name: "tr-skip-terminating"}, + Data: map[string]string{"mode": "updated"}, + }, + }, + } + + return k8sClient.Update(ctx, current) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + + By("verifying non-terminating namespaces still receive updates") + for _, ns := range targetNamespaces[:2] { + expectConfigMapData(ns, "tr-skip-terminating", map[string]string{ + "mode": "updated", + }) + } + By("verifying the terminating namespace is skipped") + Eventually(func() error { + return k8sClient.Get(ctx, types.NamespacedName{ + Name: "tr-skip-terminating", + Namespace: terminatingNamespace, + }, &corev1.ConfigMap{}) + }, defaultTimeoutInterval, defaultPollInterval).Should(HaveOccurred()) Consistently(func() error { return k8sClient.Get(ctx, types.NamespacedName{ Name: "tr-skip-terminating", @@ -315,21 +378,12 @@ rules: }, 2*resyncPeriod.Duration, defaultPollInterval).Should(HaveOccurred()) By("verifying the terminating namespace item is not kept in processedItems") - Eventually(func(g Gomega) { - current := &capsulev1beta2.TenantResource{} - - g.Expect(k8sClient.Get(ctx, types.NamespacedName{ - Name: tr.Name, - Namespace: tr.Namespace, - }, current)).To(Succeed()) - - for _, item := range current.Status.ProcessedItems { - g.Expect(item.Name).To(Equal("tr-skip-terminating")) - - g.Expect(item.Namespace).ToNot(Equal(terminatingNamespace)) - g.Expect(item.Status).To(Equal(metav1.ConditionTrue)) - } - }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + expectTenantResourceProcessedNamespaces( + baseNamespace, + tr.Name, + "tr-skip-terminating", + targetNamespaces[:2], + ) }) Context("generators and template context", func() { @@ -1283,19 +1337,13 @@ data: Context("impersonation", func() { It("reflects the resolved service account in status", func() { - tr := newRawConfigMapTenantResource(baseNamespace, "sa-resolution", map[string]string{"mode": "default-controller"}) + tr := newRawConfigMapTenantResource(baseNamespace, "sa-resolution", map[string]string{"mode": "default-service-account"}) tr.Spec.ServiceAccount = nil By("creating the TenantResource without an explicit ServiceAccount") EventuallyCreation(func() error { return k8sClient.Create(ctx, tr) }).Should(Succeed()) - By("defaulting to the controller service account") - expectResolvedServiceAccount(baseNamespace, tr.Name, "capsule", ControllerNamespace) - - By("configuring a tenant default service account") - ModifyCapsuleConfigurationOpts(func(configuration *capsulev1beta2.CapsuleConfiguration) { - configuration.Spec.Impersonation.TenantDefaultServiceAccount = "default" - }) + By("defaulting to the configured tenant service account") expectResolvedServiceAccount(baseNamespace, tr.Name, "default", baseNamespace) By("overriding with an explicit service account on the TenantResource") @@ -1950,6 +1998,40 @@ func expectTenantResourceFailed(namespace, name, contains string) { }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) } +func expectTenantResourceProcessedNamespaces(namespace, name, itemName string, expected []string) { + Eventually(func(g Gomega) { + tr := &capsulev1beta2.TenantResource{} + g.Expect(k8sClient.Get( + context.Background(), + types.NamespacedName{Name: name, Namespace: namespace}, + tr, + )).To(Succeed()) + + ready := tr.Status.Conditions.GetConditionByType(apimeta.ReadyCondition) + g.Expect(ready).NotTo(BeNil(), "TenantResource %s/%s has no Ready condition", namespace, name) + if ready == nil { + return + } + g.Expect(ready.Status).To( + Equal(metav1.ConditionTrue), + "TenantResource %s/%s reconciliation failed: %s", + namespace, + name, + ready.Message, + ) + g.Expect(tr.Status.ObservedGeneration).To(Equal(tr.Generation)) + + processedNamespaces := make([]string, 0, len(tr.Status.ProcessedItems)) + for _, item := range tr.Status.ProcessedItems { + g.Expect(item.Name).To(Equal(itemName)) + g.Expect(item.Status).To(Equal(metav1.ConditionTrue), item.Message) + processedNamespaces = append(processedNamespaces, item.Namespace) + } + + g.Expect(processedNamespaces).To(ConsistOf(expected)) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) +} + func expectResolvedServiceAccount(namespace, name, saName, saNamespace string) { Eventually(func(g Gomega) { tr := &capsulev1beta2.TenantResource{} @@ -2006,7 +2088,12 @@ func cleanupTenantResourcesWithDefaultServiceAccount(ctx context.Context, namesp func expectConfigMapData(namespace, name string, expected map[string]string) { Eventually(func(g Gomega) { cm := &corev1.ConfigMap{} - g.Expect(k8sClient.Get(context.Background(), types.NamespacedName{Name: name, Namespace: namespace}, cm)).To(Succeed()) + g.Expect(k8sClient.Get(context.Background(), types.NamespacedName{Name: name, Namespace: namespace}, cm)).To( + Succeed(), + "expected ConfigMap %s/%s", + namespace, + name, + ) for k, v := range expected { g.Expect(cm.Data).To(HaveKeyWithValue(k, v)) } @@ -2224,6 +2311,16 @@ func bindServiceAccountToConfigMapDeleter(saNamespace, saName, targetNamespace s ) } +func bindServiceAccountToTenantResourceManager(saNamespace, saName, targetNamespace string) { + bindServiceAccountToNamespacedResource( + saNamespace, + saName, + targetNamespace, + []string{"configmaps", "secrets"}, + []string{"get", "list", "watch", "create", "update", "patch", "delete"}, + ) +} + func ensureServiceAccount(namespace, name string) { ctx := context.Background() diff --git a/e2e/rules_enforce_services_test.go b/e2e/rules_enforce_services_test.go index 169be86a..05ae498b 100644 --- a/e2e/rules_enforce_services_test.go +++ b/e2e/rules_enforce_services_test.go @@ -997,9 +997,9 @@ var _ = Describe("enforcing service namespace rules", Ordered, Label("tenant", " ns := createNamespace(nil) cs := ownerClient(tnt.Spec.Owners[0].UserSpec) - createServiceAndExpectDenied(cs, ns.Name, loadBalancerService("lb-source-range-denied", "", []string{"10.0.1.0/23"}, ptr.To(false)), + createServiceAndExpectDenied(cs, ns.Name, loadBalancerService("lb-source-range-denied", "", []string{"10.0.0.0/23"}, ptr.To(false)), "loadBalancer CIDR", - "10.0.1.0/23", + "10.0.0.0/23", "spec.loadBalancerSourceRanges[0]", "Allowed CIDRs", ) diff --git a/e2e/rules_quota_test.go b/e2e/rules_quota_test.go new file mode 100644 index 00000000..f90f06a6 --- /dev/null +++ b/e2e/rules_quota_test.go @@ -0,0 +1,446 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package e2e + +import ( + "context" + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" + "github.com/projectcapsule/capsule/pkg/api/rules" + tenantutils "github.com/projectcapsule/capsule/pkg/tenant" +) + +var _ = Describe("rule-generated GlobalResourceQuota", Ordered, Label("resourcequota", "rules", "ledger", "skip-on-openshift"), func() { + const ( + tenantName = "e2e-rule-quota" + + sharedCPUA = "e2e-rule-quota-cpu-a" + sharedCPUB = "e2e-rule-quota-cpu-b" + podCountA = "e2e-rule-quota-pods-a" + podCountB = "e2e-rule-quota-pods-b" + hpaCountA = "e2e-rule-quota-hpa-a" + hpaCountB = "e2e-rule-quota-hpa-b" + podLevelA = "e2e-rule-quota-pod-level-a" + podLevelB = "e2e-rule-quota-pod-level-b" + serviceA = "e2e-rule-quota-services-a" + serviceB = "e2e-rule-quota-services-b" + unselected = "e2e-rule-quota-unselected" + ) + + ctx := context.Background() + owner := rbac.UserSpec{Name: tenantName, Kind: "User"} + quotaScopes := []struct { + quotaName string + selectorKey string + namespaces []string + hard corev1.ResourceList + }{ + { + quotaName: "shared-cpu", + selectorKey: "e2e.projectcapsule.dev/shared-cpu", + namespaces: []string{sharedCPUA, sharedCPUB}, + hard: corev1.ResourceList{ + corev1.ResourceRequestsCPU: resource.MustParse("1"), + }, + }, + { + quotaName: "pod-count", + selectorKey: "e2e.projectcapsule.dev/pod-count", + namespaces: []string{podCountA, podCountB}, + hard: corev1.ResourceList{ + corev1.ResourcePods: resource.MustParse("1"), + }, + }, + { + quotaName: "hpa-count", + selectorKey: "e2e.projectcapsule.dev/hpa-count", + namespaces: []string{hpaCountA, hpaCountB}, + hard: corev1.ResourceList{ + corev1.ResourceName("count/horizontalpodautoscalers.autoscaling"): resource.MustParse("1"), + }, + }, + { + quotaName: "pod-level-cpu", + selectorKey: "e2e.projectcapsule.dev/pod-level-cpu", + namespaces: []string{podLevelA, podLevelB}, + hard: corev1.ResourceList{ + corev1.ResourceRequestsCPU: resource.MustParse("1"), + }, + }, + { + quotaName: "service-count", + selectorKey: "e2e.projectcapsule.dev/service-count", + namespaces: []string{serviceA, serviceB}, + hard: corev1.ResourceList{ + corev1.ResourceServices: resource.MustParse("5"), + }, + }, + } + ruleBodies := make([]*rules.NamespaceRuleBodyTenant, 0, len(quotaScopes)) + for _, scope := range quotaScopes { + ruleBodies = append(ruleBodies, &rules.NamespaceRuleBodyTenant{ + NamespaceRuleBodyNamespace: &rules.NamespaceRuleBodyNamespace{ + Quota: []rules.ResourceQuotaRule{{ + Name: scope.quotaName, + ResourceQuotaSpec: corev1.ResourceQuotaSpec{Hard: scope.hard}, + }}, + }, + NamespaceSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{scope.selectorKey: "true"}, + }, + }) + } + + tnt := &capsulev1beta2.Tenant{ + ObjectMeta: metav1.ObjectMeta{ + Name: tenantName, + Labels: map[string]string{"env": "e2e"}, + }, + Spec: capsulev1beta2.TenantSpec{ + Owners: rbac.OwnerListSpec{{CoreOwnerSpec: rbac.CoreOwnerSpec{UserSpec: owner}}}, + Rules: ruleBodies, + }, + } + + BeforeAll(func() { + EventuallyCreation(func() error { + tnt.ResourceVersion = "" + + return k8sClient.Create(ctx, tnt) + }).Should(Succeed()) + TenantReadyTrue(tnt) + + for _, scope := range quotaScopes { + for _, namespace := range scope.namespaces { + ns := NewNamespace(namespace, map[string]string{ + meta.TenantLabel: tenantName, + scope.selectorKey: "true", + }) + NamespaceCreation(ns, owner, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) + } + } + By("creating a namespace outside every quota selector", func() { + ns := NewNamespace(unselected, map[string]string{meta.TenantLabel: tenantName}) + NamespaceCreation(ns, owner, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) + }) + + current := &capsulev1beta2.Tenant{} + Expect(k8sClient.Get(ctx, clientKey("", tenantName), current)).To(Succeed()) + tnt.UID = current.UID + + for _, scope := range quotaScopes { + globalQuota := &capsulev1beta2.GlobalResourceQuota{} + Eventually(func() error { + return k8sClient.Get( + ctx, + clientKey("", tenantutils.RuleGlobalResourceQuotaName(tnt, scope.quotaName)), + globalQuota, + ) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + + quotaName := globalQuota.GetResourceQuotaName() + for _, namespace := range scope.namespaces { + Eventually(func(g Gomega) { + quota := &corev1.ResourceQuota{} + g.Expect(k8sClient.Get(ctx, types.NamespacedName{ + Namespace: namespace, + Name: quotaName, + }, quota)).To(Succeed()) + g.Expect(quota.Spec.Hard).To(HaveLen(len(scope.hard))) + for name, expected := range scope.hard { + actual, found := quota.Spec.Hard[name] + g.Expect(found).To(BeTrue(), "missing hard quota resource %s", name) + g.Expect(actual.Cmp(expected)).To(Equal(0), "hard quota resource %s", name) + } + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + } + + ledgerName := globalQuota.GetLedgerName() + Eventually(func(g Gomega) { + ledger := &capsulev1beta2.QuantityLedger{} + g.Expect(k8sClient.Get(ctx, types.NamespacedName{ + Namespace: ControllerNamespace, + Name: ledgerName, + }, ledger)).To(Succeed()) + g.Expect(ledger.Status.ResourceQuota).NotTo(BeNil()) + g.Expect(ledger.Status.ResourceQuota.Initialized).To(BeTrue()) + g.Expect(ledger.Status.ResourceQuota.ObservedGeneration).To(Equal(globalQuota.Generation)) + g.Expect(ledger.Status.ResourceQuota.Namespaces).To(ConsistOf(scope.namespaces)) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + } + }) + + AfterAll(func() { + EventuallyDeletion(tnt) + }) + + It("enforces one shared limit during concurrent admissions across selected namespaces", func() { + const total = 20 + + cs := ownerClient(owner) + results := make(chan error, total) + + for i := 0; i < total; i++ { + go func(index int) { + namespace := sharedCPUA + if index%2 == 1 { + namespace = sharedCPUB + } + + pod := MakePod( + namespace, + fmt.Sprintf("rule-quota-concurrent-%02d", index), + nil, + nil, + "registry.k8s.io/pause:3.10", + "100m", + "", + ) + _, err := cs.CoreV1().Pods(namespace).Create(ctx, pod, metav1.CreateOptions{}) + results <- err + }(i) + } + + var succeeded, failed int + for i := 0; i < total; i++ { + if err := <-results; err == nil { + succeeded++ + } else { + failed++ + Expect(err.Error()).To(ContainSubstring("exceeds GlobalResourceQuota")) + } + } + + Expect(succeeded).To(Equal(10)) + Expect(failed).To(Equal(10)) + + By("not applying the rule quota outside the selected namespace set", func() { + pod := MakePod( + unselected, + "rule-quota-unselected", + nil, + nil, + "registry.k8s.io/pause:3.10", + "2", + "", + ) + _, err := cs.CoreV1().Pods(unselected).Create(ctx, pod, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + }) + }) + + It("counts Pods without compute resources and intercepts the shared object limit", func() { + cs := ownerClient(owner) + + first := MakePod( + podCountA, + "rule-quota-pod-count-first", + nil, + nil, + "registry.k8s.io/pause:3.10", + "", + "", + ) + _, err := cs.CoreV1().Pods(podCountA).Create(ctx, first, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + second := MakePod( + podCountB, + "rule-quota-pod-count-second", + nil, + nil, + "registry.k8s.io/pause:3.10", + "", + "", + ) + _, err = cs.CoreV1().Pods(podCountB).Create(ctx, second, metav1.CreateOptions{}) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("exceeds GlobalResourceQuota")) + Expect(err.Error()).To(ContainSubstring( + "pods (requested=1, current=1, projected=2, hard=1, exceededBy=1)", + )) + }) + + It("counts HorizontalPodAutoscalers and intercepts the shared object limit", func() { + cs := ownerClient(owner) + makeHPA := func(namespace, name string) *autoscalingv2.HorizontalPodAutoscaler { + return &autoscalingv2.HorizontalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, + Spec: autoscalingv2.HorizontalPodAutoscalerSpec{ + ScaleTargetRef: autoscalingv2.CrossVersionObjectReference{ + APIVersion: "apps/v1", + Kind: "Deployment", + Name: "quota-target", + }, + MaxReplicas: 3, + }, + } + } + + _, err := cs.AutoscalingV2().HorizontalPodAutoscalers(hpaCountA).Create( + ctx, + makeHPA(hpaCountA, "rule-quota-hpa-first"), + metav1.CreateOptions{}, + ) + Expect(err).NotTo(HaveOccurred()) + + _, err = cs.AutoscalingV2().HorizontalPodAutoscalers(hpaCountB).Create( + ctx, + makeHPA(hpaCountB, "rule-quota-hpa-second"), + metav1.CreateOptions{}, + ) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("exceeds GlobalResourceQuota")) + }) + + It("calculates and intercepts Pod-level resources on controller-generated Pods", func() { + cs := ownerClient(owner) + makeDeployment := func(namespace, name string) *appsv1.Deployment { + deployment := MakeDeployment(namespace, name, 1, nil, "") + deployment.Spec.Template.Spec.Resources = &corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("600m"), + }, + } + + return deployment + } + + firstName := "rule-quota-pod-level-first" + _, err := cs.AppsV1().Deployments(podLevelA).Create( + ctx, + makeDeployment(podLevelA, firstName), + metav1.CreateOptions{}, + ) + Expect(err).NotTo(HaveOccurred()) + ExpectPodsForDeployment(ctx, podLevelA, firstName, 1) + + secondName := "rule-quota-pod-level-second" + _, err = cs.AppsV1().Deployments(podLevelB).Create( + ctx, + makeDeployment(podLevelB, secondName), + metav1.CreateOptions{}, + ) + Expect(err).NotTo(HaveOccurred()) + + Eventually(func(g Gomega) { + failure, getErr := replicaSetFailureForDeployment(ctx, podLevelB, secondName) + g.Expect(getErr).NotTo(HaveOccurred()) + g.Expect(failure).NotTo(BeNil()) + g.Expect(failure.Status).To(Equal(corev1.ConditionTrue)) + g.Expect(failure.Message).To(ContainSubstring("exceeds GlobalResourceQuota")) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + ExpectPodsForDeployment(ctx, podLevelB, secondName, 0) + }) + + It("atomically stalls concurrent Service admissions at the shared limit", func() { + const total = 10 + + cs := ownerClient(owner) + results := make(chan error, total) + + for i := 0; i < total; i++ { + go func(index int) { + namespace := serviceA + if index%2 == 1 { + namespace = serviceB + } + + service := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: fmt.Sprintf("rule-quota-service-%02d", index), + }, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{{Port: 80}}, + }, + } + _, err := cs.CoreV1().Services(namespace).Create(ctx, service, metav1.CreateOptions{}) + results <- err + }(i) + } + + var succeeded, failed int + for i := 0; i < total; i++ { + if err := <-results; err == nil { + succeeded++ + } else { + failed++ + Expect(err.Error()).To(ContainSubstring("exceeds GlobalResourceQuota")) + } + } + + Expect(succeeded).To(Equal(5)) + Expect(failed).To(Equal(5)) + }) + + It("keeps the generated quota identity when rules are reordered and limits change", func() { + quotaKey := clientKey("", tenantutils.RuleGlobalResourceQuotaName(tnt, "service-count")) + marker := "e2e.projectcapsule.dev/stable-identity" + + Eventually(func() error { + current := &capsulev1beta2.GlobalResourceQuota{} + if err := k8sClient.Get(ctx, quotaKey, current); err != nil { + return err + } + if current.Annotations == nil { + current.Annotations = map[string]string{} + } + current.Annotations[marker] = "preserved" + + return k8sClient.Update(ctx, current) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + + Eventually(func() error { + current := &capsulev1beta2.Tenant{} + if err := k8sClient.Get(ctx, clientKey("", tenantName), current); err != nil { + return err + } + + var serviceRule *rules.NamespaceRuleBodyTenant + remaining := make([]*rules.NamespaceRuleBodyTenant, 0, len(current.Spec.Rules)-1) + for _, rule := range current.Spec.Rules { + if rule != nil && len(rule.Quota) == 1 && rule.Quota[0].Name == "service-count" { + serviceRule = rule + continue + } + remaining = append(remaining, rule) + } + if serviceRule == nil { + return fmt.Errorf("service-count quota rule was not found") + } + serviceRule.Quota[0].Hard[corev1.ResourceServices] = resource.MustParse("6") + current.Spec.Rules = append([]*rules.NamespaceRuleBodyTenant{serviceRule}, remaining...) + + return k8sClient.Update(ctx, current) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + + Eventually(func(g Gomega) { + current := &capsulev1beta2.GlobalResourceQuota{} + g.Expect(k8sClient.Get(ctx, quotaKey, current)).To(Succeed()) + g.Expect(current.Annotations).To(HaveKeyWithValue(marker, "preserved")) + g.Expect(current.Labels).To(HaveKeyWithValue(meta.RuleQuotaLabel, "service-count")) + actual := current.Spec.Quota.Hard[corev1.ResourceServices] + g.Expect(actual.Cmp(resource.MustParse("6"))).To(Equal(0)) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + }) +}) + +func clientKey(namespace, name string) types.NamespacedName { + return types.NamespacedName{Namespace: namespace, Name: name} +} diff --git a/e2e/suite_test.go b/e2e/suite_test.go index aae410bd..a7179729 100644 --- a/e2e/suite_test.go +++ b/e2e/suite_test.go @@ -82,6 +82,31 @@ var _ = SynchronizedAfterSuite( // Keep this empty, or put per-worker cleanup here. }, func() { + Eventually(func() error { + var quotas capsulev1beta2.GlobalResourceQuotaList + + if err := k8sClient.List( + context.TODO(), + "as, + client.MatchingLabels{"env": "e2e"}, + ); err != nil { + return err + } + + if len(quotas.Items) == 0 { + return nil + } + + for i := range quotas.Items { + quota := "as.Items[i] + if err := k8sClient.Delete(context.TODO(), quota); err != nil && !apierrors.IsNotFound(err) { + return err + } + } + + return fmt.Errorf("still have %d global resource quotas with env=e2e", len(quotas.Items)) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + Eventually(func() error { var nsList corev1.NamespaceList diff --git a/e2e/tenant_managed_resources_test.go b/e2e/tenant_managed_resources_test.go index 58c61665..b5ca3960 100644 --- a/e2e/tenant_managed_resources_test.go +++ b/e2e/tenant_managed_resources_test.go @@ -105,7 +105,7 @@ var _ = Describe("creating namespaces within a Tenant with resources", Ordered, }, { IPBlock: &networkingv1.IPBlock{ - CIDR: "192.168.0.0/12", + CIDR: "192.168.0.0/16", }, }, }, @@ -118,7 +118,7 @@ var _ = Describe("creating namespaces within a Tenant with resources", Ordered, IPBlock: &networkingv1.IPBlock{ CIDR: "0.0.0.0/0", Except: []string{ - "192.168.0.0/12", + "192.168.0.0/16", }, }, }, diff --git a/go.mod b/go.mod index e895e802..324a7b3c 100644 --- a/go.mod +++ b/go.mod @@ -33,6 +33,7 @@ require ( k8s.io/apimachinery v0.36.3 k8s.io/apiserver v0.36.3 k8s.io/client-go v0.36.3 + k8s.io/component-helpers v0.36.3 k8s.io/klog/v2 v2.140.0 k8s.io/utils v0.0.0-20260507154919-ff6756f316d2 sigs.k8s.io/cluster-api v1.13.4 diff --git a/go.sum b/go.sum index de22a5f4..db3698bc 100644 --- a/go.sum +++ b/go.sum @@ -392,6 +392,8 @@ k8s.io/client-go v0.36.3 h1:M4JdVzXxYcZk4fGpfDdYnxSwhLKWCFoQsHW6t+z8Hfg= k8s.io/client-go v0.36.3/go.mod h1:gcPwr0c87vjjG6HB6pWEqOeuYVoXSsREjzux2j6GF30= k8s.io/component-base v0.36.3 h1:vc/UFvPCkW0irPz84LAodAL1j3f4xktPM6dDJIEheAY= k8s.io/component-base v0.36.3/go.mod h1:hZbNFG+gCMl9EbykDGEu73feKP9/Cq6JsV4pTo9GTO8= +k8s.io/component-helpers v0.36.3 h1:hya22S0Mto0SlHaiD4kMIi817f/tK7uTMsShxrDKQaY= +k8s.io/component-helpers v0.36.3/go.mod h1:QjREK1lOFXR+jxTqzrtHgOtzUc2s9sm8zuFSiK+TW+c= k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc= k8s.io/klog/v2 v2.140.0/go.mod h1:o+/RWfJ6PwpnFn7OyAG3QnO47BFsymfEfrz6XyYSSp0= k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a h1:xCeOEAOoGYl2jnJoHkC3hkbPJgdATINPMAxaynU2Ovg= diff --git a/hack/distro/capsule/example-setup/global-resource-quotas.yaml b/hack/distro/capsule/example-setup/global-resource-quotas.yaml new file mode 100644 index 00000000..3b2c69a2 --- /dev/null +++ b/hack/distro/capsule/example-setup/global-resource-quotas.yaml @@ -0,0 +1,15 @@ +--- +apiVersion: capsule.clastix.io/v1beta2 +kind: GlobalResourceQuota +metadata: + name: green-shared-compute +spec: + namespaceSelectors: + - matchLabels: + capsule.clastix.io/tenant: green + quota: + hard: + limits.cpu: "8" + limits.memory: 16Gi + requests.cpu: "8" + requests.memory: 16Gi diff --git a/hack/distro/capsule/example-setup/kustomization.yaml b/hack/distro/capsule/example-setup/kustomization.yaml index a60fba4d..602b044e 100644 --- a/hack/distro/capsule/example-setup/kustomization.yaml +++ b/hack/distro/capsule/example-setup/kustomization.yaml @@ -5,5 +5,6 @@ resources: - tenants.yaml - resource.yaml - pools.yaml + - global-resource-quotas.yaml - rbac.yaml - custom-quotas.yaml diff --git a/hack/distro/capsule/example-setup/tenants.yaml b/hack/distro/capsule/example-setup/tenants.yaml index 378d303d..d7952be3 100644 --- a/hack/distro/capsule/example-setup/tenants.yaml +++ b/hack/distro/capsule/example-setup/tenants.yaml @@ -10,37 +10,62 @@ spec: - name: alice kind: User rules: + - namespaceSelector: + matchExpressions: + - key: env + operator: In + values: + - "test" + quota: + - name: "max-pods" + hard: + pods: "10" + permissions: + promotions: + - clusterRoles: + - "secret-replicator" - quota: - - hard: + - name: shared-compute + hard: limits.cpu: "8" limits.memory: 16Gi requests.cpu: "8" requests.memory: 16Gi - classes: - gateway: - - matchLabels: - team: platform - ingress: - - matchLabels: - team: platform - storage: - - matchLabels: - team: platform - priority: - - matchLabels: - team: platform - runtime: - - matchLabels: - team: platform - cluster: - - matchLabels: - team: platform - namespaceSelector: - matchExpressions: - - key: env - operator: In - values: - - "test" + # classes: + # cluster: + # - matchLabels: + # team: platform + # namespaceSelector: + # matchExpressions: + # - key: env + # operator: In + # values: + # - "test" + # classes: + # gateway: + # - matchLabels: + # team: platform + # ingress: + # - matchLabels: + # team: platform + # storage: + # - matchLabels: + # team: platform + # priority: + # - matchLabels: + # team: platform + # runtime: + # - matchLabels: + # team: platform + # cluster: + # - matchLabels: + # team: platform + # namespaceSelector: + # matchExpressions: + # - key: env + # operator: In + # values: + # - "test" permissions: bindings: @@ -60,7 +85,7 @@ spec: - "Namespace" labels: pod-security.kubernetes.io/enforce: - managed: "restricted" + managed: "baseline" - audience: - kind: "Custom" name: "CapsuleUser" @@ -101,16 +126,6 @@ spec: additionalMetadataList: - labels: customer: a - resourceQuotas: - scope: Tenant - items: - - hard: - limits.cpu: "8" - limits.memory: 16Gi - requests.cpu: "8" - requests.memory: 16Gi - - hard: - pods: "10" --- apiVersion: capsule.clastix.io/v1beta2 kind: Tenant @@ -155,25 +170,6 @@ spec: ports: - from: 30000 to: 32767 - - enforce: - action: "allow" - metadata: - - kinds: - - "ConfigMap" - labels: - "corp.com/tenant": - required: true - values: - - exact: - - test - annotations: - "example.corp/cost-center": - required: true - values: - - exp: "^INV-[0-9]{4}$" - exact: - - prod - - test --- apiVersion: capsule.clastix.io/v1beta2 kind: Tenant diff --git a/hack/kind-cluster.yaml b/hack/kind-cluster.yaml index 86a250bd..63ec4c86 100644 --- a/hack/kind-cluster.yaml +++ b/hack/kind-cluster.yaml @@ -4,6 +4,8 @@ apiVersion: kind.x-k8s.io/v1alpha4 name: capsule featureGates: ImageVolume: true + # Alpha and disabled by default on the Kubernetes 1.32 and 1.33 e2e jobs. + PodLevelResources: true nodes: - role: control-plane - role: worker diff --git a/internal/controllers/globalresourcequotas/controller.go b/internal/controllers/globalresourcequotas/controller.go new file mode 100644 index 00000000..434c4df0 --- /dev/null +++ b/internal/controllers/globalresourcequotas/controller.go @@ -0,0 +1,352 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package globalresourcequotas + +import ( + "context" + "fmt" + "reflect" + "slices" + + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/events" + "k8s.io/client-go/util/retry" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + ctrlutils "github.com/projectcapsule/capsule/internal/controllers/utils" + "github.com/projectcapsule/capsule/internal/metrics" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/runtime/predicates" + "github.com/projectcapsule/capsule/pkg/runtime/selectors" +) + +type Controller struct { + client.Client + + reader client.Reader + log logr.Logger + recorder events.EventRecorder + metrics *metrics.GlobalResourceQuotaRecorder +} + +func (r *Controller) SetupWithManager(mgr ctrl.Manager, options ctrlutils.ControllerOptions) error { + r.reader = mgr.GetAPIReader() + + return ctrl.NewControllerManagedBy(mgr). + Named("capsule/global-resource-quotas"). + For( + &capsulev1beta2.GlobalResourceQuota{}, + builder.WithPredicates(predicate.Or( + predicate.GenerationChangedPredicate{}, + predicates.UpdatedMetadataPredicate{}, + predicates.DeletionChangedPredicate{}, + )), + ). + Owns( + &corev1.ResourceQuota{}, + builder.WithPredicates(predicate.Or( + predicate.GenerationChangedPredicate{}, + predicates.UpdatedMetadataPredicate{}, + predicates.DeletionChangedPredicate{}, + predicates.ResourceQuotaUsageChangedPredicate{}, + )), + ). + Owns(&capsulev1beta2.QuantityLedger{}). + Watches( + &corev1.Namespace{}, + handler.EnqueueRequestsFromMapFunc(r.globalQuotasForNamespace), + builder.WithPredicates(predicates.UpdatedMetadataPredicate{}), + ). + WithOptions(options.Runtime.ToControllerOptions()). + Complete(r) +} + +func (r *Controller) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.Result, error) { + instance := &capsulev1beta2.GlobalResourceQuota{} + if err := r.Get(ctx, request.NamespacedName, instance); err != nil { + if apierrors.IsNotFound(err) { + r.metrics.Delete(request.Name) + + return ctrl.Result{}, nil + } + + return ctrl.Result{}, err + } + + status, initialized, err := r.reconcile(ctx, instance) + if status == nil { + status = instance.Status.DeepCopy() + } + + ready := meta.NewReadyCondition(instance) + if err != nil { + ready.Status = metav1.ConditionFalse + ready.Reason = meta.FailedReason + ready.Message = err.Error() + } else if !initialized { + ready.Status = metav1.ConditionFalse + ready.Reason = meta.ReconcilingReason + ready.Message = "waiting for ResourceQuota usage initialization" + } + + status.Conditions.UpdateConditionByType(ready) + status.ObservedGeneration = instance.Generation + + if updateErr := r.updateStatus(ctx, instance, *status); updateErr != nil { + return ctrl.Result{}, updateErr + } + + instance.Status = *status + r.metrics.Record(instance) + + return ctrl.Result{}, err +} + +func (r *Controller) reconcile( + ctx context.Context, + instance *capsulev1beta2.GlobalResourceQuota, +) (*capsulev1beta2.GlobalResourceQuotaStatus, bool, error) { + namespaces, err := selectors.GetNamespacesMatchingSelectors( + ctx, + r.reader, + instance.Spec.NamespaceSelectors, + ) + if err != nil { + return nil, false, err + } + + if err := r.syncResourceQuotas(ctx, instance, namespaces); err != nil { + return nil, false, err + } + + status, initialized, err := r.observeUsage(ctx, instance, namespaces) + if err != nil { + return nil, false, err + } + + ledger, err := r.ensureLedger(ctx, instance) + if err != nil { + return status, false, err + } + + if err := r.reconcileLedger( + ctx, + ledger, + instance.Generation, + status.Namespaces, + status.Total.Used, + initialized, + ); err != nil { + return status, false, err + } + + return status, initialized, nil +} + +func (r *Controller) syncResourceQuotas( + ctx context.Context, + instance *capsulev1beta2.GlobalResourceQuota, + namespaces []corev1.Namespace, +) error { + selected := make(map[string]struct{}, len(namespaces)) + + for i := range namespaces { + namespace := &namespaces[i] + selected[namespace.Name] = struct{}{} + + target := &corev1.ResourceQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: instance.GetResourceQuotaName(), + Namespace: namespace.Name, + }, + } + + if err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { + _, err := controllerutil.CreateOrUpdate(ctx, r.Client, target, func() error { + targetLabels := target.GetLabels() + if targetLabels == nil { + targetLabels = map[string]string{} + } + + targetLabels[meta.NewManagedByCapsuleLabel] = meta.ValueController + targetLabels[meta.GlobalResourceQuotaLabel] = instance.Name + target.SetLabels(targetLabels) + target.Spec = *instance.Spec.Quota.DeepCopy() + + return controllerutil.SetControllerReference(instance, target, r.Scheme()) + }) + + return err + }); err != nil { + if apierrors.HasStatusCause(err, corev1.NamespaceTerminatingCause) { + continue + } + + return fmt.Errorf("sync ResourceQuota in namespace %s: %w", namespace.Name, err) + } + } + + list := &corev1.ResourceQuotaList{} + if err := r.List(ctx, list, client.MatchingLabels{ + meta.NewManagedByCapsuleLabel: meta.ValueController, + meta.GlobalResourceQuotaLabel: instance.Name, + }); err != nil { + return err + } + + for i := range list.Items { + item := &list.Items[i] + if _, keep := selected[item.Namespace]; keep { + continue + } + + if err := r.Delete(ctx, item); err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("delete stale ResourceQuota %s/%s: %w", item.Namespace, item.Name, err) + } + } + + return nil +} + +func (r *Controller) observeUsage( + ctx context.Context, + instance *capsulev1beta2.GlobalResourceQuota, + namespaces []corev1.Namespace, +) (*capsulev1beta2.GlobalResourceQuotaStatus, bool, error) { + status := instance.Status.DeepCopy() + status.Total.Hard = instance.Spec.Quota.Hard.DeepCopy() + status.Total.Used = capsulev1beta2.ZeroResourceList(instance.Spec.Quota.Hard) + status.NamespaceUsage = make(capsulev1beta2.GlobalResourceQuotaNamespaceUsage, len(namespaces)) + instanceCopy := instance.DeepCopy() + instanceCopy.Status = *status + instanceCopy.AssignNamespaces(namespaces) + status.Namespaces = instanceCopy.Status.Namespaces + status.NamespaceSize = instanceCopy.Status.NamespaceSize + + initialized := true + + for i := range namespaces { + namespace := namespaces[i].Name + used := capsulev1beta2.ZeroResourceList(instance.Spec.Quota.Hard) + quota := &corev1.ResourceQuota{} + + err := r.reader.Get(ctx, types.NamespacedName{ + Namespace: namespace, + Name: instance.GetResourceQuotaName(), + }, quota) + if err != nil { + if apierrors.IsNotFound(err) { + initialized = false + status.NamespaceUsage[namespace] = capsulev1beta2.GlobalResourceQuotaNamespaceStatus{Used: used} + + continue + } + + return status, false, err + } + + if !resourceQuotaStatusReady(quota, instance.Spec.Quota.Hard) { + initialized = false + } + + for name := range instance.Spec.Quota.Hard { + value := quota.Status.Used[name] + used[name] = value.DeepCopy() + total := status.Total.Used[name] + total.Add(value) + status.Total.Used[name] = total + } + + status.NamespaceUsage[namespace] = capsulev1beta2.GlobalResourceQuotaNamespaceStatus{Used: used} + } + + statusCopy := instance.DeepCopy() + statusCopy.Status = *status + statusCopy.CalculateAvailable() + + return &statusCopy.Status, initialized, nil +} + +func (r *Controller) updateStatus( + ctx context.Context, + instance *capsulev1beta2.GlobalResourceQuota, + status capsulev1beta2.GlobalResourceQuotaStatus, +) error { + return retry.RetryOnConflict(retry.DefaultBackoff, func() error { + current := &capsulev1beta2.GlobalResourceQuota{} + if err := r.reader.Get(ctx, client.ObjectKeyFromObject(instance), current); err != nil { + return err + } + + if reflect.DeepEqual(current.Status, status) { + return nil + } + + current.Status = *status.DeepCopy() + + return r.Status().Update(ctx, current) + }) +} + +func (r *Controller) globalQuotasForNamespace(ctx context.Context, object client.Object) []reconcile.Request { + namespace, ok := object.(*corev1.Namespace) + if !ok { + return nil + } + + list := &capsulev1beta2.GlobalResourceQuotaList{} + if err := r.List(ctx, list); err != nil { + r.log.Error(err, "failed to list GlobalResourceQuotas", "namespace", namespace.Name) + + return nil + } + + requests := make([]reconcile.Request, 0) + + for i := range list.Items { + item := &list.Items[i] + matched := slices.Contains(item.Status.Namespaces, namespace.Name) + + for _, namespaceSelector := range item.Spec.NamespaceSelectors { + if namespaceSelector.LabelSelector == nil { + continue + } + + selector, err := metav1.LabelSelectorAsSelector(namespaceSelector.LabelSelector) + if err == nil && selector.Matches(labels.Set(namespace.Labels)) { + matched = true + + break + } + } + + if matched { + requests = append(requests, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(item)}) + } + } + + return requests +} + +func resourceQuotaStatusReady(quota *corev1.ResourceQuota, hard corev1.ResourceList) bool { + for name := range hard { + if _, ok := quota.Status.Hard[name]; !ok { + return false + } + } + + return true +} diff --git a/internal/controllers/globalresourcequotas/controller_test.go b/internal/controllers/globalresourcequotas/controller_test.go new file mode 100644 index 00000000..fb6a455b --- /dev/null +++ b/internal/controllers/globalresourcequotas/controller_test.go @@ -0,0 +1,181 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package globalresourcequotas + +import ( + "context" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/pkg/runtime/selectors" +) + +func TestObserveUsageTracksTotalAndNamespaces(t *testing.T) { + t.Parallel() + + scheme := runtime.NewScheme() + if err := corev1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + if err := capsulev1beta2.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + + hard := corev1.ResourceList{ + corev1.ResourceRequestsCPU: resource.MustParse("8"), + corev1.ResourceRequestsMemory: resource.MustParse("16Gi"), + } + quota := &capsulev1beta2.GlobalResourceQuota{ + ObjectMeta: metav1.ObjectMeta{Name: "shared", UID: types.UID("quota-uid")}, + Spec: capsulev1beta2.GlobalResourceQuotaSpec{ + Quota: corev1.ResourceQuotaSpec{Hard: hard}, + }, + } + namespaceA := corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: "a"}, + Status: corev1.NamespaceStatus{Phase: corev1.NamespaceActive}, + } + namespaceB := corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: "b"}, + Status: corev1.NamespaceStatus{Phase: corev1.NamespaceActive}, + } + resourceQuotaA := observedResourceQuota(quota, namespaceA.Name, corev1.ResourceList{ + corev1.ResourceRequestsCPU: resource.MustParse("2"), + corev1.ResourceRequestsMemory: resource.MustParse("3Gi"), + }) + resourceQuotaB := observedResourceQuota(quota, namespaceB.Name, corev1.ResourceList{ + corev1.ResourceRequestsCPU: resource.MustParse("1"), + corev1.ResourceRequestsMemory: resource.MustParse("4Gi"), + }) + + cl := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(quota, resourceQuotaA, resourceQuotaB). + Build() + controller := &Controller{Client: cl, reader: cl} + + status, initialized, err := controller.observeUsage( + context.Background(), + quota, + []corev1.Namespace{namespaceB, namespaceA}, + ) + if err != nil { + t.Fatalf("observeUsage() error = %v", err) + } + if !initialized { + t.Fatal("observeUsage() initialized = false, want true") + } + if status.NamespaceSize != 2 || len(status.NamespaceUsage) != 2 { + t.Fatalf("namespace status = size %d usage %#v", status.NamespaceSize, status.NamespaceUsage) + } + if status.Namespaces[0] != "a" || status.Namespaces[1] != "b" { + t.Fatalf("ordered namespaces = %#v, want [a b]", status.Namespaces) + } + assertResource(t, status.Total.Used, corev1.ResourceRequestsCPU, "3") + assertResource(t, status.Total.Used, corev1.ResourceRequestsMemory, "7Gi") + assertResource(t, status.Total.Available, corev1.ResourceRequestsCPU, "5") + assertResource(t, status.NamespaceUsage["b"].Used, corev1.ResourceRequestsMemory, "4Gi") +} + +func TestReconcileLedgerStatusConsumesObservedUsage(t *testing.T) { + t.Parallel() + + now := metav1.Now() + expires := metav1.NewTime(now.Add(time.Minute)) + current := &capsulev1beta2.QuantityLedgerResourceQuotaStatus{ + Used: corev1.ResourceList{corev1.ResourceRequestsCPU: resource.MustParse("0")}, + Reservations: []capsulev1beta2.QuantityLedgerResourceQuotaReservation{{ + ID: "request", + Delta: corev1.ResourceList{corev1.ResourceRequestsCPU: resource.MustParse("2")}, + ExpiresAt: &expires, + }}, + } + + next := reconcileLedgerStatus( + current, + 3, + []string{"a", "b"}, + corev1.ResourceList{corev1.ResourceRequestsCPU: resource.MustParse("1")}, + true, + now, + ) + + if len(next.Reservations) != 1 { + t.Fatalf("reservations = %d, want 1", len(next.Reservations)) + } + assertResource(t, next.Reservations[0].Delta, corev1.ResourceRequestsCPU, "1") + assertResource(t, next.Used, corev1.ResourceRequestsCPU, "1") + assertResource(t, next.Allocated, corev1.ResourceRequestsCPU, "2") + if next.ObservedGeneration != 3 || len(next.Namespaces) != 2 { + t.Fatalf("ledger snapshot = generation %d, namespaces %#v", next.ObservedGeneration, next.Namespaces) + } +} + +func TestMatchingNamespaceSelectorsUseOR(t *testing.T) { + t.Parallel() + + quota := &capsulev1beta2.GlobalResourceQuota{ + Spec: capsulev1beta2.GlobalResourceQuotaSpec{ + NamespaceSelectors: []selectors.NamespaceSelector{ + {LabelSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"team": "a"}}}, + {LabelSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"team": "b"}}}, + }, + }, + } + namespace := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{ + Name: "b", Labels: map[string]string{"team": "b"}, + }} + + cl := fake.NewClientBuilder().WithObjects(namespace).Build() + matched, err := selectors.GetNamespacesMatchingSelectors( + context.Background(), + cl, + quota.Spec.NamespaceSelectors, + ) + if err != nil { + t.Fatal(err) + } + if len(matched) != 1 || matched[0].Name != namespace.Name { + t.Fatalf("matched namespaces = %#v, want b", matched) + } +} + +func observedResourceQuota( + quota *capsulev1beta2.GlobalResourceQuota, + namespace string, + used corev1.ResourceList, +) *corev1.ResourceQuota { + return &corev1.ResourceQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: quota.GetResourceQuotaName(), + Namespace: namespace, + }, + Status: corev1.ResourceQuotaStatus{ + Hard: quota.Spec.Quota.Hard.DeepCopy(), + Used: used.DeepCopy(), + }, + } +} + +func assertResource( + t *testing.T, + resources corev1.ResourceList, + name corev1.ResourceName, + want string, +) { + t.Helper() + + got := resources[name] + if got.Cmp(resource.MustParse(want)) != 0 { + t.Fatalf("%s = %s, want %s", name, got.String(), want) + } +} diff --git a/internal/controllers/globalresourcequotas/ledger.go b/internal/controllers/globalresourcequotas/ledger.go new file mode 100644 index 00000000..1473fb5b --- /dev/null +++ b/internal/controllers/globalresourcequotas/ledger.go @@ -0,0 +1,195 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package globalresourcequotas + +import ( + "context" + "reflect" + "slices" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/retry" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/runtime/configuration" + runtimequota "github.com/projectcapsule/capsule/pkg/runtime/quota" +) + +func (r *Controller) ensureLedger( + ctx context.Context, + quota *capsulev1beta2.GlobalResourceQuota, +) (*capsulev1beta2.QuantityLedger, error) { + ledger := &capsulev1beta2.QuantityLedger{ + ObjectMeta: metav1.ObjectMeta{ + Name: quota.GetLedgerName(), + Namespace: configuration.ControllerNamespace(), + }, + } + + _, err := controllerutil.CreateOrUpdate(ctx, r.Client, ledger, func() error { + ledgerLabels := ledger.GetLabels() + if ledgerLabels == nil { + ledgerLabels = map[string]string{} + } + + ledgerLabels[meta.NewManagedByCapsuleLabel] = meta.ValueController + ledgerLabels[meta.GlobalResourceQuotaLabel] = quota.Name + ledger.SetLabels(ledgerLabels) + ledger.Spec.TargetRef = capsulev1beta2.QuantityLedgerTargetRef{ + APIGroup: capsulev1beta2.GroupVersion.Group, + Kind: "GlobalResourceQuota", + Name: quota.Name, + UID: quota.UID, + } + + return controllerutil.SetControllerReference(quota, ledger, r.Scheme()) + }) + if err != nil { + return nil, err + } + + return ledger, nil +} + +func (r *Controller) reconcileLedger( + ctx context.Context, + ledger *capsulev1beta2.QuantityLedger, + generation int64, + namespaces []string, + used corev1.ResourceList, + initialized bool, +) error { + return retry.RetryOnConflict(retry.DefaultBackoff, func() error { + current := &capsulev1beta2.QuantityLedger{} + if err := r.reader.Get(ctx, types.NamespacedName{ + Namespace: ledger.Namespace, + Name: ledger.Name, + }, current); err != nil { + return err + } + + before := current.Status.ResourceQuota.DeepCopy() + + next := reconcileLedgerStatus( + current.Status.ResourceQuota, + generation, + namespaces, + used, + initialized, + metav1.Now(), + ) + if reflect.DeepEqual(before, next) { + return nil + } + + current.Status.ResourceQuota = next + + return r.Status().Update(ctx, current) + }) +} + +func reconcileLedgerStatus( + current *capsulev1beta2.QuantityLedgerResourceQuotaStatus, + generation int64, + namespaces []string, + used corev1.ResourceList, + initialized bool, + now metav1.Time, +) *capsulev1beta2.QuantityLedgerResourceQuotaStatus { + if current == nil { + current = &capsulev1beta2.QuantityLedgerResourceQuotaStatus{} + } else { + current = current.DeepCopy() + } + + increase := positiveDifference(used, current.Used) + active := make([]capsulev1beta2.QuantityLedgerResourceQuotaReservation, 0, len(current.Reservations)) + + for _, reservation := range current.Reservations { + if reservation.ExpiresAt != nil && reservation.ExpiresAt.Before(&now) { + continue + } + + reservation.Delta = consumeResourceList(reservation.Delta, increase) + if resourceListPositive(reservation.Delta) { + active = append(active, reservation) + } + } + + reserved := capsulev1beta2.ZeroResourceList(used) + for _, reservation := range active { + addResourceList(reserved, reservation.Delta) + } + + allocated := used.DeepCopy() + addResourceList(allocated, reserved) + + current.ObservedGeneration = generation + current.Initialized = initialized + current.Namespaces = slices.Clone(namespaces) + current.Used = used.DeepCopy() + current.Reserved = reserved + current.Allocated = allocated + current.Reservations = active + + return current +} + +func positiveDifference(next, previous corev1.ResourceList) corev1.ResourceList { + out := make(corev1.ResourceList, len(next)) + + for name, quantity := range next { + delta := quantity.DeepCopy() + delta.Sub(previous[name]) + runtimequota.ClampQuantityToZero(&delta) + out[name] = delta + } + + return out +} + +func consumeResourceList(delta, available corev1.ResourceList) corev1.ResourceList { + out := delta.DeepCopy() + for name, quantity := range out { + increase := available[name] + if quantity.Sign() <= 0 || increase.Sign() <= 0 { + continue + } + + consumed := quantity.DeepCopy() + if consumed.Cmp(increase) > 0 { + consumed = increase.DeepCopy() + } + + quantity.Sub(consumed) + out[name] = quantity + + increase.Sub(consumed) + available[name] = increase + } + + return out +} + +func addResourceList(target, addition corev1.ResourceList) { + for name, quantity := range addition { + current := target[name] + current.Add(quantity) + target[name] = current + } +} + +func resourceListPositive(resources corev1.ResourceList) bool { + for _, quantity := range resources { + if quantity.Sign() > 0 { + return true + } + } + + return false +} diff --git a/internal/controllers/globalresourcequotas/manager.go b/internal/controllers/globalresourcequotas/manager.go new file mode 100644 index 00000000..f1ecb3c4 --- /dev/null +++ b/internal/controllers/globalresourcequotas/manager.go @@ -0,0 +1,34 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package globalresourcequotas + +import ( + "fmt" + + "github.com/go-logr/logr" + "k8s.io/client-go/tools/events" + "sigs.k8s.io/controller-runtime/pkg/manager" + + "github.com/projectcapsule/capsule/internal/controllers/utils" + "github.com/projectcapsule/capsule/internal/metrics" +) + +func Add( + log logr.Logger, + mgr manager.Manager, + recorder events.EventRecorder, + cfg utils.ControllerOptions, +) error { + controller := &Controller{ + Client: mgr.GetClient(), + log: log, + recorder: recorder, + metrics: metrics.MustMakeGlobalResourceQuotaRecorder(), + } + if err := controller.SetupWithManager(mgr, cfg); err != nil { + return fmt.Errorf("unable to create GlobalResourceQuota controller: %w", err) + } + + return nil +} diff --git a/internal/controllers/rulestatus/manager.go b/internal/controllers/rulestatus/manager.go index 8516c780..1854b9db 100644 --- a/internal/controllers/rulestatus/manager.go +++ b/internal/controllers/rulestatus/manager.go @@ -157,13 +157,17 @@ func (r Manager) reconcile(ctx context.Context, instance *capsulev1beta2.RuleSta continue } - enforce := rule.Enforce.DeepCopy() + statusRule := rule.DeepCopy() + // RuleStatus is an enforcement cache. Quota definitions are reconciled + // independently as GlobalResourceQuotas and may include legacy entries + // which predate stable quota names. + statusRule.Quota = nil + enforce := rule.Enforce.DeepCopy() for i := range enforce.Metadata { enforce.Metadata[i].APIGroups = enforce.Metadata[i].StatusAPIGroups() } - statusRule := rule.DeepCopy() statusRule.Enforce = enforce ruleStatus = append(ruleStatus, statusRule) } @@ -248,8 +252,13 @@ func (r *Manager) updateReconcilingStatus(ctx context.Context, instance *capsule return err } + cleanedQuota := removeQuotaDefinitions(&latest.Status) if latest.Status.ObservedGeneration == instance.GetGeneration() { - return nil + if !cleanedQuota { + return nil + } + + return r.Status().Update(ctx, latest) } latest.Status.Conditions.UpdateConditionByType(meta.NewReadyConditionReconcilingReason(instance)) @@ -257,3 +266,23 @@ func (r *Manager) updateReconcilingStatus(ctx context.Context, instance *capsule return r.Status().Update(ctx, latest) }) } + +//nolint:staticcheck // The deprecated flattened Rule must be cleaned for objects written by older Capsule versions. +func removeQuotaDefinitions(status *capsulev1beta2.RuleStatusStatus) bool { + if status == nil { + return false + } + + changed := len(status.Rule.Quota) > 0 + + for _, rule := range status.Rules { + if rule != nil && len(rule.Quota) > 0 { + changed = true + rule.Quota = nil + } + } + + status.Rule.Quota = nil + + return changed +} diff --git a/internal/controllers/rulestatus/manager_test.go b/internal/controllers/rulestatus/manager_test.go new file mode 100644 index 00000000..37bb3ab9 --- /dev/null +++ b/internal/controllers/rulestatus/manager_test.go @@ -0,0 +1,70 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package rulestatus + +import ( + "context" + "testing" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/pkg/api/rules" +) + +func TestReconcileExcludesQuotaFromRuleStatus(t *testing.T) { + t.Parallel() + + unnamedQuota := rules.ResourceQuotaRule{ + ResourceQuotaSpec: corev1.ResourceQuotaSpec{Hard: corev1.ResourceList{ + corev1.ResourceRequestsCPU: resource.MustParse("1"), + }}, + } + instance := &capsulev1beta2.RuleStatus{ + Spec: []*rules.NamespaceRuleBodyNamespace{ + {Quota: []rules.ResourceQuotaRule{unnamedQuota}}, + { + Quota: []rules.ResourceQuotaRule{unnamedQuota}, + Enforce: &rules.NamespaceRuleEnforceBody{Action: rules.ActionTypeDeny}, + }, + }, + } + + if err := (Manager{}).reconcile(context.Background(), instance); err != nil { + t.Fatalf("reconcile() error = %v", err) + } + if len(instance.Status.Rules) != 1 { + t.Fatalf("status rules = %d, want one enforcement rule", len(instance.Status.Rules)) + } + if len(instance.Status.Rules[0].Quota) != 0 { + t.Fatalf("status quota = %#v, want none", instance.Status.Rules[0].Quota) + } + if instance.Status.Rules[0].Enforce == nil { + t.Fatal("status enforcement rule was removed") + } +} + +func TestRemoveQuotaDefinitionsCleansLegacyStatus(t *testing.T) { + t.Parallel() + + status := &capsulev1beta2.RuleStatusStatus{ + Rule: rules.NamespaceRuleBodyNamespace{Quota: []rules.ResourceQuotaRule{{}}}, + Rules: []*rules.NamespaceRuleBodyNamespace{ + nil, + {Quota: []rules.ResourceQuotaRule{{}}}, + }, + } + + if changed := removeQuotaDefinitions(status); !changed { + t.Fatal("legacy quota definitions were not reported as changed") + } + + if len(status.Rule.Quota) != 0 || len(status.Rules[1].Quota) != 0 { + t.Fatalf("legacy quota definitions were not removed: %#v", status) + } + if changed := removeQuotaDefinitions(status); changed { + t.Fatal("clean status was reported as changed") + } +} diff --git a/internal/controllers/tenant/globalresourcequotas.go b/internal/controllers/tenant/globalresourcequotas.go new file mode 100644 index 00000000..382b5f01 --- /dev/null +++ b/internal/controllers/tenant/globalresourcequotas.go @@ -0,0 +1,119 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package tenant + +import ( + "context" + "fmt" + "strings" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/labels" + k8svalidation "k8s.io/apimachinery/pkg/util/validation" + "k8s.io/client-go/util/retry" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/pkg/api/meta" + tenantutils "github.com/projectcapsule/capsule/pkg/tenant" +) + +func (r *Manager) syncGlobalResourceQuotas( + ctx context.Context, + tnt *capsulev1beta2.Tenant, +) error { + desired := make(map[string]struct{}) + quotaNames := make(map[string]struct{}) + + for ruleIndex, rule := range tnt.Spec.Rules { + if rule == nil || rule.NamespaceRuleBodyNamespace == nil { + continue + } + + for itemIndex := range rule.Quota { + quotaName := rule.Quota[itemIndex].Name + if errs := k8svalidation.IsDNS1123Label(quotaName); len(errs) > 0 { + return fmt.Errorf( + "rules[%d].quota[%d].name %q is invalid: %s", + ruleIndex, + itemIndex, + quotaName, + strings.Join(errs, "; "), + ) + } + + if _, duplicate := quotaNames[quotaName]; duplicate { + return fmt.Errorf("rules[%d].quota[%d].name %q is duplicated", ruleIndex, itemIndex, quotaName) + } + + quotaNames[quotaName] = struct{}{} + + if err := tenantutils.ValidateRuleGlobalResourceQuotaName(tnt, quotaName); err != nil { + return fmt.Errorf("rules[%d].quota[%d]: %w", ruleIndex, itemIndex, err) + } + + target := tenantutils.RuleGlobalResourceQuota(tnt, ruleIndex, itemIndex) + desired[target.Name] = struct{}{} + + if err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { + _, err := controllerutil.CreateOrUpdate(ctx, r.Client, target, func() error { + currentLabels := target.GetLabels() + if currentLabels == nil { + currentLabels = map[string]string{} + } + + currentLabels[meta.NewManagedByCapsuleLabel] = meta.ValueController + currentLabels[meta.NewTenantLabel] = tnt.Name + currentLabels[meta.RuleQuotaLabel] = quotaName + target.SetLabels(currentLabels) + + desiredQuota := tenantutils.RuleGlobalResourceQuota(tnt, ruleIndex, itemIndex) + target.Spec = *desiredQuota.Spec.DeepCopy() + + return controllerutil.SetControllerReference(tnt, target, r.Scheme()) + }) + + return err + }); err != nil { + return fmt.Errorf("sync GlobalResourceQuota %s: %w", target.Name, err) + } + } + } + + return r.pruneGlobalResourceQuotas(ctx, tnt, desired) +} + +func (r *Manager) pruneGlobalResourceQuotas( + ctx context.Context, + tnt *capsulev1beta2.Tenant, + desired map[string]struct{}, +) error { + list := &capsulev1beta2.GlobalResourceQuotaList{} + + selector := labels.SelectorFromSet(labels.Set{ + meta.NewManagedByCapsuleLabel: meta.ValueController, + meta.NewTenantLabel: tnt.Name, + }) + if err := r.List(ctx, list, &client.ListOptions{LabelSelector: selector}); err != nil { + return err + } + + for i := range list.Items { + item := &list.Items[i] + if _, generated := item.Labels[meta.RuleQuotaLabel]; !generated { + continue + } + + if _, keep := desired[item.Name]; keep { + continue + } + + if err := r.Delete(ctx, item); err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("delete stale GlobalResourceQuota %s: %w", item.Name, err) + } + } + + return nil +} diff --git a/internal/controllers/tenant/globalresourcequotas_test.go b/internal/controllers/tenant/globalresourcequotas_test.go new file mode 100644 index 00000000..af8a6c9f --- /dev/null +++ b/internal/controllers/tenant/globalresourcequotas_test.go @@ -0,0 +1,130 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package tenant + +import ( + "context" + "testing" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rules" + tenantutils "github.com/projectcapsule/capsule/pkg/tenant" +) + +func TestSyncGlobalResourceQuotasGeneratesAndPrunesRuleQuotas(t *testing.T) { + t.Parallel() + + scheme := runtime.NewScheme() + if err := capsulev1beta2.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + + tnt := &capsulev1beta2.Tenant{ + ObjectMeta: metav1.ObjectMeta{Name: "tenant-a", UID: types.UID("tenant-uid")}, + Spec: capsulev1beta2.TenantSpec{Rules: []*rules.NamespaceRuleBodyTenant{{ + NamespaceRuleBodyNamespace: &rules.NamespaceRuleBodyNamespace{ + Quota: []rules.ResourceQuotaRule{{ + Name: "shared-compute", + ResourceQuotaSpec: corev1.ResourceQuotaSpec{Hard: corev1.ResourceList{ + corev1.ResourceRequestsCPU: resource.MustParse("8"), + }}, + }}, + }, + NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"tier": "paid"}}, + }}}, + } + cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(tnt).Build() + manager := &Manager{Client: cl} + + if err := manager.syncGlobalResourceQuotas(context.Background(), tnt); err != nil { + t.Fatalf("syncGlobalResourceQuotas() error = %v", err) + } + + key := client.ObjectKey{Name: tenantutils.RuleGlobalResourceQuotaName(tnt, "shared-compute")} + generated := &capsulev1beta2.GlobalResourceQuota{} + if err := cl.Get(context.Background(), key, generated); err != nil { + t.Fatalf("get generated GlobalResourceQuota: %v", err) + } + if generated.Labels[meta.RuleQuotaLabel] != "shared-compute" { + t.Fatalf("rule quota label = %q, want shared-compute", generated.Labels[meta.RuleQuotaLabel]) + } + selector := generated.Spec.NamespaceSelectors[0].LabelSelector + if selector.MatchLabels[meta.TenantLabel] != tnt.Name || selector.MatchLabels["tier"] != "paid" { + t.Fatalf("generated selector = %#v", selector) + } + + generated.Annotations = map[string]string{"test.projectcapsule.dev/identity": "preserved"} + if err := cl.Update(context.Background(), generated); err != nil { + t.Fatalf("annotate generated GlobalResourceQuota: %v", err) + } + + sharedRule := tnt.Spec.Rules[0] + sharedRule.NamespaceSelector = &metav1.LabelSelector{MatchLabels: map[string]string{"tier": "enterprise"}} + sharedRule.Quota[0].Hard[corev1.ResourceRequestsCPU] = resource.MustParse("12") + tnt.Spec.Rules = []*rules.NamespaceRuleBodyTenant{ + { + NamespaceRuleBodyNamespace: &rules.NamespaceRuleBodyNamespace{ + Quota: []rules.ResourceQuotaRule{{ + Name: "service-count", + ResourceQuotaSpec: corev1.ResourceQuotaSpec{Hard: corev1.ResourceList{ + corev1.ResourceServices: resource.MustParse("5"), + }}, + }}, + }, + }, + sharedRule, + } + if err := manager.syncGlobalResourceQuotas(context.Background(), tnt); err != nil { + t.Fatalf("sync reordered GlobalResourceQuotas: %v", err) + } + + updated := &capsulev1beta2.GlobalResourceQuota{} + if err := cl.Get(context.Background(), key, updated); err != nil { + t.Fatalf("get stable GlobalResourceQuota after reorder: %v", err) + } + if updated.Annotations["test.projectcapsule.dev/identity"] != "preserved" { + t.Fatal("generated GlobalResourceQuota was replaced after rule reorder") + } + updatedSelector := updated.Spec.NamespaceSelectors[0].LabelSelector + if updatedSelector.MatchLabels["tier"] != "enterprise" { + t.Fatalf("updated selector = %#v, want tier=enterprise", updatedSelector) + } + if got := updated.Spec.Quota.Hard[corev1.ResourceRequestsCPU]; got.Cmp(resource.MustParse("12")) != 0 { + t.Fatalf("updated hard requests.cpu = %s, want 12", got.String()) + } + + list := &capsulev1beta2.GlobalResourceQuotaList{} + if err := cl.List(context.Background(), list); err != nil { + t.Fatalf("list generated GlobalResourceQuotas: %v", err) + } + if len(list.Items) != 2 { + t.Fatalf("generated GlobalResourceQuotas = %d, want 2", len(list.Items)) + } + + tnt.Spec.Rules = nil + if err := manager.syncGlobalResourceQuotas(context.Background(), tnt); err != nil { + t.Fatalf("prune GlobalResourceQuota: %v", err) + } + err := cl.Get(context.Background(), key, &capsulev1beta2.GlobalResourceQuota{}) + if !apierrors.IsNotFound(err) { + t.Fatalf("get pruned GlobalResourceQuota error = %v, want NotFound", err) + } + list = &capsulev1beta2.GlobalResourceQuotaList{} + if err := cl.List(context.Background(), list); err != nil { + t.Fatalf("list pruned GlobalResourceQuotas: %v", err) + } + if len(list.Items) != 0 { + t.Fatalf("GlobalResourceQuotas after prune = %d, want 0", len(list.Items)) + } +} diff --git a/internal/controllers/tenant/manager.go b/internal/controllers/tenant/manager.go index 9113433c..15191148 100644 --- a/internal/controllers/tenant/manager.go +++ b/internal/controllers/tenant/manager.go @@ -83,6 +83,14 @@ func (r *Manager) SetupWithManager(mgr ctrl.Manager, ctrlConfig utils.Controller ). Owns(&networkingv1.NetworkPolicy{}, builder.WithPredicates(predicates.TenantManagedResourceChangedPredicate{})). Owns(&corev1.LimitRange{}, builder.WithPredicates(predicates.TenantManagedResourceChangedPredicate{})). + Owns( + &capsulev1beta2.GlobalResourceQuota{}, + builder.WithPredicates(predicate.Or( + predicate.GenerationChangedPredicate{}, + predicates.UpdatedMetadataPredicate{}, + predicates.DeletionChangedPredicate{}, + )), + ). Watches( &corev1.ResourceQuota{}, handler.Funcs{ @@ -403,6 +411,12 @@ func (r *Manager) reconcile(ctx context.Context, log logr.Logger, instance *caps errs = append(errs, fmt.Errorf("cannot sync resourcequota items: %w", err)) } + log.V(4).Info("starting processing of rule GlobalResourceQuotas") + + if err = r.syncGlobalResourceQuotas(ctx, instance); err != nil { + errs = append(errs, fmt.Errorf("cannot sync rule global resource quotas: %w", err)) + } + log.V(4).Info("ensuring RoleBindings for Owners and Tenant") if err = r.syncRoleBindings(ctx, log, instance); err != nil { diff --git a/internal/controllers/tls/utils.go b/internal/controllers/tls/utils.go index f8c37309..974572b8 100644 --- a/internal/controllers/tls/utils.go +++ b/internal/controllers/tls/utils.go @@ -42,6 +42,9 @@ func (r Reconciler) managedCRDs() map[string]ManagedCRD { "globalcustomquotas": { Name: "globalcustomquotas.capsule.clastix.io", }, + "globalresourcequotas": { + Name: "globalresourcequotas.capsule.clastix.io", + }, "globaltenantresources": { Name: "globaltenantresources.capsule.clastix.io", }, diff --git a/internal/metrics/global_resourcequota_recorder.go b/internal/metrics/global_resourcequota_recorder.go new file mode 100644 index 00000000..56d66d48 --- /dev/null +++ b/internal/metrics/global_resourcequota_recorder.go @@ -0,0 +1,148 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + crtlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/pkg/api/meta" +) + +type GlobalResourceQuotaRecorder struct { + ConditionGauge *prometheus.GaugeVec + ResourceLimitGauge *prometheus.GaugeVec + ResourceUsageGauge *prometheus.GaugeVec + ResourceAvailableGauge *prometheus.GaugeVec + ResourceUsagePercentageGauge *prometheus.GaugeVec + NamespaceUsageGauge *prometheus.GaugeVec + NamespaceUsagePercentageGauge *prometheus.GaugeVec +} + +func MustMakeGlobalResourceQuotaRecorder() *GlobalResourceQuotaRecorder { + recorder := NewGlobalResourceQuotaRecorder() + crtlmetrics.Registry.MustRegister(recorder.Collectors()...) + + return recorder +} + +func NewGlobalResourceQuotaRecorder() *GlobalResourceQuotaRecorder { + const label = "global_resource_quota" + + return &GlobalResourceQuotaRecorder{ + ConditionGauge: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: metricsPrefix, + Name: "global_resource_quota_condition", + Help: "Current condition for a GlobalResourceQuota.", + }, []string{label, "condition"}), + ResourceLimitGauge: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: metricsPrefix, + Name: "global_resource_quota_limit", + Help: "Shared hard limit for a GlobalResourceQuota resource.", + }, []string{label, "resource"}), + ResourceUsageGauge: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: metricsPrefix, + Name: "global_resource_quota_usage", + Help: "Observed aggregate usage for a GlobalResourceQuota resource.", + }, []string{label, "resource"}), + ResourceAvailableGauge: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: metricsPrefix, + Name: "global_resource_quota_available", + Help: "Available aggregate capacity for a GlobalResourceQuota resource.", + }, []string{label, "resource"}), + ResourceUsagePercentageGauge: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: metricsPrefix, + Name: "global_resource_quota_usage_percentage", + Help: "Observed aggregate usage percentage for a GlobalResourceQuota resource.", + }, []string{label, "resource"}), + NamespaceUsageGauge: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: metricsPrefix, + Name: "global_resource_quota_namespace_usage", + Help: "Observed usage per namespace for a GlobalResourceQuota resource.", + }, []string{label, "target_namespace", "resource"}), + NamespaceUsagePercentageGauge: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: metricsPrefix, + Name: "global_resource_quota_namespace_usage_percentage", + Help: "Observed per-namespace usage as a percentage of the shared limit.", + }, []string{label, "target_namespace", "resource"}), + } +} + +func (r *GlobalResourceQuotaRecorder) Collectors() []prometheus.Collector { + return []prometheus.Collector{ + r.ConditionGauge, + r.ResourceLimitGauge, + r.ResourceUsageGauge, + r.ResourceAvailableGauge, + r.ResourceUsagePercentageGauge, + r.NamespaceUsageGauge, + r.NamespaceUsagePercentageGauge, + } +} + +func (r *GlobalResourceQuotaRecorder) Record(quota *capsulev1beta2.GlobalResourceQuota) { + r.Delete(quota.Name) + + for _, conditionType := range []string{meta.ReadyCondition} { + condition := quota.Status.Conditions.GetConditionByType(conditionType) + if condition == nil { + continue + } + + value := float64(0) + if condition.Status == metav1.ConditionTrue { + value = 1 + } + + r.ConditionGauge.WithLabelValues(quota.Name, conditionType).Set(value) + } + + for name, hard := range quota.Status.Total.Hard { + used := quota.Status.Total.Used[name] + available := quota.Status.Total.Available[name] + + r.ResourceLimitGauge.WithLabelValues(quota.Name, name.String()).Set(quantityMetric(hard)) + r.ResourceUsageGauge.WithLabelValues(quota.Name, name.String()).Set(quantityMetric(used)) + r.ResourceAvailableGauge.WithLabelValues(quota.Name, name.String()).Set(quantityMetric(available)) + r.ResourceUsagePercentageGauge.WithLabelValues(quota.Name, name.String()).Set(quantityPercentage(used, hard)) + } + + for namespace, usage := range quota.Status.NamespaceUsage { + for name, used := range usage.Used { + hard := quota.Status.Total.Hard[name] + r.NamespaceUsageGauge.WithLabelValues(quota.Name, namespace, name.String()).Set(quantityMetric(used)) + r.NamespaceUsagePercentageGauge.WithLabelValues( + quota.Name, + namespace, + name.String(), + ).Set(quantityPercentage(used, hard)) + } + } +} + +func (r *GlobalResourceQuotaRecorder) Delete(name string) { + labels := prometheus.Labels{"global_resource_quota": name} + r.ConditionGauge.DeletePartialMatch(labels) + r.ResourceLimitGauge.DeletePartialMatch(labels) + r.ResourceUsageGauge.DeletePartialMatch(labels) + r.ResourceAvailableGauge.DeletePartialMatch(labels) + r.ResourceUsagePercentageGauge.DeletePartialMatch(labels) + r.NamespaceUsageGauge.DeletePartialMatch(labels) + r.NamespaceUsagePercentageGauge.DeletePartialMatch(labels) +} + +func quantityMetric(quantity resource.Quantity) float64 { + return float64(quantity.MilliValue()) / 1000 +} + +func quantityPercentage(used, hard resource.Quantity) float64 { + if hard.MilliValue() <= 0 { + return 0 + } + + return float64(used.MilliValue()) / float64(hard.MilliValue()) * 100 +} diff --git a/internal/metrics/global_resourcequota_recorder_test.go b/internal/metrics/global_resourcequota_recorder_test.go new file mode 100644 index 00000000..9b37d265 --- /dev/null +++ b/internal/metrics/global_resourcequota_recorder_test.go @@ -0,0 +1,94 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package metrics + +import ( + "testing" + + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" +) + +func TestGlobalResourceQuotaRecorderTracksAggregateAndNamespaceUsage(t *testing.T) { + t.Parallel() + + recorder := NewGlobalResourceQuotaRecorder() + quota := &capsulev1beta2.GlobalResourceQuota{} + quota.Name = "shared" + quota.Status.Total = capsulev1beta2.GlobalResourceQuotaUsage{ + Hard: corev1.ResourceList{corev1.ResourceRequestsCPU: resource.MustParse("8")}, + Used: corev1.ResourceList{corev1.ResourceRequestsCPU: resource.MustParse("2")}, + Available: corev1.ResourceList{corev1.ResourceRequestsCPU: resource.MustParse("6")}, + } + quota.Status.NamespaceUsage = capsulev1beta2.GlobalResourceQuotaNamespaceUsage{ + "tenant-a": { + Used: corev1.ResourceList{corev1.ResourceRequestsCPU: resource.MustParse("1.5")}, + }, + } + + recorder.Record(quota) + + assertGauge(t, recorder.ResourceLimitGauge, 8, "shared", string(corev1.ResourceRequestsCPU)) + assertGauge(t, recorder.ResourceUsageGauge, 2, "shared", string(corev1.ResourceRequestsCPU)) + assertGauge(t, recorder.ResourceAvailableGauge, 6, "shared", string(corev1.ResourceRequestsCPU)) + assertGauge(t, recorder.ResourceUsagePercentageGauge, 25, "shared", string(corev1.ResourceRequestsCPU)) + assertGauge( + t, + recorder.NamespaceUsageGauge, + 1.5, + "shared", + "tenant-a", + string(corev1.ResourceRequestsCPU), + ) + assertGauge( + t, + recorder.NamespaceUsagePercentageGauge, + 18.75, + "shared", + "tenant-a", + string(corev1.ResourceRequestsCPU), + ) + + recorder.Delete(quota.Name) + if got := metricCount(recorder.ResourceUsageGauge); got != 0 { + t.Fatalf("usage metric count after delete = %d, want 0", got) + } +} + +type gaugeMetric interface { + GetMetricWithLabelValues(lvs ...string) (prometheus.Gauge, error) +} + +func assertGauge(t *testing.T, gauge gaugeMetric, want float64, labels ...string) { + t.Helper() + + metric, err := gauge.GetMetricWithLabelValues(labels...) + if err != nil { + t.Fatal(err) + } + value := &dto.Metric{} + if err := metric.Write(value); err != nil { + t.Fatal(err) + } + if got := value.GetGauge().GetValue(); got != want { + t.Fatalf("metric %v = %v, want %v", labels, got, want) + } +} + +func metricCount(collector prometheus.Collector) int { + metrics := make(chan prometheus.Metric, 32) + collector.Collect(metrics) + close(metrics) + + count := 0 + for range metrics { + count++ + } + + return count +} diff --git a/internal/quota/evaluator/evaluator.go b/internal/quota/evaluator/evaluator.go new file mode 100644 index 00000000..468db8f2 --- /dev/null +++ b/internal/quota/evaluator/evaluator.go @@ -0,0 +1,637 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package evaluator contains the admission-side resource calculations used by +// GlobalResourceQuota. The calculations are adapted from the Kubernetes +// ResourceQuota core evaluators at the version matching this module's +// Kubernetes dependencies. +package evaluator + +import ( + "encoding/json" + "fmt" + "slices" + "strings" + "time" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/selection" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apiserver/pkg/quota/v1/generic" + resourcehelper "k8s.io/component-helpers/resource" + storagehelpers "k8s.io/component-helpers/storage/volume" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +const storageClassSuffix = ".storageclass.storage.k8s.io/" + +var validationResources = sets.New( + corev1.ResourceCPU, + corev1.ResourceMemory, + corev1.ResourceRequestsCPU, + corev1.ResourceRequestsMemory, + corev1.ResourceLimitsCPU, + corev1.ResourceLimitsMemory, +) + +var legacyObjectCountAliases = map[string]corev1.ResourceName{ + "configmaps": corev1.ResourceConfigMaps, + "resourcequotas": corev1.ResourceQuotas, + "replicationcontrollers": corev1.ResourceReplicationControllers, + "secrets": corev1.ResourceSecrets, +} + +// Result holds one decoded admission request and its native quota usage. +type Result struct { + NewUsage corev1.ResourceList + OldUsage corev1.ResourceList + New runtime.Object + Old runtime.Object +} + +// Evaluate decodes an admission request once and calculates the same native +// resource names used by the Kubernetes core quota evaluators. +func Evaluate(req admission.Request) (Result, bool, error) { + if req.SubResource != "" && req.SubResource != "resize" && req.SubResource != "status" { + return Result{}, false, nil + } + + resourceName := req.Resource.Resource + + switch { + case req.Resource.Group == "" && resourceName == "pods": + return evaluatePod(req) + case req.Resource.Group == "" && resourceName == "services": + return evaluateService(req) + case req.Resource.Group == "" && resourceName == "persistentvolumeclaims": + return evaluatePVC(req) + default: + if req.SubResource != "" || req.Operation != "CREATE" { + return Result{}, false, nil + } + + object := &metav1.PartialObjectMetadata{} + if err := json.Unmarshal(req.Object.Raw, object); err != nil { + return Result{}, false, err + } + + usage := objectCountUsage(req.Resource.Group, resourceName) + + return Result{NewUsage: usage, New: object}, true, nil + } +} + +func evaluatePod(req admission.Request) (Result, bool, error) { + if req.Operation != "CREATE" && req.Operation != "UPDATE" { + return Result{}, false, nil + } + + newPod := &corev1.Pod{} + if err := json.Unmarshal(req.Object.Raw, newPod); err != nil { + return Result{}, false, fmt.Errorf("decode Pod: %w", err) + } + + newUsage := podUsage(newPod, time.Now()) + result := Result{NewUsage: newUsage, New: newPod} + + if req.Operation == "UPDATE" { + oldPod := &corev1.Pod{} + if err := json.Unmarshal(req.OldObject.Raw, oldPod); err != nil { + return Result{}, false, fmt.Errorf("decode old Pod: %w", err) + } + + result.Old = oldPod + result.OldUsage = podUsage(oldPod, time.Now()) + } + + return result, true, nil +} + +func evaluateService(req admission.Request) (Result, bool, error) { + if req.SubResource != "" || (req.Operation != "CREATE" && req.Operation != "UPDATE") { + return Result{}, false, nil + } + + service := &corev1.Service{} + if err := json.Unmarshal(req.Object.Raw, service); err != nil { + return Result{}, false, fmt.Errorf("decode Service: %w", err) + } + + result := Result{NewUsage: serviceUsage(service), New: service} + + if req.Operation == "UPDATE" { + oldService := &corev1.Service{} + if err := json.Unmarshal(req.OldObject.Raw, oldService); err != nil { + return Result{}, false, fmt.Errorf("decode old Service: %w", err) + } + + result.Old = oldService + result.OldUsage = serviceUsage(oldService) + } + + return result, true, nil +} + +func evaluatePVC(req admission.Request) (Result, bool, error) { + if (req.SubResource != "" && req.SubResource != "status") || + (req.Operation != "CREATE" && req.Operation != "UPDATE") { + return Result{}, false, nil + } + + pvc := &corev1.PersistentVolumeClaim{} + if err := json.Unmarshal(req.Object.Raw, pvc); err != nil { + return Result{}, false, fmt.Errorf("decode PersistentVolumeClaim: %w", err) + } + + result := Result{NewUsage: pvcUsage(pvc), New: pvc} + + if req.Operation == "UPDATE" { + oldPVC := &corev1.PersistentVolumeClaim{} + if err := json.Unmarshal(req.OldObject.Raw, oldPVC); err != nil { + return Result{}, false, fmt.Errorf("decode old PersistentVolumeClaim: %w", err) + } + + result.Old = oldPVC + result.OldUsage = pvcUsage(oldPVC) + } + + return result, true, nil +} + +func objectCountUsage(group, resourceName string) corev1.ResourceList { + countName := generic.ObjectCountQuotaResourceNameFor( + schema.GroupResource{Group: group, Resource: resourceName}, + ) + one := *resource.NewQuantity(1, resource.DecimalSI) + result := corev1.ResourceList{countName: one} + + if group == "" { + if alias, ok := legacyObjectCountAliases[resourceName]; ok { + result[alias] = one + } + } + + return result +} + +func podUsage(pod *corev1.Pod, now time.Time) corev1.ResourceList { + result := objectCountUsage("", "pods") + if !quotaPod(pod, now) { + return result + } + + opts := resourcehelper.PodResourcesOptions{ + UseStatusResources: true, + SkipPodLevelResources: false, + } + requests := resourcehelper.PodRequests(pod, opts) + limits := resourcehelper.PodLimits(pod, opts) + addResourceList(result, podComputeUsage(requests, limits)) + + return result +} + +func podComputeUsage(requests, limits corev1.ResourceList) corev1.ResourceList { + result := corev1.ResourceList{ + corev1.ResourcePods: *resource.NewQuantity(1, resource.DecimalSI), + } + + addRequest := func(name, plain, prefixed corev1.ResourceName) { + if quantity, found := requests[name]; found { + result[plain] = quantity + result[prefixed] = quantity + } + } + + addRequest(corev1.ResourceCPU, corev1.ResourceCPU, corev1.ResourceRequestsCPU) + addRequest(corev1.ResourceMemory, corev1.ResourceMemory, corev1.ResourceRequestsMemory) + addRequest(corev1.ResourceEphemeralStorage, corev1.ResourceEphemeralStorage, corev1.ResourceRequestsEphemeralStorage) + + if quantity, found := limits[corev1.ResourceCPU]; found { + result[corev1.ResourceLimitsCPU] = quantity + } + + if quantity, found := limits[corev1.ResourceMemory]; found { + result[corev1.ResourceLimitsMemory] = quantity + } + + if quantity, found := limits[corev1.ResourceEphemeralStorage]; found { + result[corev1.ResourceLimitsEphemeralStorage] = quantity + } + + for name, quantity := range requests { + switch { + case strings.HasPrefix(string(name), corev1.ResourceHugePagesPrefix): + result[name] = quantity + result[corev1.ResourceName(corev1.DefaultResourceRequestsPrefix+string(name))] = quantity + case isExtendedResourceName(name): + result[corev1.ResourceName(corev1.DefaultResourceRequestsPrefix+string(name))] = quantity + } + } + + return result +} + +func serviceUsage(service *corev1.Service) corev1.ResourceList { + result := objectCountUsage("", "services") + result[corev1.ResourceServices] = *resource.NewQuantity(1, resource.DecimalSI) + result[corev1.ResourceServicesLoadBalancers] = *resource.NewQuantity(0, resource.DecimalSI) + result[corev1.ResourceServicesNodePorts] = *resource.NewQuantity(0, resource.DecimalSI) + + ports := int64(len(service.Spec.Ports)) + + switch service.Spec.Type { + case corev1.ServiceTypeClusterIP, corev1.ServiceTypeExternalName: + case corev1.ServiceTypeNodePort: + result[corev1.ResourceServicesNodePorts] = *resource.NewQuantity(ports, resource.DecimalSI) + case corev1.ServiceTypeLoadBalancer: + if ptr.Deref(service.Spec.AllocateLoadBalancerNodePorts, true) { + result[corev1.ResourceServicesNodePorts] = *resource.NewQuantity(ports, resource.DecimalSI) + } else { + var count int64 + + for _, port := range service.Spec.Ports { + if port.NodePort != 0 { + count++ + } + } + + result[corev1.ResourceServicesNodePorts] = *resource.NewQuantity(count, resource.DecimalSI) + } + + result[corev1.ResourceServicesLoadBalancers] = *resource.NewQuantity(1, resource.DecimalSI) + } + + return result +} + +func pvcUsage(pvc *corev1.PersistentVolumeClaim) corev1.ResourceList { + result := objectCountUsage("", "persistentvolumeclaims") + one := *resource.NewQuantity(1, resource.DecimalSI) + result[corev1.ResourcePersistentVolumeClaims] = one + + storageClass := storagehelpers.GetPersistentVolumeClaimClass(pvc) + if storageClass != "" { + result[resourceByStorageClass(storageClass, corev1.ResourcePersistentVolumeClaims)] = one + } + + requested, ok := pvc.Spec.Resources.Requests[corev1.ResourceStorage] + if !ok { + return result + } + + rounded := requested.DeepCopy() + + _ = rounded.RoundUp(0) + + if allocated, ok := pvc.Status.AllocatedResources[corev1.ResourceStorage]; ok && allocated.Cmp(rounded) > 0 { + rounded = allocated.DeepCopy() + _ = rounded.RoundUp(0) + } + + result[corev1.ResourceRequestsStorage] = rounded + if storageClass != "" { + result[resourceByStorageClass(storageClass, corev1.ResourceRequestsStorage)] = rounded + } + + return result +} + +func resourceByStorageClass(storageClass string, name corev1.ResourceName) corev1.ResourceName { + return corev1.ResourceName(storageClass + storageClassSuffix + string(name)) +} + +// MatchesScopes mirrors generic quota scope matching. +func MatchesScopes(spec corev1.ResourceQuotaSpec, object runtime.Object) (bool, error) { + requirements := make([]corev1.ScopedResourceSelectorRequirement, 0, len(spec.Scopes)) + for _, scope := range spec.Scopes { + requirements = append(requirements, corev1.ScopedResourceSelectorRequirement{ + ScopeName: scope, + Operator: corev1.ScopeSelectorOpExists, + }) + } + + if spec.ScopeSelector != nil { + requirements = append(requirements, spec.ScopeSelector.MatchExpressions...) + } + + for _, requirement := range requirements { + matches, err := matchesScope(requirement, object) + if err != nil || !matches { + return matches, err + } + } + + return true, nil +} + +func matchesScope(requirement corev1.ScopedResourceSelectorRequirement, object runtime.Object) (bool, error) { + switch typed := object.(type) { + case *corev1.Pod: + return podMatchesScope(requirement, typed) + case *corev1.PersistentVolumeClaim: + return pvcMatchesScope(requirement, typed) + default: + return false, nil + } +} + +func podMatchesScope(requirement corev1.ScopedResourceSelectorRequirement, pod *corev1.Pod) (bool, error) { + switch requirement.ScopeName { + case corev1.ResourceQuotaScopeTerminating: + return isTerminating(pod), nil + case corev1.ResourceQuotaScopeNotTerminating: + return !isTerminating(pod), nil + case corev1.ResourceQuotaScopeBestEffort: + return podQOS(pod) == corev1.PodQOSBestEffort, nil + case corev1.ResourceQuotaScopeNotBestEffort: + return podQOS(pod) != corev1.PodQOSBestEffort, nil + case corev1.ResourceQuotaScopePriorityClass: + if requirement.Operator == corev1.ScopeSelectorOpExists { + return pod.Spec.PriorityClassName != "", nil + } + + return requirementMatches(requirement, []string{pod.Spec.PriorityClassName}) + case corev1.ResourceQuotaScopeCrossNamespacePodAffinity: + return usesCrossNamespacePodAffinity(pod), nil + case corev1.ResourceQuotaScopeVolumeAttributesClass: + return false, nil + default: + return false, nil + } +} + +func pvcMatchesScope( + requirement corev1.ScopedResourceSelectorRequirement, + pvc *corev1.PersistentVolumeClaim, +) (bool, error) { + if requirement.ScopeName != corev1.ResourceQuotaScopeVolumeAttributesClass { + return false, nil + } + + values := sets.New[string]() + if value := ptr.Deref(pvc.Spec.VolumeAttributesClassName, ""); value != "" { + values.Insert(value) + } + + if value := ptr.Deref(pvc.Status.CurrentVolumeAttributesClassName, ""); value != "" { + values.Insert(value) + } + + if pvc.Status.ModifyVolumeStatus != nil && pvc.Status.ModifyVolumeStatus.TargetVolumeAttributesClassName != "" { + values.Insert(pvc.Status.ModifyVolumeStatus.TargetVolumeAttributesClassName) + } + + if requirement.Operator == corev1.ScopeSelectorOpExists { + return values.Len() > 0, nil + } + + return requirementMatches(requirement, values.UnsortedList()) +} + +func requirementMatches( + requirement corev1.ScopedResourceSelectorRequirement, + values []string, +) (bool, error) { + operator, err := scopeSelectorOperator(requirement.Operator) + if err != nil { + return false, err + } + + labelRequirement, err := labels.NewRequirement( + string(requirement.ScopeName), + operator, + requirement.Values, + ) + if err != nil { + return false, err + } + + if len(values) == 0 { + return labelRequirement.Matches(labels.Set{}), nil + } + + for _, value := range values { + if labelRequirement.Matches(labels.Set{string(requirement.ScopeName): value}) { + return true, nil + } + } + + return false, nil +} + +func scopeSelectorOperator(operator corev1.ScopeSelectorOperator) (selection.Operator, error) { + switch operator { + case corev1.ScopeSelectorOpIn: + return selection.In, nil + case corev1.ScopeSelectorOpNotIn: + return selection.NotIn, nil + case corev1.ScopeSelectorOpExists: + return selection.Exists, nil + case corev1.ScopeSelectorOpDoesNotExist: + return selection.DoesNotExist, nil + default: + return "", fmt.Errorf("unsupported scope selector operator %q", operator) + } +} + +// ValidateConstraints preserves Kubernetes' historical requirement that every +// container explicitly sets CPU/memory resources when those resources are +// quota-controlled. +func ValidateConstraints(hard corev1.ResourceList, object runtime.Object) error { + pod, ok := object.(*corev1.Pod) + if !ok { + return nil + } + + // Kubernetes skips the legacy per-container presence check when a supported + // Pod-level request or limit is set. Disabled Pod-level fields are removed by + // the API server before validating admission webhooks receive the Pod. + if resourcehelper.IsPodLevelResourcesSet(pod) { + return nil + } + + required := sets.New[corev1.ResourceName]() + + for name := range hard { + if validationResources.Has(name) { + required.Insert(name) + } + } + + missing := map[corev1.ResourceName][]string{} + + containers := append(append([]corev1.Container{}, pod.Spec.Containers...), pod.Spec.InitContainers...) + + for _, container := range containers { + usage := podComputeUsage(container.Resources.Requests, container.Resources.Limits) + for name := range required { + if _, ok := usage[name]; !ok { + missing[name] = append(missing[name], container.Name) + } + } + } + + if len(missing) == 0 { + return nil + } + + parts := make([]string, 0, len(missing)) + for _, name := range sets.List(sets.KeySet(missing)) { + parts = append(parts, fmt.Sprintf("%s for: %s", name, strings.Join(missing[name], ","))) + } + + return fmt.Errorf("must specify %s", strings.Join(parts, "; ")) +} + +func quotaPod(pod *corev1.Pod, now time.Time) bool { + if pod.Status.Phase == corev1.PodFailed || pod.Status.Phase == corev1.PodSucceeded { + return false + } + + if pod.DeletionTimestamp != nil && pod.DeletionGracePeriodSeconds != nil { + deadline := pod.DeletionTimestamp.Add(time.Duration(*pod.DeletionGracePeriodSeconds) * time.Second) + if now.After(deadline) { + return false + } + } + + return true +} + +func isTerminating(pod *corev1.Pod) bool { + return pod.Spec.ActiveDeadlineSeconds != nil && *pod.Spec.ActiveDeadlineSeconds >= 0 +} + +func podQOS(pod *corev1.Pod) corev1.PodQOSClass { + if pod.Status.QOSClass != "" { + return pod.Status.QOSClass + } + + requests := corev1.ResourceList{} + limits := corev1.ResourceList{} + guaranteed := true + + process := func(target corev1.ResourceList, resources corev1.ResourceList) { + for _, name := range []corev1.ResourceName{corev1.ResourceCPU, corev1.ResourceMemory} { + if quantity, ok := resources[name]; ok && quantity.Sign() > 0 { + addQuantity(target, name, quantity) + } + } + } + hasQoSLimits := func(resources corev1.ResourceList) bool { + for _, name := range []corev1.ResourceName{corev1.ResourceCPU, corev1.ResourceMemory} { + quantity, ok := resources[name] + if !ok || quantity.Sign() <= 0 { + return false + } + } + + return true + } + + if pod.Spec.Resources != nil { + process(requests, pod.Spec.Resources.Requests) + process(limits, pod.Spec.Resources.Limits) + guaranteed = hasQoSLimits(pod.Spec.Resources.Limits) + } else { + containers := append(append([]corev1.Container{}, pod.Spec.Containers...), pod.Spec.InitContainers...) + for _, container := range containers { + process(requests, container.Resources.Requests) + process(limits, container.Resources.Limits) + + if !hasQoSLimits(container.Resources.Limits) { + guaranteed = false + } + } + } + + if len(requests) == 0 && len(limits) == 0 { + return corev1.PodQOSBestEffort + } + + if guaranteed && len(requests) == len(limits) { + for name, request := range requests { + if limit, ok := limits[name]; !ok || request.Cmp(limit) != 0 { + guaranteed = false + + break + } + } + } + + if guaranteed && len(requests) == len(limits) { + return corev1.PodQOSGuaranteed + } + + return corev1.PodQOSBurstable +} + +func addQuantity(target corev1.ResourceList, name corev1.ResourceName, quantity resource.Quantity) { + current := target[name] + current.Add(quantity) + target[name] = current +} + +func addResourceList(target, addition corev1.ResourceList) { + for name, quantity := range addition { + addQuantity(target, name, quantity) + } +} + +func isExtendedResourceName(name corev1.ResourceName) bool { + value := string(name) + + return strings.Contains(value, "/") && !strings.Contains(value, corev1.ResourceDefaultNamespacePrefix) +} + +func crossNamespacePodAffinityTerm(term corev1.PodAffinityTerm) bool { + return len(term.Namespaces) != 0 || term.NamespaceSelector != nil +} + +func usesCrossNamespacePodAffinity(pod *corev1.Pod) bool { + if pod.Spec.Affinity == nil { + return false + } + + check := func(terms []corev1.PodAffinityTerm, weighted []corev1.WeightedPodAffinityTerm) bool { + return slices.ContainsFunc(terms, crossNamespacePodAffinityTerm) || + slices.ContainsFunc(weighted, func(term corev1.WeightedPodAffinityTerm) bool { + return crossNamespacePodAffinityTerm(term.PodAffinityTerm) + }) + } + + if affinity := pod.Spec.Affinity.PodAffinity; affinity != nil && + check(affinity.RequiredDuringSchedulingIgnoredDuringExecution, affinity.PreferredDuringSchedulingIgnoredDuringExecution) { + return true + } + + if affinity := pod.Spec.Affinity.PodAntiAffinity; affinity != nil && + check(affinity.RequiredDuringSchedulingIgnoredDuringExecution, affinity.PreferredDuringSchedulingIgnoredDuringExecution) { + return true + } + + return false +} diff --git a/internal/quota/evaluator/evaluator_test.go b/internal/quota/evaluator/evaluator_test.go new file mode 100644 index 00000000..d59790a3 --- /dev/null +++ b/internal/quota/evaluator/evaluator_test.go @@ -0,0 +1,471 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package evaluator + +import ( + "encoding/json" + "testing" + + admissionv1 "k8s.io/api/admission/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +func TestEvaluatePodUsesUpstreamResourceCalculation(t *testing.T) { + t.Parallel() + + pod := &corev1.Pod{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: "app", + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1"), + corev1.ResourceMemory: resource.MustParse("1Gi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("2"), + corev1.ResourceMemory: resource.MustParse("2Gi"), + }, + }, + }}, + InitContainers: []corev1.Container{{ + Name: "init", + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("3"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("4"), + }, + }, + }}, + Overhead: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("100m"), + }, + }, + } + + result, handled, err := Evaluate(requestFor(t, admissionv1.Create, "pods", "Pod", pod, nil)) + if err != nil { + t.Fatalf("Evaluate() error = %v", err) + } + if !handled { + t.Fatal("Evaluate() did not handle Pod") + } + + assertQuantity(t, result.NewUsage, corev1.ResourceRequestsCPU, "3100m") + assertQuantity(t, result.NewUsage, corev1.ResourceLimitsCPU, "4100m") + assertQuantity(t, result.NewUsage, corev1.ResourceRequestsMemory, "1Gi") + assertQuantity(t, result.NewUsage, corev1.ResourceLimitsMemory, "2Gi") + assertQuantity(t, result.NewUsage, corev1.ResourcePods, "1") + assertQuantity(t, result.NewUsage, corev1.ResourceName("count/pods"), "1") +} + +func TestEvaluatePodUsesEphemeralStorageResources(t *testing.T) { + t.Parallel() + + pod := &corev1.Pod{Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: "app", + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceEphemeralStorage: resource.MustParse("1Gi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceEphemeralStorage: resource.MustParse("2Gi"), + }, + }, + }}, + InitContainers: []corev1.Container{{ + Name: "init", + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceEphemeralStorage: resource.MustParse("3Gi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceEphemeralStorage: resource.MustParse("4Gi"), + }, + }, + }}, + Overhead: corev1.ResourceList{ + corev1.ResourceEphemeralStorage: resource.MustParse("500Mi"), + }, + }} + + result, handled, err := Evaluate(requestFor(t, admissionv1.Create, "pods", "Pod", pod, nil)) + if err != nil { + t.Fatalf("Evaluate() error = %v", err) + } + if !handled { + t.Fatal("Evaluate() did not handle Pod") + } + + assertQuantity(t, result.NewUsage, corev1.ResourceEphemeralStorage, "3572Mi") + assertQuantity(t, result.NewUsage, corev1.ResourceRequestsEphemeralStorage, "3572Mi") + assertQuantity(t, result.NewUsage, corev1.ResourceLimitsEphemeralStorage, "4596Mi") +} + +func TestEvaluateTerminalPodOnlyConsumesObjectCount(t *testing.T) { + t.Parallel() + + pod := &corev1.Pod{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: "app", + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceEphemeralStorage: resource.MustParse("1Gi"), + }, + }, + }}, + }, + Status: corev1.PodStatus{Phase: corev1.PodSucceeded}, + } + + result, handled, err := Evaluate(requestFor(t, admissionv1.Create, "pods", "Pod", pod, nil)) + if err != nil { + t.Fatalf("Evaluate() error = %v", err) + } + if !handled { + t.Fatal("Evaluate() did not handle Pod") + } + + assertQuantity(t, result.NewUsage, corev1.ResourceName("count/pods"), "1") + for _, name := range []corev1.ResourceName{ + corev1.ResourcePods, + corev1.ResourceEphemeralStorage, + corev1.ResourceRequestsEphemeralStorage, + } { + if _, found := result.NewUsage[name]; found { + t.Fatalf("terminal Pod unexpectedly consumed %q", name) + } + } +} + +func TestEvaluatePodUsesPodLevelResourceRequests(t *testing.T) { + t.Parallel() + + pod := &corev1.Pod{Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Name: "app"}}, + Resources: &corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("600m"), + corev1.ResourceMemory: resource.MustParse("512Mi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("800m"), + corev1.ResourceMemory: resource.MustParse("1Gi"), + }, + }, + }} + + result, handled, err := Evaluate(requestFor(t, admissionv1.Create, "pods", "Pod", pod, nil)) + if err != nil { + t.Fatalf("Evaluate() error = %v", err) + } + if !handled { + t.Fatal("Evaluate() did not handle Pod") + } + + assertQuantity(t, result.NewUsage, corev1.ResourceRequestsCPU, "600m") + assertQuantity(t, result.NewUsage, corev1.ResourceRequestsMemory, "512Mi") + assertQuantity(t, result.NewUsage, corev1.ResourceLimitsCPU, "800m") + assertQuantity(t, result.NewUsage, corev1.ResourceLimitsMemory, "1Gi") +} + +func TestValidateConstraintsAllowsPodLevelResources(t *testing.T) { + t.Parallel() + + hard := corev1.ResourceList{ + corev1.ResourceRequestsCPU: resource.MustParse("8"), + corev1.ResourceRequestsMemory: resource.MustParse("16Gi"), + corev1.ResourceLimitsCPU: resource.MustParse("8"), + corev1.ResourceLimitsMemory: resource.MustParse("16Gi"), + } + pod := &corev1.Pod{Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: "nginx", + Resources: corev1.ResourceRequirements{}, + }}, + Resources: &corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("100m"), + corev1.ResourceMemory: resource.MustParse("256Mi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1"), + corev1.ResourceMemory: resource.MustParse("1Gi"), + }, + }, + }} + + if err := ValidateConstraints(hard, pod); err != nil { + t.Fatalf("ValidateConstraints() rejected Pod-level resources: %v", err) + } +} + +func TestValidateConstraintsDoesNotTreatUnsupportedPodLevelResourcesAsCompute(t *testing.T) { + t.Parallel() + + hard := corev1.ResourceList{ + corev1.ResourceRequestsCPU: resource.MustParse("8"), + } + pod := &corev1.Pod{Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: "nginx", + Resources: corev1.ResourceRequirements{}, + }}, + Resources: &corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceEphemeralStorage: resource.MustParse("1Gi"), + }, + }, + }} + + err := ValidateConstraints(hard, pod) + if err == nil { + t.Fatal("ValidateConstraints() accepted unsupported Pod-level resources") + } + if got, want := err.Error(), "must specify requests.cpu for: nginx"; got != want { + t.Fatalf("ValidateConstraints() error = %q, want %q", got, want) + } +} + +func TestEvaluateServiceUpdateReturnsOldAndNewUsage(t *testing.T) { + t.Parallel() + + oldService := &corev1.Service{Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeClusterIP, + Ports: []corev1.ServicePort{{Port: 80}}, + }} + newService := oldService.DeepCopy() + newService.Spec.Type = corev1.ServiceTypeLoadBalancer + newService.Spec.Ports = append(newService.Spec.Ports, corev1.ServicePort{Port: 443}) + + result, handled, err := Evaluate(requestFor(t, admissionv1.Update, "services", "Service", newService, oldService)) + if err != nil { + t.Fatalf("Evaluate() error = %v", err) + } + if !handled { + t.Fatal("Evaluate() did not handle Service") + } + + assertQuantity(t, result.OldUsage, corev1.ResourceServicesLoadBalancers, "0") + assertQuantity(t, result.OldUsage, corev1.ResourceServices, "1") + assertQuantity(t, result.OldUsage, corev1.ResourceName("count/services"), "1") + assertQuantity(t, result.NewUsage, corev1.ResourceServicesLoadBalancers, "1") + assertQuantity(t, result.NewUsage, corev1.ResourceServicesNodePorts, "2") +} + +func TestEvaluateObjectCountNames(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + group string + version string + resource string + kind string + expected corev1.ResourceName + legacyName corev1.ResourceName + }{ + { + name: "core resource has generic and legacy count", + version: "v1", + resource: "configmaps", + kind: "ConfigMap", + expected: corev1.ResourceName("count/configmaps"), + legacyName: corev1.ResourceConfigMaps, + }, + { + name: "grouped resource has qualified generic count", + group: "apps", + version: "v1", + resource: "deployments", + kind: "Deployment", + expected: corev1.ResourceName("count/deployments.apps"), + }, + } + + for _, test := range tests { + test := test + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + object := &metav1.PartialObjectMetadata{ + ObjectMeta: metav1.ObjectMeta{Name: "example", Namespace: "tenant-a"}, + } + req := requestFor(t, admissionv1.Create, test.resource, test.kind, object, nil) + req.Resource = metav1.GroupVersionResource{ + Group: test.group, Version: test.version, Resource: test.resource, + } + req.Kind = metav1.GroupVersionKind{ + Group: test.group, Version: test.version, Kind: test.kind, + } + + result, handled, err := Evaluate(req) + if err != nil { + t.Fatalf("Evaluate() error = %v", err) + } + if !handled { + t.Fatalf("Evaluate() did not handle %s", test.kind) + } + + assertQuantity(t, result.NewUsage, test.expected, "1") + if test.legacyName != "" { + assertQuantity(t, result.NewUsage, test.legacyName, "1") + } + }) + } +} + +func TestEvaluateCountsHorizontalPodAutoscalers(t *testing.T) { + t.Parallel() + + hpa := &autoscalingv2.HorizontalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{Name: "example", Namespace: "tenant-a"}, + Spec: autoscalingv2.HorizontalPodAutoscalerSpec{ + ScaleTargetRef: autoscalingv2.CrossVersionObjectReference{ + APIVersion: "apps/v1", + Kind: "Deployment", + Name: "example", + }, + MaxReplicas: 3, + }, + } + req := requestFor( + t, + admissionv1.Create, + "horizontalpodautoscalers", + "HorizontalPodAutoscaler", + hpa, + nil, + ) + req.Resource = metav1.GroupVersionResource{ + Group: "autoscaling", Version: "v2", Resource: "horizontalpodautoscalers", + } + req.Kind = metav1.GroupVersionKind{ + Group: "autoscaling", Version: "v2", Kind: "HorizontalPodAutoscaler", + } + + result, handled, err := Evaluate(req) + if err != nil { + t.Fatalf("Evaluate() error = %v", err) + } + if !handled { + t.Fatal("Evaluate() did not handle HorizontalPodAutoscaler") + } + + assertQuantity( + t, + result.NewUsage, + corev1.ResourceName("count/horizontalpodautoscalers.autoscaling"), + "1", + ) +} + +func TestMatchesPodQuotaScopes(t *testing.T) { + t.Parallel() + + priority := "high" + pod := &corev1.Pod{Spec: corev1.PodSpec{PriorityClassName: priority}} + spec := corev1.ResourceQuotaSpec{ScopeSelector: &corev1.ScopeSelector{ + MatchExpressions: []corev1.ScopedResourceSelectorRequirement{{ + ScopeName: corev1.ResourceQuotaScopePriorityClass, + Operator: corev1.ScopeSelectorOpIn, + Values: []string{"high"}, + }}, + }} + + matches, err := MatchesScopes(spec, pod) + if err != nil { + t.Fatalf("MatchesScopes() error = %v", err) + } + if !matches { + t.Fatal("MatchesScopes() = false, want true") + } +} + +func TestMatchesBestEffortScopeUsesPodLevelResources(t *testing.T) { + t.Parallel() + + pod := &corev1.Pod{Spec: corev1.PodSpec{ + Resources: &corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1"), + corev1.ResourceMemory: resource.MustParse("1Gi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1"), + corev1.ResourceMemory: resource.MustParse("1Gi"), + }, + }, + }} + spec := corev1.ResourceQuotaSpec{Scopes: []corev1.ResourceQuotaScope{ + corev1.ResourceQuotaScopeBestEffort, + }} + + matches, err := MatchesScopes(spec, pod) + if err != nil { + t.Fatalf("MatchesScopes() error = %v", err) + } + if matches { + t.Fatal("pod-level resources were classified as BestEffort") + } +} + +func requestFor( + t *testing.T, + operation admissionv1.Operation, + resourceName string, + kind string, + object runtime.Object, + old runtime.Object, +) admission.Request { + t.Helper() + + raw, err := json.Marshal(object) + if err != nil { + t.Fatal(err) + } + + var oldRaw []byte + if old != nil { + oldRaw, err = json.Marshal(old) + if err != nil { + t.Fatal(err) + } + } + + return admission.Request{AdmissionRequest: admissionv1.AdmissionRequest{ + Operation: operation, + Resource: metav1.GroupVersionResource{Group: "", Version: "v1", Resource: resourceName}, + Kind: metav1.GroupVersionKind{Group: "", Version: "v1", Kind: kind}, + Object: runtime.RawExtension{Raw: raw}, + OldObject: runtime.RawExtension{Raw: oldRaw}, + RequestKind: &metav1.GroupVersionKind{ + Group: "", Version: "v1", Kind: kind, + }, + RequestResource: &metav1.GroupVersionResource{ + Group: "", Version: "v1", Resource: resourceName, + }, + }} +} + +func assertQuantity(t *testing.T, list corev1.ResourceList, name corev1.ResourceName, want string) { + t.Helper() + + got, ok := list[name] + if !ok { + t.Fatalf("resource %q is missing from %#v", name, list) + } + if got.Cmp(resource.MustParse(want)) != 0 { + t.Fatalf("resource %q = %s, want %s", name, got.String(), want) + } +} diff --git a/internal/webhook/globalresourcequota/calculation.go b/internal/webhook/globalresourcequota/calculation.go new file mode 100644 index 00000000..c296d36c --- /dev/null +++ b/internal/webhook/globalresourcequota/calculation.go @@ -0,0 +1,745 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package globalresourcequota + +import ( + "context" + "encoding/json" + "fmt" + "slices" + "strings" + "time" + + admissionv1 "k8s.io/api/admission/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/util/retry" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + quotaevaluator "github.com/projectcapsule/capsule/internal/quota/evaluator" + "github.com/projectcapsule/capsule/pkg/api/meta" + ad "github.com/projectcapsule/capsule/pkg/runtime/admission" + "github.com/projectcapsule/capsule/pkg/runtime/configuration" + "github.com/projectcapsule/capsule/pkg/runtime/events" + "github.com/projectcapsule/capsule/pkg/runtime/handlers" + runtimequota "github.com/projectcapsule/capsule/pkg/runtime/quota" +) + +const maxReservations = 1024 + +var ledgerBackoff = wait.Backoff{ + Steps: 8, + Duration: 10 * time.Millisecond, + Factor: 1.6, + Jitter: 0.2, +} + +type handler struct{} + +type appliedReservation struct { + Key types.NamespacedName + ID string +} + +func Handler() handlers.Handler { + return &handler{} +} + +func (h *handler) OnCreate( + c client.Client, + reader client.Reader, + _ admission.Decoder, + _ events.EventRecorder, +) handlers.Func { + return h.handle(c, reader) +} + +func (h *handler) OnUpdate( + c client.Client, + reader client.Reader, + _ admission.Decoder, + _ events.EventRecorder, +) handlers.Func { + return h.handle(c, reader) +} + +func (h *handler) OnDelete( + client.Client, + client.Reader, + admission.Decoder, + events.EventRecorder, +) handlers.Func { + return func(context.Context, admission.Request) *admission.Response { + return nil + } +} + +func (h *handler) handle(c client.Client, reader client.Reader) handlers.Func { + return func(ctx context.Context, req admission.Request) *admission.Response { + if isGlobalResourceQuotaRequest(req) { + return validateGlobalResourceQuotaRequest(ctx, reader, req) + } + + return enforceGlobalResourceQuotaRequest(ctx, c, reader, req) + } +} + +func validateGlobalResourceQuotaRequest( + ctx context.Context, + reader client.Reader, + req admission.Request, +) *admission.Response { + quota := &capsulev1beta2.GlobalResourceQuota{} + if err := json.Unmarshal(req.Object.Raw, quota); err != nil { + return ad.Denyf("GlobalResourceQuota could not be decoded: %v", err) + } + + if err := validateGlobalResourceQuota(quota); err != nil { + return ad.Denyf("invalid GlobalResourceQuota: %v", err) + } + + if req.Operation != admissionv1.Update { + return nil + } + + oldQuota := &capsulev1beta2.GlobalResourceQuota{} + if err := json.Unmarshal(req.OldObject.Raw, oldQuota); err != nil { + return ad.Denyf("previous GlobalResourceQuota could not be decoded: %v", err) + } + + if err := validateHardLimit(quota.Spec.Quota.Hard, oldQuota.Status.Total.Used); err != nil { + return ad.Denyf("invalid GlobalResourceQuota: %v", err) + } + + ledger := &capsulev1beta2.QuantityLedger{} + + err := reader.Get(ctx, types.NamespacedName{ + Namespace: configuration.ControllerNamespace(), + Name: oldQuota.GetLedgerName(), + }, ledger) + + switch { + case apierrors.IsNotFound(err): + return nil + case err != nil: + return ad.ErroredResponse(err) + case ledger.Spec.TargetRef.UID != oldQuota.UID || ledger.Status.ResourceQuota == nil: + return nil + } + + if err := validateHardLimit( + quota.Spec.Quota.Hard, + ledger.Status.ResourceQuota.Allocated, + ); err != nil { + return ad.Denyf("invalid GlobalResourceQuota: %v", err) + } + + return nil +} + +func enforceGlobalResourceQuotaRequest( + ctx context.Context, + c client.Client, + reader client.Reader, + req admission.Request, +) *admission.Response { + if req.Namespace == "" { + return nil + } + + namespace := &corev1.Namespace{} + if err := c.Get(ctx, client.ObjectKey{Name: req.Namespace}, namespace); err != nil { + if apierrors.IsNotFound(err) { + return nil + } + + return ad.ErroredResponse(err) + } + + allQuotas := &capsulev1beta2.GlobalResourceQuotaList{} + if err := c.List(ctx, allQuotas); err != nil { + return ad.ErroredResponse(err) + } + + quotaList, err := matchingGlobalResourceQuotas(namespace, allQuotas.Items) + if err != nil { + return ad.ErroredResponse(err) + } + + if len(quotaList) == 0 { + return nil + } + + evaluation, handled, err := quotaevaluator.Evaluate(req) + if err != nil { + return ad.Denyf("GlobalResourceQuota usage could not be calculated: %v", err) + } + + if !handled || isManagedResourceQuota(req, evaluation.New) { + // Native ResourceQuotas are implementation details. Counting their + // creation could prevent the quota authorizing them from initializing. + return nil + } + + applied := make([]appliedReservation, 0, len(quotaList)) + + for _, quota := range quotaList { + reservation, response := reserveForGlobalResourceQuota(ctx, c, reader, req, quota, evaluation) + if response != nil { + rollbackReservations(ctx, c, reader, applied) + + return response + } + + if reservation != nil { + applied = append(applied, *reservation) + } + } + + return nil +} + +func reserveForGlobalResourceQuota( + ctx context.Context, + c client.Client, + reader client.Reader, + req admission.Request, + quota *capsulev1beta2.GlobalResourceQuota, + evaluation quotaevaluator.Result, +) (*appliedReservation, *admission.Response) { + if quota.DeletionTimestamp != nil { + return nil, nil + } + + oldUsage, newUsage, err := usageForQuota(quota.Spec.Quota, evaluation) + if err != nil { + return nil, ad.Denyf( + "resource cannot be evaluated against GlobalResourceQuota %q: %v", + quota.Name, + err, + ) + } + + if !resourceListPositive(newUsage) && !resourceListPositive(oldUsage) { + return nil, nil + } + + delta := positiveDifference(newUsage, oldUsage) + if !resourceListPositive(delta) { + return nil, nil + } + + ledgerKey := types.NamespacedName{ + Namespace: configuration.ControllerNamespace(), + Name: quota.GetLedgerName(), + } + reservation := newReservation(req, quota.Name, newUsage, delta) + + allowed, projected, applied, err := reserve( + ctx, + c, + reader, + ledgerKey, + quota, + reservation, + req.DryRun != nil && *req.DryRun, + ) + if err != nil { + if apierrors.IsNotFound(err) { + return nil, ad.Denyf( + "GlobalResourceQuota %q is not ready: QuantityLedger %s does not exist", + quota.Name, + ledgerKey.String(), + ) + } + + return nil, ad.ErroredResponse(err) + } + + if !allowed { + return nil, ad.Denyf( + "resource exceeds GlobalResourceQuota %q: %s", + quota.Name, + formatExceededResources(delta, projected, quota.Spec.Quota.Hard), + ) + } + + if !applied || (req.DryRun != nil && *req.DryRun) { + return nil, nil + } + + return &appliedReservation{Key: ledgerKey, ID: reservation.ID}, nil +} + +func isGlobalResourceQuotaRequest(req admission.Request) bool { + return req.Resource.Group == capsulev1beta2.GroupVersion.Group && + req.Resource.Resource == "globalresourcequotas" && + req.SubResource == "" +} + +func validateGlobalResourceQuota(quota *capsulev1beta2.GlobalResourceQuota) error { + if len(quota.Spec.Quota.Hard) == 0 { + return fmt.Errorf("spec.quota.hard must contain at least one resource") + } + + for name, quantity := range quota.Spec.Quota.Hard { + if quantity.Sign() < 0 { + return fmt.Errorf("spec.quota.hard[%q] must not be negative", name) + } + } + + for index, namespaceSelector := range quota.Spec.NamespaceSelectors { + if namespaceSelector.LabelSelector == nil { + continue + } + + if _, err := metav1.LabelSelectorAsSelector(namespaceSelector.LabelSelector); err != nil { + return fmt.Errorf("spec.namespaceSelectors[%d] is invalid: %w", index, err) + } + } + + return nil +} + +func validateHardLimit(hard, allocated corev1.ResourceList) error { + for name, usage := range allocated { + if usage.Sign() <= 0 { + continue + } + + limit, exists := hard[name] + if !exists { + return fmt.Errorf( + "spec.quota.hard[%q] cannot be removed while %s is allocated", + name, + usage.String(), + ) + } + + if limit.Cmp(usage) < 0 { + return fmt.Errorf( + "spec.quota.hard[%q] cannot be reduced to %s while %s is allocated", + name, + limit.String(), + usage.String(), + ) + } + } + + return nil +} + +func isManagedResourceQuota(req admission.Request, object any) bool { + if req.Resource.Group != "" || req.Resource.Resource != "resourcequotas" { + return false + } + + metadata, ok := object.(metav1.Object) + if !ok { + return false + } + + objectLabels := metadata.GetLabels() + + return objectLabels[meta.NewManagedByCapsuleLabel] == meta.ValueController && + objectLabels[meta.GlobalResourceQuotaLabel] != "" +} + +func matchingGlobalResourceQuotas( + namespace *corev1.Namespace, + quotas []capsulev1beta2.GlobalResourceQuota, +) ([]*capsulev1beta2.GlobalResourceQuota, error) { + out := make([]*capsulev1beta2.GlobalResourceQuota, 0) + namespaceLabels := labels.Set(namespace.Labels) + + for i := range quotas { + quota := "as[i] + + for _, namespaceSelector := range quota.Spec.NamespaceSelectors { + if namespaceSelector.LabelSelector == nil { + continue + } + + selector, err := metav1.LabelSelectorAsSelector(namespaceSelector.LabelSelector) + if err != nil { + return nil, fmt.Errorf("GlobalResourceQuota %q has an invalid namespace selector: %w", quota.Name, err) + } + + if selector.Matches(namespaceLabels) { + out = append(out, quota) + + break + } + } + } + + return out, nil +} + +func usageForQuota( + spec corev1.ResourceQuotaSpec, + evaluation quotaevaluator.Result, +) (corev1.ResourceList, corev1.ResourceList, error) { + newUsage := corev1.ResourceList{} + oldUsage := corev1.ResourceList{} + + if evaluation.New != nil { + matches, err := quotaevaluator.MatchesScopes(spec, evaluation.New) + if err != nil { + return nil, nil, err + } + + if matches { + if err := quotaevaluator.ValidateConstraints(spec.Hard, evaluation.New); err != nil { + return nil, nil, err + } + + newUsage = maskResourceList(evaluation.NewUsage, spec.Hard) + } + } + + if evaluation.Old != nil { + matches, err := quotaevaluator.MatchesScopes(spec, evaluation.Old) + if err != nil { + return nil, nil, err + } + + if matches { + oldUsage = maskResourceList(evaluation.OldUsage, spec.Hard) + } + } + + return oldUsage, newUsage, nil +} + +func reserve( + ctx context.Context, + c client.Client, + reader client.Reader, + key types.NamespacedName, + quota *capsulev1beta2.GlobalResourceQuota, + reservation capsulev1beta2.QuantityLedgerResourceQuotaReservation, + dryRun bool, +) (allowed bool, allocated corev1.ResourceList, applied bool, err error) { + hard := quota.Spec.Quota.Hard + + err = retry.RetryOnConflict(ledgerBackoff, func() error { + applied = false + + ledger := &capsulev1beta2.QuantityLedger{} + if getErr := reader.Get(ctx, key, ledger); getErr != nil { + return getErr + } + + target := ledger.Spec.TargetRef + if target.Kind != "GlobalResourceQuota" || + target.Name != quota.Name || + target.UID != quota.UID { + return fmt.Errorf("QuantityLedger %s has a stale GlobalResourceQuota target", key.String()) + } + + if ledger.Status.ResourceQuota == nil || !ledger.Status.ResourceQuota.Initialized { + return fmt.Errorf("GlobalResourceQuota QuantityLedger %s is not initialized", key.String()) + } + + if ledger.Status.ResourceQuota.ObservedGeneration != quota.Generation { + return fmt.Errorf( + "GlobalResourceQuota QuantityLedger %s has not observed generation %d", + key.String(), + quota.Generation, + ) + } + + if !slices.Contains(ledger.Status.ResourceQuota.Namespaces, reservation.ObjectRef.Namespace) { + return fmt.Errorf( + "GlobalResourceQuota QuantityLedger %s has not observed namespace %s", + key.String(), + reservation.ObjectRef.Namespace, + ) + } + + now := metav1.Now() + active := make([]capsulev1beta2.QuantityLedgerResourceQuotaReservation, 0, len(ledger.Status.ResourceQuota.Reservations)+1) + found := false + + for _, existing := range ledger.Status.ResourceQuota.Reservations { + if existing.ExpiresAt != nil && existing.ExpiresAt.Before(&now) { + continue + } + + if existing.ID == reservation.ID { + found = true + applied = true + existing.Usage = reservation.Usage.DeepCopy() + existing.Delta = reservation.Delta.DeepCopy() + existing.ObjectRef = reservation.ObjectRef + existing.UpdatedAt = now + existing.ExpiresAt = reservation.ExpiresAt + } + + active = append(active, existing) + } + + if !found { + if len(active) >= maxReservations { + return fmt.Errorf("GlobalResourceQuota QuantityLedger %s has too many inflight reservations", key.String()) + } + + active = append(active, reservation) + applied = true + } + + reserved := sumReservations(active, hard) + next := ledger.Status.ResourceQuota.Used.DeepCopy() + addResourceList(next, reserved) + allocated = next.DeepCopy() + + if exceeds(next, hard) { + allowed = false + applied = false + + return nil + } + + if dryRun { + allowed = true + applied = false + + return nil + } + + ledger.Status.ResourceQuota.Reservations = active + ledger.Status.ResourceQuota.Reserved = reserved + ledger.Status.ResourceQuota.Allocated = next + + if updateErr := c.Status().Update(ctx, ledger); updateErr != nil { + applied = false + + return updateErr + } + + allowed = true + + return nil + }) + + return allowed, allocated, applied, err +} + +func rollbackReservations( + ctx context.Context, + c client.Client, + reader client.Reader, + applied []appliedReservation, +) { + for _, item := range slices.Backward(applied) { + _ = rollbackReservation(ctx, c, reader, item.Key, item.ID) + } +} + +func rollbackReservation( + ctx context.Context, + c client.Client, + reader client.Reader, + key types.NamespacedName, + id string, +) error { + return retry.RetryOnConflict(retry.DefaultBackoff, func() error { + ledger := &capsulev1beta2.QuantityLedger{} + if err := reader.Get(ctx, key, ledger); err != nil { + if apierrors.IsNotFound(err) { + return nil + } + + return err + } + + if ledger.Status.ResourceQuota == nil { + return nil + } + + active := make([]capsulev1beta2.QuantityLedgerResourceQuotaReservation, 0, len(ledger.Status.ResourceQuota.Reservations)) + removed := false + + for _, reservation := range ledger.Status.ResourceQuota.Reservations { + if reservation.ID == id { + removed = true + + continue + } + + active = append(active, reservation) + } + + if !removed { + return nil + } + + reserved := sumReservations(active, ledger.Status.ResourceQuota.Used) + allocated := ledger.Status.ResourceQuota.Used.DeepCopy() + addResourceList(allocated, reserved) + + ledger.Status.ResourceQuota.Reservations = active + ledger.Status.ResourceQuota.Reserved = reserved + ledger.Status.ResourceQuota.Allocated = allocated + + return c.Status().Update(ctx, ledger) + }) +} + +func newReservation( + req admission.Request, + quotaKey string, + usage corev1.ResourceList, + delta corev1.ResourceList, +) capsulev1beta2.QuantityLedgerResourceQuotaReservation { + now := metav1.Now() + expires := metav1.NewTime(now.Add(2 * time.Minute)) + + return capsulev1beta2.QuantityLedgerResourceQuotaReservation{ + ID: fmt.Sprintf("%s/%s", req.UID, quotaKey), + ObjectRef: capsulev1beta2.QuantityLedgerObjectRef{ + APIGroup: req.Kind.Group, + APIVersion: req.Kind.Version, + Kind: req.Kind.Kind, + Namespace: req.Namespace, + Name: req.Name, + }, + Usage: usage.DeepCopy(), + Delta: delta.DeepCopy(), + CreatedAt: now, + UpdatedAt: now, + ExpiresAt: &expires, + } +} + +func maskResourceList(usage, hard corev1.ResourceList) corev1.ResourceList { + out := make(corev1.ResourceList) + + for name := range hard { + if quantity, ok := usage[name]; ok { + out[name] = quantity.DeepCopy() + } + } + + return out +} + +func positiveDifference(next, previous corev1.ResourceList) corev1.ResourceList { + out := make(corev1.ResourceList) + + for name, quantity := range next { + delta := quantity.DeepCopy() + delta.Sub(previous[name]) + runtimequota.ClampQuantityToZero(&delta) + out[name] = delta + } + + return out +} + +func sumReservations( + reservations []capsulev1beta2.QuantityLedgerResourceQuotaReservation, + resources corev1.ResourceList, +) corev1.ResourceList { + out := zeroResourceList(resources) + for _, reservation := range reservations { + addResourceList(out, reservation.Delta) + } + + return out +} + +func zeroResourceList(resources corev1.ResourceList) corev1.ResourceList { + out := make(corev1.ResourceList, len(resources)) + for name := range resources { + out[name] = *resource.NewQuantity(0, resource.DecimalSI) + } + + return out +} + +func addResourceList(target, addition corev1.ResourceList) { + for name, quantity := range addition { + current := target[name] + current.Add(quantity) + target[name] = current + } +} + +func exceeds(usage, hard corev1.ResourceList) bool { + for name, limit := range hard { + quantity := usage[name] + if quantity.Cmp(limit) > 0 { + return true + } + } + + return false +} + +func resourceListPositive(resources corev1.ResourceList) bool { + for _, quantity := range resources { + if quantity.Sign() > 0 { + return true + } + } + + return false +} + +func formatExceededResources(requested, projected, hard corev1.ResourceList) string { + names := make([]corev1.ResourceName, 0, len(hard)) + + for name, limit := range hard { + projectedQuantity := quantityForResource(projected, name) + if projectedQuantity.Cmp(limit) > 0 { + names = append(names, name) + } + } + + slices.Sort(names) + + details := make([]string, 0, len(names)) + + for _, name := range names { + requestedQuantity := quantityForResource(requested, name) + projectedQuantity := quantityForResource(projected, name) + hardQuantity := quantityForResource(hard, name) + + currentQuantity := projectedQuantity.DeepCopy() + currentQuantity.Sub(requestedQuantity) + runtimequota.ClampQuantityToZero(¤tQuantity) + + exceededBy := projectedQuantity.DeepCopy() + exceededBy.Sub(hardQuantity) + + details = append(details, fmt.Sprintf( + "%s (requested=%s, current=%s, projected=%s, hard=%s, exceededBy=%s)", + name, + requestedQuantity.String(), + currentQuantity.String(), + projectedQuantity.String(), + hardQuantity.String(), + exceededBy.String(), + )) + } + + return strings.Join(details, "; ") +} + +func quantityForResource(resources corev1.ResourceList, name corev1.ResourceName) resource.Quantity { + if quantity, found := resources[name]; found { + return quantity.DeepCopy() + } + + return *resource.NewQuantity(0, resource.DecimalSI) +} diff --git a/internal/webhook/globalresourcequota/calculation_test.go b/internal/webhook/globalresourcequota/calculation_test.go new file mode 100644 index 00000000..5ff2b395 --- /dev/null +++ b/internal/webhook/globalresourcequota/calculation_test.go @@ -0,0 +1,417 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package globalresourcequota + +import ( + "context" + "fmt" + "sync" + "sync/atomic" + "testing" + + admissionv1 "k8s.io/api/admission/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/runtime/selectors" +) + +func TestReserveIsAtomicAcrossResources(t *testing.T) { + t.Parallel() + + key := types.NamespacedName{Namespace: "capsule-system", Name: "rule-quota"} + hard := corev1.ResourceList{ + corev1.ResourceRequestsCPU: resource.MustParse("8"), + corev1.ResourceRequestsMemory: resource.MustParse("16Gi"), + } + quota := globalQuotaForTest("atomic", hard) + ledger := initializedLedger(key, quota, corev1.ResourceList{ + corev1.ResourceRequestsCPU: resource.MustParse("7"), + corev1.ResourceRequestsMemory: resource.MustParse("10Gi"), + }) + cl := ledgerClient(t, ledger) + + denied := reservationForTest("denied", corev1.ResourceList{ + corev1.ResourceRequestsCPU: resource.MustParse("1"), + corev1.ResourceRequestsMemory: resource.MustParse("7Gi"), + }) + allowed, _, applied, err := reserve(context.Background(), cl, cl, key, quota, denied, false) + if err != nil { + t.Fatalf("reserve(denied) error = %v", err) + } + if allowed || applied { + t.Fatalf("reserve(denied) = allowed %v, applied %v; want false, false", allowed, applied) + } + + current := &capsulev1beta2.QuantityLedger{} + if err := cl.Get(context.Background(), key, current); err != nil { + t.Fatal(err) + } + if len(current.Status.ResourceQuota.Reservations) != 0 { + t.Fatalf("denied reservation was persisted: %#v", current.Status.ResourceQuota.Reservations) + } + + accepted := reservationForTest("accepted", corev1.ResourceList{ + corev1.ResourceRequestsCPU: resource.MustParse("1"), + corev1.ResourceRequestsMemory: resource.MustParse("6Gi"), + }) + allowed, _, applied, err = reserve(context.Background(), cl, cl, key, quota, accepted, false) + if err != nil { + t.Fatalf("reserve(accepted) error = %v", err) + } + if !allowed || !applied { + t.Fatalf("reserve(accepted) = allowed %v, applied %v; want true, true", allowed, applied) + } +} + +func TestReserveReportsUpdatedReservationForRollback(t *testing.T) { + t.Parallel() + + key := types.NamespacedName{Namespace: "capsule-system", Name: "updated"} + hard := corev1.ResourceList{corev1.ResourceRequestsCPU: resource.MustParse("10")} + quota := globalQuotaForTest("updated", hard) + ledger := initializedLedger(key, quota, zeroResourceList(hard)) + existing := reservationForTest("same-admission", corev1.ResourceList{ + corev1.ResourceRequestsCPU: resource.MustParse("1"), + }) + ledger.Status.ResourceQuota.Reservations = []capsulev1beta2.QuantityLedgerResourceQuotaReservation{existing} + ledger.Status.ResourceQuota.Reserved = existing.Delta.DeepCopy() + ledger.Status.ResourceQuota.Allocated = existing.Delta.DeepCopy() + cl := ledgerClient(t, ledger) + + updated := reservationForTest("same-admission", corev1.ResourceList{ + corev1.ResourceRequestsCPU: resource.MustParse("2"), + }) + allowed, _, applied, err := reserve(context.Background(), cl, cl, key, quota, updated, false) + if err != nil { + t.Fatalf("reserve(updated) error = %v", err) + } + if !allowed || !applied { + t.Fatalf("reserve(updated) = allowed %v, applied %v; want true, true", allowed, applied) + } + persisted := &capsulev1beta2.QuantityLedger{} + if err := cl.Get(context.Background(), key, persisted); err != nil { + t.Fatal(err) + } + if len(persisted.Status.ResourceQuota.Reservations) != 1 { + t.Fatalf("stored reservations = %d, want 1", len(persisted.Status.ResourceQuota.Reservations)) + } + assertLedgerQuantity( + t, + persisted.Status.ResourceQuota.Reservations[0].Delta, + corev1.ResourceRequestsCPU, + "2", + ) + + if err := rollbackReservation(context.Background(), cl, cl, key, updated.ID); err != nil { + t.Fatalf("rollbackReservation(updated) error = %v", err) + } + + current := &capsulev1beta2.QuantityLedger{} + if err := cl.Get(context.Background(), key, current); err != nil { + t.Fatal(err) + } + if len(current.Status.ResourceQuota.Reservations) != 0 { + t.Fatalf("updated reservation was not rolled back: %#v", current.Status.ResourceQuota.Reservations) + } + assertLedgerQuantity(t, current.Status.ResourceQuota.Reserved, corev1.ResourceRequestsCPU, "0") + assertLedgerQuantity(t, current.Status.ResourceQuota.Allocated, corev1.ResourceRequestsCPU, "0") +} + +func TestConcurrentReservationsCannotOversubscribe(t *testing.T) { + t.Parallel() + + key := types.NamespacedName{Namespace: "capsule-system", Name: "concurrent"} + hard := corev1.ResourceList{corev1.ResourceRequestsCPU: resource.MustParse("10")} + quota := globalQuotaForTest("concurrent", hard) + cl := ledgerClient(t, initializedLedger(key, quota, zeroResourceList(hard))) + + var allowed atomic.Int32 + errs := make(chan error, 20) + var wg sync.WaitGroup + + for i := 0; i < 20; i++ { + wg.Add(1) + + go func(index int) { + defer wg.Done() + + reservation := reservationForTest( + fmt.Sprintf("request-%d", index), + corev1.ResourceList{corev1.ResourceRequestsCPU: resource.MustParse("1")}, + ) + ok, _, _, err := reserve(context.Background(), cl, cl, key, quota, reservation, false) + if err != nil { + errs <- err + + return + } + if ok { + allowed.Add(1) + } + }(i) + } + + wg.Wait() + close(errs) + for err := range errs { + t.Errorf("concurrent reserve error = %v", err) + } + + if got := allowed.Load(); got != 10 { + t.Fatalf("allowed reservations = %d, want 10", got) + } + + current := &capsulev1beta2.QuantityLedger{} + if err := cl.Get(context.Background(), key, current); err != nil { + t.Fatal(err) + } + if len(current.Status.ResourceQuota.Reservations) != 10 { + t.Fatalf("stored reservations = %d, want 10", len(current.Status.ResourceQuota.Reservations)) + } + assertLedgerQuantity(t, current.Status.ResourceQuota.Allocated, corev1.ResourceRequestsCPU, "10") +} + +func TestManagedResourceQuotaIsExcludedFromAccounting(t *testing.T) { + t.Parallel() + + req := admissionRequest("resourcequotas") + object := &metav1.PartialObjectMetadata{ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + meta.NewManagedByCapsuleLabel: meta.ValueController, + meta.GlobalResourceQuotaLabel: "shared", + }, + }} + + if !isManagedResourceQuota(req, object) { + t.Fatal("managed GlobalResourceQuota child was not excluded") + } + + object.Labels = nil + if isManagedResourceQuota(req, object) { + t.Fatal("unmanaged ResourceQuota was excluded") + } +} + +func TestValidateGlobalResourceQuota(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + quota *capsulev1beta2.GlobalResourceQuota + wantErr bool + }{ + { + name: "requires hard resources", + quota: globalQuotaForTest("empty", nil), + wantErr: true, + }, + { + name: "rejects negative quantities", + quota: globalQuotaForTest("negative", corev1.ResourceList{ + corev1.ResourceRequestsCPU: resource.MustParse("-1"), + }), + wantErr: true, + }, + { + name: "accepts an empty selector as all namespaces", + quota: func() *capsulev1beta2.GlobalResourceQuota { + quota := globalQuotaForTest("valid", corev1.ResourceList{ + corev1.ResourceRequestsCPU: resource.MustParse("8"), + }) + quota.Spec.NamespaceSelectors = []selectors.NamespaceSelector{{ + LabelSelector: &metav1.LabelSelector{}, + }} + + return quota + }(), + }, + } + + for _, test := range tests { + test := test + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + err := validateGlobalResourceQuota(test.quota) + if (err != nil) != test.wantErr { + t.Fatalf("validateGlobalResourceQuota() error = %v, wantErr %v", err, test.wantErr) + } + }) + } +} + +func TestValidateHardLimitAgainstAllocatedUsage(t *testing.T) { + t.Parallel() + + allocated := corev1.ResourceList{ + corev1.ResourceRequestsCPU: resource.MustParse("3"), + } + + if err := validateHardLimit(corev1.ResourceList{ + corev1.ResourceRequestsCPU: resource.MustParse("3"), + }, allocated); err != nil { + t.Fatalf("equal hard limit rejected: %v", err) + } + if err := validateHardLimit(corev1.ResourceList{ + corev1.ResourceRequestsCPU: resource.MustParse("2"), + }, allocated); err == nil { + t.Fatal("hard limit below allocated usage was accepted") + } + if err := validateHardLimit(corev1.ResourceList{}, allocated); err == nil { + t.Fatal("allocated resource was removed from hard limit") + } +} + +func TestFormatExceededResources(t *testing.T) { + t.Parallel() + + requested := corev1.ResourceList{ + corev1.ResourceLimitsCPU: resource.MustParse("1"), + corev1.ResourceLimitsMemory: resource.MustParse("1Gi"), + corev1.ResourceRequestsCPU: resource.MustParse("100m"), + corev1.ResourceRequestsMemory: resource.MustParse("256Mi"), + } + projected := corev1.ResourceList{ + corev1.ResourceLimitsCPU: resource.MustParse("9"), + corev1.ResourceLimitsMemory: resource.MustParse("9Gi"), + corev1.ResourceRequestsCPU: resource.MustParse("900m"), + corev1.ResourceRequestsMemory: resource.MustParse("2304Mi"), + } + hard := corev1.ResourceList{ + corev1.ResourceLimitsCPU: resource.MustParse("8"), + corev1.ResourceLimitsMemory: resource.MustParse("16Gi"), + corev1.ResourceRequestsCPU: resource.MustParse("8"), + corev1.ResourceRequestsMemory: resource.MustParse("16Gi"), + } + + want := "limits.cpu (requested=1, current=8, projected=9, hard=8, exceededBy=1)" + if got := formatExceededResources(requested, projected, hard); got != want { + t.Fatalf("formatExceededResources() = %q, want %q", got, want) + } +} + +func TestFormatExceededResourcesSortsAndFormatsEveryExceededLimit(t *testing.T) { + t.Parallel() + + requested := corev1.ResourceList{ + corev1.ResourceLimitsMemory: resource.MustParse("512Mi"), + corev1.ResourceRequestsCPU: resource.MustParse("750m"), + } + projected := corev1.ResourceList{ + corev1.ResourceLimitsMemory: resource.MustParse("1536Mi"), + corev1.ResourceRequestsCPU: resource.MustParse("1500m"), + } + hard := corev1.ResourceList{ + corev1.ResourceLimitsMemory: resource.MustParse("1Gi"), + corev1.ResourceRequestsCPU: resource.MustParse("1"), + } + + want := "limits.memory (requested=512Mi, current=1Gi, projected=1536Mi, hard=1Gi, exceededBy=512Mi); " + + "requests.cpu (requested=750m, current=750m, projected=1500m, hard=1, exceededBy=500m)" + if got := formatExceededResources(requested, projected, hard); got != want { + t.Fatalf("formatExceededResources() = %q, want %q", got, want) + } +} + +func admissionRequest(resourceName string) admission.Request { + return admission.Request{AdmissionRequest: admissionv1.AdmissionRequest{ + Resource: metav1.GroupVersionResource{Group: "", Version: "v1", Resource: resourceName}, + }} +} + +func initializedLedger( + key types.NamespacedName, + quota *capsulev1beta2.GlobalResourceQuota, + used corev1.ResourceList, +) *capsulev1beta2.QuantityLedger { + hard := quota.Spec.Quota.Hard + + return &capsulev1beta2.QuantityLedger{ + ObjectMeta: metav1.ObjectMeta{Name: key.Name, Namespace: key.Namespace}, + Spec: capsulev1beta2.QuantityLedgerSpec{ + TargetRef: capsulev1beta2.QuantityLedgerTargetRef{ + Kind: "GlobalResourceQuota", + Name: quota.Name, + UID: quota.UID, + }, + }, + Status: capsulev1beta2.QuantityLedgerStatus{ + ResourceQuota: &capsulev1beta2.QuantityLedgerResourceQuotaStatus{ + ObservedGeneration: quota.Generation, + Initialized: true, + Namespaces: []string{"tenant-a"}, + Used: used.DeepCopy(), + Reserved: zeroResourceList(hard), + Allocated: used.DeepCopy(), + }, + }, + } +} + +func globalQuotaForTest(name string, hard corev1.ResourceList) *capsulev1beta2.GlobalResourceQuota { + return &capsulev1beta2.GlobalResourceQuota{ + ObjectMeta: metav1.ObjectMeta{Name: name, UID: types.UID(name + "-uid"), Generation: 1}, + Spec: capsulev1beta2.GlobalResourceQuotaSpec{ + Quota: corev1.ResourceQuotaSpec{Hard: hard.DeepCopy()}, + }, + } +} + +func reservationForTest( + id string, + delta corev1.ResourceList, +) capsulev1beta2.QuantityLedgerResourceQuotaReservation { + now := metav1.Now() + + return capsulev1beta2.QuantityLedgerResourceQuotaReservation{ + ID: id, + Usage: delta.DeepCopy(), + Delta: delta.DeepCopy(), + ObjectRef: capsulev1beta2.QuantityLedgerObjectRef{ + APIVersion: "v1", + Kind: "Pod", + Namespace: "tenant-a", + }, + CreatedAt: now, + UpdatedAt: now, + } +} + +func ledgerClient(t *testing.T, objects ...client.Object) client.Client { + t.Helper() + + scheme := runtime.NewScheme() + if err := capsulev1beta2.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + + return fake.NewClientBuilder(). + WithScheme(scheme). + WithStatusSubresource(&capsulev1beta2.QuantityLedger{}). + WithObjects(objects...). + Build() +} + +func assertLedgerQuantity( + t *testing.T, + list corev1.ResourceList, + name corev1.ResourceName, + want string, +) { + t.Helper() + + got := list[name] + if got.Cmp(resource.MustParse(want)) != 0 { + t.Fatalf("%s = %s, want %s", name, got.String(), want) + } +} diff --git a/internal/webhook/route/globalresourcequota.go b/internal/webhook/route/globalresourcequota.go new file mode 100644 index 00000000..b41a5e38 --- /dev/null +++ b/internal/webhook/route/globalresourcequota.go @@ -0,0 +1,22 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package route + +import "github.com/projectcapsule/capsule/pkg/runtime/handlers" + +type globalResourceQuotaCalculation struct { + handlers []handlers.Handler +} + +func GlobalResourceQuotaCalculation(handler ...handlers.Handler) handlers.Webhook { + return &globalResourceQuotaCalculation{handlers: handler} +} + +func (w *globalResourceQuotaCalculation) GetHandlers() []handlers.Handler { + return w.handlers +} + +func (w *globalResourceQuotaCalculation) GetPath() string { + return "/global-resource-quotas/calculations" +} diff --git a/internal/webhook/tenant/validation/rule_validator.go b/internal/webhook/tenant/validation/rule_validator.go index 092fa904..f7b41ca3 100644 --- a/internal/webhook/tenant/validation/rule_validator.go +++ b/internal/webhook/tenant/validation/rule_validator.go @@ -5,6 +5,7 @@ package validation import ( "context" + "fmt" k8smeta "k8s.io/apimachinery/pkg/api/meta" "sigs.k8s.io/controller-runtime/pkg/client" @@ -16,6 +17,7 @@ import ( ad "github.com/projectcapsule/capsule/pkg/runtime/admission" "github.com/projectcapsule/capsule/pkg/runtime/events" "github.com/projectcapsule/capsule/pkg/runtime/handlers" + tenantutils "github.com/projectcapsule/capsule/pkg/tenant" ) type RuleValidationHandler struct { @@ -89,7 +91,7 @@ func (h *RuleValidationHandler) handle( } body := rule.NamespaceRuleBodyNamespace - if body.Enforce == nil { + if body.Enforce == nil && len(body.Quota) == 0 { continue } @@ -104,5 +106,17 @@ func (h *RuleValidationHandler) handle( return ad.Deny(err.Error()) } + for ruleIndex, rule := range tnt.Spec.Rules { + if rule == nil || rule.NamespaceRuleBodyNamespace == nil { + continue + } + + for quotaIndex, quota := range rule.Quota { + if err := tenantutils.ValidateRuleGlobalResourceQuotaName(tnt, quota.Name); err != nil { + return ad.Deny(fmt.Sprintf("rules[%d].quota[%d]: %v", ruleIndex, quotaIndex, err)) + } + } + } + return nil } diff --git a/pkg/api/meta/labels.go b/pkg/api/meta/labels.go index e747273e..bfc4df55 100644 --- a/pkg/api/meta/labels.go +++ b/pkg/api/meta/labels.go @@ -22,6 +22,8 @@ const ( ResourcePoolLabel = "projectcapsule.dev/pool" + GlobalResourceQuotaLabel = "projectcapsule.dev/global-resource-quota" + FreezeLabel = "projectcapsule.dev/freeze" OwnerPromotionLabel = "owner.projectcapsule.dev/promote" @@ -41,6 +43,7 @@ const ( LimitRangeLabel = "capsule.clastix.io/limit-range" NetworkPolicyLabel = "capsule.clastix.io/network-policy" ResourceQuotaLabel = "capsule.clastix.io/resource-quota" + RuleQuotaLabel = "projectcapsule.dev/rule-quota" RolebindingLabel = "capsule.clastix.io/role-binding" ) diff --git a/pkg/api/meta/metadata.go b/pkg/api/meta/metadata.go index 3bf67182..975d9fe8 100644 --- a/pkg/api/meta/metadata.go +++ b/pkg/api/meta/metadata.go @@ -26,6 +26,7 @@ func NewManagedMetadata( TenantLabel, NewTenantLabel, ResourcePoolLabel, + GlobalResourceQuotaLabel, FreezeLabel, OwnerPromotionLabel, ServiceAccountPromotionLabel, @@ -38,6 +39,7 @@ func NewManagedMetadata( LimitRangeLabel, NetworkPolicyLabel, ResourceQuotaLabel, + RuleQuotaLabel, RolebindingLabel, ), annotations: stringSet( diff --git a/pkg/api/rules/resource_quota_types.go b/pkg/api/rules/resource_quota_types.go new file mode 100644 index 00000000..c3c4c419 --- /dev/null +++ b/pkg/api/rules/resource_quota_types.go @@ -0,0 +1,22 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package rules + +import corev1 "k8s.io/api/core/v1" + +// ResourceQuotaRule defines a named ResourceQuota specification generated by a +// Tenant rule. Name is the durable identity of the generated +// GlobalResourceQuota and must be unique across all rules of a Tenant. +// +kubebuilder:object:generate=true +type ResourceQuotaRule struct { + corev1.ResourceQuotaSpec `json:",inline"` + + // Name is the stable identity of this quota within the Tenant. Changing the + // name replaces the generated GlobalResourceQuota; changing the quota or its + // namespace selector updates the existing object. + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=63 + // +kubebuilder:validation:Pattern=`^[a-z0-9]([-a-z0-9]*[a-z0-9])?$` + Name string `json:"name"` +} diff --git a/pkg/api/rules/rule_body_types.go b/pkg/api/rules/rule_body_types.go index ffd58166..cd6397a1 100644 --- a/pkg/api/rules/rule_body_types.go +++ b/pkg/api/rules/rule_body_types.go @@ -3,9 +3,7 @@ package rules -import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) +import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" // For future implementation where users might manage RuleStatus CRs themselves // +kubebuilder:object:generate=true @@ -15,6 +13,14 @@ type NamespaceRuleBodyNamespace struct { // +optional Audience []Audience `json:"audience,omitempty"` + // Quota contains native Kubernetes ResourceQuota specifications shared by + // all namespaces selected by this rule. Unlike Enforce, quota accounting is + // independent of the request audience. + // +optional + // +listType=map + // +listMapKey=name + Quota []ResourceQuotaRule `json:"quota,omitempty"` + // Enforcement for given rule //+optional Enforce *NamespaceRuleEnforceBody `json:"enforce,omitzero"` diff --git a/pkg/api/rules/zz_generated.deepcopy.go b/pkg/api/rules/zz_generated.deepcopy.go index ce64df1d..8db5eb6e 100644 --- a/pkg/api/rules/zz_generated.deepcopy.go +++ b/pkg/api/rules/zz_generated.deepcopy.go @@ -99,6 +99,13 @@ func (in *NamespaceRuleBodyNamespace) DeepCopyInto(out *NamespaceRuleBodyNamespa *out = make([]Audience, len(*in)) copy(*out, *in) } + if in.Quota != nil { + in, out := &in.Quota, &out.Quota + *out = make([]ResourceQuotaRule, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } if in.Enforce != nil { in, out := &in.Enforce, &out.Enforce *out = new(NamespaceRuleEnforceBody) @@ -347,6 +354,22 @@ func (in *OCIRegistry) DeepCopy() *OCIRegistry { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ResourceQuotaRule) DeepCopyInto(out *ResourceQuotaRule) { + *out = *in + in.ResourceQuotaSpec.DeepCopyInto(&out.ResourceQuotaSpec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ResourceQuotaRule. +func (in *ResourceQuotaRule) DeepCopy() *ResourceQuotaRule { + if in == nil { + return nil + } + out := new(ResourceQuotaRule) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ServiceExternalNameRule) DeepCopyInto(out *ServiceExternalNameRule) { *out = *in diff --git a/pkg/ruleengine/validate.go b/pkg/ruleengine/validate.go index fee92b7b..92f0792a 100644 --- a/pkg/ruleengine/validate.go +++ b/pkg/ruleengine/validate.go @@ -21,6 +21,8 @@ func ValidateRuleStatusBody( mapper k8smeta.RESTMapper, bodies []*rules.NamespaceRuleBodyNamespace, ) error { + quotaNames := make(map[string]string) + for i, rule := range bodies { if rule == nil { continue @@ -30,6 +32,10 @@ func ValidateRuleStatusBody( return err } + if err := validateQuotaRules(i, rule.Quota, quotaNames); err != nil { + return err + } + if rule.Enforce == nil { continue } @@ -54,6 +60,43 @@ func ValidateRuleStatusBody( return nil } +func validateQuotaRules(ruleIndex int, quotas []rules.ResourceQuotaRule, names map[string]string) error { + for quotaIndex, quota := range quotas { + path := fmt.Sprintf("rules[%d].quota[%d]", ruleIndex, quotaIndex) + if errs := k8svalidation.IsDNS1123Label(quota.Name); len(errs) > 0 { + return fmt.Errorf("%s.name %q is invalid: %s", path, quota.Name, strings.Join(errs, "; ")) + } + + if previous, found := names[quota.Name]; found { + return fmt.Errorf( + "%s.name %q is invalid: quota name is already used by %s", + path, + quota.Name, + previous, + ) + } + + names[quota.Name] = path + + if len(quota.Hard) == 0 { + return fmt.Errorf("%s.hard is invalid: at least one resource is required", path) + } + + for name, quantity := range quota.Hard { + if quantity.Sign() < 0 { + return fmt.Errorf( + "rules[%d].quota[%d].hard[%q] is invalid: quantity must not be negative", + ruleIndex, + quotaIndex, + name, + ) + } + } + } + + return nil +} + func validateIngressRules( ruleIndex int, ingress rules.NamespaceRuleEnforceIngressBody, diff --git a/pkg/ruleengine/validate_test.go b/pkg/ruleengine/validate_test.go index ec2b0fc3..ca3041c6 100644 --- a/pkg/ruleengine/validate_test.go +++ b/pkg/ruleengine/validate_test.go @@ -7,7 +7,9 @@ import ( "strings" "testing" + corev1 "k8s.io/api/core/v1" apimeta "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/utils/ptr" @@ -37,6 +39,73 @@ func TestValidateRuleStatusBody(t *testing.T) { }, }, }, + { + name: "valid quota-only rule", + mapper: mapper, + bodies: []*rules.NamespaceRuleBodyNamespace{{ + Quota: []rules.ResourceQuotaRule{{ + Name: "shared-compute", + ResourceQuotaSpec: corev1.ResourceQuotaSpec{Hard: corev1.ResourceList{ + corev1.ResourceRequestsCPU: resource.MustParse("8"), + }}, + }}, + }}, + }, + { + name: "quota name is required", + mapper: mapper, + bodies: []*rules.NamespaceRuleBodyNamespace{{ + Quota: []rules.ResourceQuotaRule{{ + ResourceQuotaSpec: corev1.ResourceQuotaSpec{Hard: corev1.ResourceList{ + corev1.ResourceRequestsCPU: resource.MustParse("8"), + }}, + }}, + }}, + wantErr: "rules[0].quota[0].name", + }, + { + name: "quota name must be a DNS label", + mapper: mapper, + bodies: []*rules.NamespaceRuleBodyNamespace{{ + Quota: []rules.ResourceQuotaRule{{ + Name: "Shared_Compute", + ResourceQuotaSpec: corev1.ResourceQuotaSpec{Hard: corev1.ResourceList{ + corev1.ResourceRequestsCPU: resource.MustParse("8"), + }}, + }}, + }}, + wantErr: `rules[0].quota[0].name "Shared_Compute" is invalid`, + }, + { + name: "quota hard must not be empty", + mapper: mapper, + bodies: []*rules.NamespaceRuleBodyNamespace{{ + Quota: []rules.ResourceQuotaRule{{Name: "shared-compute"}}, + }}, + wantErr: "rules[0].quota[0].hard is invalid", + }, + { + name: "quota hard must not be negative", + mapper: mapper, + bodies: []*rules.NamespaceRuleBodyNamespace{{ + Quota: []rules.ResourceQuotaRule{{ + Name: "shared-compute", + ResourceQuotaSpec: corev1.ResourceQuotaSpec{Hard: corev1.ResourceList{ + corev1.ResourceRequestsCPU: resource.MustParse("-1"), + }}, + }}, + }}, + wantErr: `rules[0].quota[0].hard["requests.cpu"] is invalid`, + }, + { + name: "quota names must be unique across rules", + mapper: mapper, + bodies: []*rules.NamespaceRuleBodyNamespace{ + {Quota: []rules.ResourceQuotaRule{{Name: "shared-compute", ResourceQuotaSpec: corev1.ResourceQuotaSpec{Hard: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1")}}}}}, + {Quota: []rules.ResourceQuotaRule{{Name: "shared-compute", ResourceQuotaSpec: corev1.ResourceQuotaSpec{Hard: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("1Gi")}}}}}, + }, + wantErr: `rules[1].quota[0].name "shared-compute" is invalid: quota name is already used by rules[0].quota[0]`, + }, { name: "valid workload service and metadata rules", mapper: mapper, diff --git a/pkg/runtime/workloads/qos_class.go b/pkg/runtime/workloads/qos_class.go index 6d575d89..33742214 100644 --- a/pkg/runtime/workloads/qos_class.go +++ b/pkg/runtime/workloads/qos_class.go @@ -144,12 +144,6 @@ func positiveResource(resources corev1.ResourceList, name corev1.ResourceName) ( return quantity, true } -//nolint:exhaustive func isSupportedQoSComputeResource(name corev1.ResourceName) bool { - switch name { - case corev1.ResourceCPU, corev1.ResourceMemory: - return true - default: - return false - } + return name == corev1.ResourceCPU || name == corev1.ResourceMemory } diff --git a/pkg/tenant/metdata.go b/pkg/tenant/metdata.go index ccb23373..ded990cd 100644 --- a/pkg/tenant/metdata.go +++ b/pkg/tenant/metdata.go @@ -177,7 +177,7 @@ func BuildNamespaceAnnotationsForTenant(tnt *capsulev1beta2.Tenant) map[string]s annotations[meta.AvailableIngressClassesAnnotation] = strings.Join(ic.Exact, ",") } - //nolint:staticcheck + //nolint:staticcheck,nolintlint // Preserve annotations for the deprecated v1beta2 field until it is removed. if len(ic.Regex) > 0 { annotations[meta.AvailableIngressClassesRegexpAnnotation] = ic.Regex } @@ -188,7 +188,7 @@ func BuildNamespaceAnnotationsForTenant(tnt *capsulev1beta2.Tenant) map[string]s annotations[meta.AvailableStorageClassesAnnotation] = strings.Join(sc.Exact, ",") } - //nolint:staticcheck + //nolint:staticcheck,nolintlint // Preserve annotations for the deprecated v1beta2 field until it is removed. if len(sc.Regex) > 0 { annotations[meta.AvailableStorageClassesRegexpAnnotation] = sc.Regex } diff --git a/pkg/tenant/rule_quota.go b/pkg/tenant/rule_quota.go new file mode 100644 index 00000000..57a7ff9b --- /dev/null +++ b/pkg/tenant/rule_quota.go @@ -0,0 +1,61 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package tenant + +import ( + "fmt" + "strings" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8svalidation "k8s.io/apimachinery/pkg/util/validation" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/runtime/selectors" +) + +func RuleGlobalResourceQuotaName(tnt *capsulev1beta2.Tenant, quotaName string) string { + return fmt.Sprintf("%s-%s", tnt.Name, quotaName) +} + +func ValidateRuleGlobalResourceQuotaName(tnt *capsulev1beta2.Tenant, quotaName string) error { + name := RuleGlobalResourceQuotaName(tnt, quotaName) + if errs := k8svalidation.IsDNS1123Subdomain(name); len(errs) > 0 { + return fmt.Errorf("generated GlobalResourceQuota name %q is invalid: %s", name, strings.Join(errs, "; ")) + } + + return nil +} + +func RuleGlobalResourceQuota( + tnt *capsulev1beta2.Tenant, + ruleIndex int, + itemIndex int, +) *capsulev1beta2.GlobalResourceQuota { + rule := tnt.Spec.Rules[ruleIndex] + quota := rule.Quota[itemIndex] + + selector := &metav1.LabelSelector{} + if rule.NamespaceSelector != nil { + selector = rule.NamespaceSelector.DeepCopy() + } + + if selector.MatchLabels == nil { + selector.MatchLabels = map[string]string{} + } + + // Tenant membership is part of the generated selector so a rule can never + // consume quota from a namespace owned by another Tenant. + selector.MatchLabels[meta.TenantLabel] = tnt.Name + + return &capsulev1beta2.GlobalResourceQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: RuleGlobalResourceQuotaName(tnt, quota.Name), + }, + Spec: capsulev1beta2.GlobalResourceQuotaSpec{ + NamespaceSelectors: []selectors.NamespaceSelector{{LabelSelector: selector}}, + Quota: *quota.ResourceQuotaSpec.DeepCopy(), + }, + } +} diff --git a/pkg/tenant/rule_quota_test.go b/pkg/tenant/rule_quota_test.go new file mode 100644 index 00000000..d24325ec --- /dev/null +++ b/pkg/tenant/rule_quota_test.go @@ -0,0 +1,83 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package tenant + +import ( + "strings" + "testing" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rules" +) + +func TestRuleGlobalResourceQuota(t *testing.T) { + t.Parallel() + + tnt := &capsulev1beta2.Tenant{ + ObjectMeta: metav1.ObjectMeta{Name: "tenant-a", UID: types.UID("tenant-uid")}, + Spec: capsulev1beta2.TenantSpec{Rules: []*rules.NamespaceRuleBodyTenant{{ + NamespaceRuleBodyNamespace: &rules.NamespaceRuleBodyNamespace{ + Quota: []rules.ResourceQuotaRule{{ + Name: "shared-compute", + ResourceQuotaSpec: corev1.ResourceQuotaSpec{Hard: corev1.ResourceList{ + corev1.ResourceRequestsMemory: resource.MustParse("16Gi"), + }}, + }}, + }, + NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"tier": "paid"}}, + }}}, + } + + quota := RuleGlobalResourceQuota(tnt, 0, 0) + if quota.Name != RuleGlobalResourceQuotaName(tnt, "shared-compute") { + t.Fatalf("generated name = %q, want deterministic helper name", quota.Name) + } + if len(quota.Spec.NamespaceSelectors) != 1 { + t.Fatalf("namespace selectors = %d, want 1", len(quota.Spec.NamespaceSelectors)) + } + selector := quota.Spec.NamespaceSelectors[0].LabelSelector + if selector.MatchLabels["tier"] != "paid" || selector.MatchLabels[meta.TenantLabel] != tnt.Name { + t.Fatalf("generated selector = %#v, want tenant and rule labels", selector) + } + if got := quota.Spec.Quota.Hard[corev1.ResourceRequestsMemory]; got.Cmp(resource.MustParse("16Gi")) != 0 { + t.Fatalf("generated quota hard = %s, want 16Gi", got.String()) + } +} + +func TestRuleGlobalResourceQuotaNameIsStableAcrossRuleChanges(t *testing.T) { + t.Parallel() + + tnt := &capsulev1beta2.Tenant{ + ObjectMeta: metav1.ObjectMeta{Name: "tenant-a", UID: types.UID("tenant-uid")}, + } + + want := RuleGlobalResourceQuotaName(tnt, "shared-compute") + if want != "tenant-a-shared-compute" { + t.Fatalf("generated name = %q, want Tenant and quota name", want) + } + if got := RuleGlobalResourceQuotaName(tnt, "shared-compute"); got != want { + t.Fatalf("same quota name generated %q, want %q", got, want) + } + if got := RuleGlobalResourceQuotaName(tnt, "service-count"); got == want { + t.Fatalf("different quota names generated the same object name %q", got) + } + + recreated := tnt.DeepCopy() + recreated.UID = types.UID("replacement-tenant-uid") + if got := RuleGlobalResourceQuotaName(recreated, "shared-compute"); got != want { + t.Fatalf("recreated Tenant generated %q, want stable name %q", got, want) + } + + longTenant := tnt.DeepCopy() + longTenant.Name = strings.Repeat("a", 250) + if err := ValidateRuleGlobalResourceQuotaName(longTenant, "shared-compute"); err == nil { + t.Fatal("overlong generated name was accepted") + } +} diff --git a/pkg/tenant/rules.go b/pkg/tenant/rules.go index 92787de7..a44a8aeb 100644 --- a/pkg/tenant/rules.go +++ b/pkg/tenant/rules.go @@ -85,7 +85,12 @@ func BuildNamespaceRuleBodyStatus( continue } - selected = append(selected, body.DeepCopy()) + statusBody := body.DeepCopy() + // Quotas are Tenant-level inputs used to generate cluster-scoped + // GlobalResourceQuotas. They are not evaluated from per-namespace + // RuleStatus objects and must not be projected into them. + statusBody.Quota = nil + selected = append(selected, statusBody) } rendered, err := template.RenderNamespaceRuleBodies( @@ -104,6 +109,7 @@ func BuildNamespaceRuleBodyStatus( continue } + body.Quota = nil out = append(out, body) } diff --git a/pkg/tenant/rules_promotions_test.go b/pkg/tenant/rules_promotions_test.go index 606b359c..4615a91b 100644 --- a/pkg/tenant/rules_promotions_test.go +++ b/pkg/tenant/rules_promotions_test.go @@ -62,7 +62,8 @@ func TestBuildNamespaceRuleBodyStatus(t *testing.T) { "{{ .namespace.status.phase }}", }}}, }, - }} + }, Quota: []rules.ResourceQuotaRule{{Name: "shared-compute"}}} + quotaOnly := &rules.NamespaceRuleBodyNamespace{Quota: []rules.ResourceQuotaRule{{Name: "object-counts"}}} unmatched := &rules.NamespaceRuleBodyNamespace{Enforce: &rules.NamespaceRuleEnforceBody{Action: rules.ActionTypeDeny}} tnt.Spec.Rules = []*rules.NamespaceRuleBodyTenant{ { @@ -73,6 +74,10 @@ func TestBuildNamespaceRuleBodyStatus(t *testing.T) { NamespaceRuleBodyNamespace: unmatched, NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"env": "dev"}}, }, + { + NamespaceRuleBodyNamespace: quotaOnly, + NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"env": "prod"}}, + }, nil, {}, } @@ -88,6 +93,9 @@ func TestBuildNamespaceRuleBodyStatus(t *testing.T) { if len(got) != 1 || got[0].Enforce.Action != rules.ActionTypeAudit { t.Fatalf("BuildNamespaceRuleBodyStatus() = %#v, want one audit rule", got) } + if len(got[0].Quota) != 0 { + t.Fatalf("BuildNamespaceRuleBodyStatus() quota = %#v, want none", got[0].Quota) + } if !reflect.DeepEqual(got[0].Enforce.Workloads.Schedulers[0].Exact, []string{"Active", "Active"}) { t.Fatalf("BuildNamespaceRuleBodyStatus() scheduler = %#v", got[0].Enforce.Workloads.Schedulers) }