diff --git a/Makefile b/Makefile index 21466415..a192d00a 100644 --- a/Makefile +++ b/Makefile @@ -215,7 +215,8 @@ dev-setup: dev-setup-cert-manager --set 'webhooks.hooks.calculations.rules[0].operations[1]=UPDATE' \ --set 'webhooks.hooks.calculations.rules[0].operations[2]=DELETE' \ --set 'webhooks.hooks.calculations.rules[0].resources[0]=pods' \ - --set 'webhooks.hooks.calculations.rules[0].resources[1]=persistentvolumeclaims' \ + --set 'webhooks.hooks.calculations.rules[0].resources[1]=pods/status' \ + --set 'webhooks.hooks.calculations.rules[0].resources[2]=persistentvolumeclaims' \ --set 'webhooks.hooks.calculations.rules[0].scope=Namespaced' \ --set 'webhooks.hooks.calculations.namespaceSelector.matchLabels.env=e2e' \ capsule \ @@ -499,7 +500,8 @@ e2e-install: helm-controller-version ko-build-all dev-install-gw-api-crds --set 'webhooks.hooks.calculations.rules[0].operations[1]=UPDATE' \ --set 'webhooks.hooks.calculations.rules[0].operations[2]=DELETE' \ --set 'webhooks.hooks.calculations.rules[0].resources[0]=pods' \ - --set 'webhooks.hooks.calculations.rules[0].resources[1]=persistentvolumeclaims' \ + --set 'webhooks.hooks.calculations.rules[0].resources[1]=pods/status' \ + --set 'webhooks.hooks.calculations.rules[0].resources[2]=persistentvolumeclaims' \ --set 'webhooks.hooks.calculations.rules[0].scope=Namespaced' \ --set 'webhooks.hooks.calculations.namespaceSelector.matchLabels.env=e2e' \ capsule \ diff --git a/api/v1beta2/customquota_func.go b/api/v1beta2/customquota_func.go index 592791ef..b776810b 100644 --- a/api/v1beta2/customquota_func.go +++ b/api/v1beta2/customquota_func.go @@ -26,3 +26,27 @@ func (c *CustomQuotaSpec) CollectJSONPathExpressions() (expressions []string) { return expressions } + +func (c *CustomQuotaSpec) CollectCELExpressions() (expressions []string) { + set := map[string]struct{}{} + + for _, source := range c.Sources { + if source.CEL != "" { + set[source.CEL] = struct{}{} + } + + for _, sel := range source.Selectors { + for _, expression := range sel.CELExpressions { + if expression != "" { + set[expression] = struct{}{} + } + } + } + } + + for expression := range set { + expressions = append(expressions, expression) + } + + return expressions +} diff --git a/api/v1beta2/customquota_status.go b/api/v1beta2/customquota_status.go index d68f8a15..3947449f 100644 --- a/api/v1beta2/customquota_status.go +++ b/api/v1beta2/customquota_status.go @@ -50,7 +50,7 @@ type CustomQuotaStatusTarget struct { metav1.GroupVersionKind `json:",inline"` CustomQuotaSpecSourceConfig `json:",inline"` - // Path on GVK where usage is evaluated + // Scope of the GVK where usage is evaluated. Scope k8smeta.RESTScopeName `json:"scope,omitempty"` } diff --git a/api/v1beta2/customquota_types.go b/api/v1beta2/customquota_types.go index 20f5cbf6..352b4c67 100644 --- a/api/v1beta2/customquota_types.go +++ b/api/v1beta2/customquota_types.go @@ -33,7 +33,7 @@ type CustomQuotaOptionsSpec struct { EmitPerClaimMetrics bool `json:"emitMetricPerClaimUsage,omitempty"` } -// +kubebuilder:validation:XValidation:rule="self.op == 'count' ? !has(self.path) || size(self.path) == 0 : has(self.path) && size(self.path) > 0",message="path must be empty when op is 'count'; otherwise path must be set and non-empty" +// +kubebuilder:validation:XValidation:rule="self.op == 'count' ? ((!has(self.path) || size(self.path) == 0) && (!has(self.cel) || size(self.cel) == 0)) : ((has(self.path) && size(self.path) > 0) != (has(self.cel) && size(self.cel) > 0))",message="path and cel must be empty when op is 'count'; otherwise exactly one of path or cel must be set and non-empty" type CustomQuotaSpecSource struct { runtime.VersionKind `json:",inline"` CustomQuotaSpecSourceConfig `json:",inline"` @@ -46,6 +46,14 @@ type CustomQuotaSpecSourceConfig struct { // +optional Path string `json:"path,omitempty"` + // CEL expression evaluated against the source object. + // The object is available as "object". + // Must evaluate to kubernetes.Quantity or list. + // Mutually exclusive with path and must be empty when op is "count". + // +kubebuilder:validation:MaxLength=4096 + // +optional + CEL string `json:"cel,omitempty"` + // Operation used to evaluate usage. // +kubebuilder:default:=add Operation quota.Operation `json:"op,omitempty"` diff --git a/api/v1beta2/quantityledgers_status.go b/api/v1beta2/quantityledgers_status.go index 6d53d666..d36026c0 100644 --- a/api/v1beta2/quantityledgers_status.go +++ b/api/v1beta2/quantityledgers_status.go @@ -21,6 +21,16 @@ type QuantityLedgerReservation struct { // Amount reserved for this request. Usage resource.Quantity `json:"usage"` + // Delta is the additional amount held against the quota while the admitted + // object is materializing. For creates this is normally equal to Usage. For + // updates it is max(newUsage-oldUsage, 0), so admission never releases + // capacity before the API server has persisted the update. + // + // A nil value is interpreted as Usage for backwards compatibility with + // ledgers written before this field was introduced. + // +optional + Delta *resource.Quantity `json:"delta,omitempty"` + // Object that this reservation is intended to create/update. ObjectRef QuantityLedgerObjectRef `json:"objectRef"` @@ -38,6 +48,11 @@ type QuantityLedgerReservation struct { // QuantityLedgerPendingDelete tracks objects that are expected to disappear from claims // soon, but may still temporarily appear during rebuild due to propagation delay. type QuantityLedgerPendingDelete struct { + // ID identifies the admission request that added this hint. It allows a + // failed multi-quota admission to roll back only its own hint. + // +optional + ID string `json:"id,omitempty"` + ObjectRef QuantityLedgerObjectRef `json:"objectRef"` CreatedAt metav1.Time `json:"createdAt"` } diff --git a/api/v1beta2/zz_generated.deepcopy.go b/api/v1beta2/zz_generated.deepcopy.go index 7e8cbcdd..2bcb3901 100644 --- a/api/v1beta2/zz_generated.deepcopy.go +++ b/api/v1beta2/zz_generated.deepcopy.go @@ -975,6 +975,11 @@ func (in *QuantityLedgerPendingDelete) DeepCopy() *QuantityLedgerPendingDelete { func (in *QuantityLedgerReservation) DeepCopyInto(out *QuantityLedgerReservation) { *out = *in out.Usage = in.Usage.DeepCopy() + if in.Delta != nil { + in, out := &in.Delta, &out.Delta + x := (*in).DeepCopy() + *out = &x + } out.ObjectRef = in.ObjectRef in.CreatedAt.DeepCopyInto(&out.CreatedAt) in.UpdatedAt.DeepCopyInto(&out.UpdatedAt) diff --git a/charts/capsule/README.md b/charts/capsule/README.md index a1d1f6c3..3d0381f4 100644 --- a/charts/capsule/README.md +++ b/charts/capsule/README.md @@ -231,7 +231,7 @@ The following Values have changed key or Value: | webhooks.hooks.calculations.matchPolicy | string | `"Equivalent"` | [MatchPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) | | webhooks.hooks.calculations.namespaceSelector | object | `{}` | [NamespaceSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-namespaceselector) | | webhooks.hooks.calculations.objectSelector | object | `{}` | [ObjectSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-objectselector) | -| webhooks.hooks.calculations.rules | list | `[]` | [Rules](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-rules) | +| webhooks.hooks.calculations.rules | list | `[]` | [Rules](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-rules) Include an explicit status subresource (for example, pods/status) when CustomQuota field selectors depend on status fields. | | webhooks.hooks.config.enabled | bool | `true` | Enable the Hook | | webhooks.hooks.config.failurePolicy | string | `"Ignore"` | [FailurePolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#failure-policy) | | webhooks.hooks.config.matchConditions | list | `[]` | [MatchConditions](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) | @@ -281,16 +281,16 @@ The following Values have changed key or Value: | webhooks.hooks.gateways.objectSelector | object | `{}` | [ObjectSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-objectselector) | | webhooks.hooks.gateways.opts | object | `{}` | Capsule Hook Options | | webhooks.hooks.gateways.reinvocationPolicy | string | `"Never"` | [ReinvocationPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#reinvocation-policy) | -| webhooks.hooks.generic | object | `{"enabled":true,"failurePolicy":"Fail","matchConditions":[{"expression":"!has(request.subResource) || request.subResource == \"\"","name":"ignore-subresources"},{"expression":"request.resource.resource != \"events\"","name":"ignore-events"}],"matchPolicy":"Equivalent","namespaceSelector":{"matchExpressions":[{"key":"capsule.clastix.io/tenant","operator":"Exists"}]},"objectSelector":{},"opts":{},"reinvocationPolicy":"Never","rules":[{"apiGroups":["*"],"apiVersions":["*"],"operations":["CREATE","UPDATE"],"resources":["*"],"scope":"Namespaced"}]}` | Generic Rules API | +| webhooks.hooks.generic | object | `{"enabled":true,"failurePolicy":"Fail","matchConditions":[{"expression":"request.resource.resource != \"events\"","name":"ignore-events"}],"matchPolicy":"Equivalent","namespaceSelector":{"matchExpressions":[{"key":"capsule.clastix.io/tenant","operator":"Exists"}]},"objectSelector":{},"opts":{},"reinvocationPolicy":"Never","rules":[{"apiGroups":["*"],"apiVersions":["*"],"operations":["CREATE","UPDATE"],"resources":["*/*"],"scope":"Namespaced"}]}` | Generic Rules API, including the rule-engine validation for Pods and Services | | webhooks.hooks.generic.enabled | bool | `true` | Enable the Hook | | webhooks.hooks.generic.failurePolicy | string | `"Fail"` | [FailurePolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#failure-policy) | -| webhooks.hooks.generic.matchConditions | list | `[{"expression":"!has(request.subResource) || request.subResource == \"\"","name":"ignore-subresources"},{"expression":"request.resource.resource != \"events\"","name":"ignore-events"}]` | [MatchConditions](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) | +| webhooks.hooks.generic.matchConditions | list | `[{"expression":"request.resource.resource != \"events\"","name":"ignore-events"}]` | [MatchConditions](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) | | webhooks.hooks.generic.matchPolicy | string | `"Equivalent"` | [MatchPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) | | webhooks.hooks.generic.namespaceSelector | object | `{"matchExpressions":[{"key":"capsule.clastix.io/tenant","operator":"Exists"}]}` | [NamespaceSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-namespaceselector) | | webhooks.hooks.generic.objectSelector | object | `{}` | [ObjectSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-objectselector) | | webhooks.hooks.generic.opts | object | `{}` | Capsule Hook Options | | webhooks.hooks.generic.reinvocationPolicy | string | `"Never"` | [ReinvocationPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#reinvocation-policy) | -| webhooks.hooks.generic.rules | list | `[{"apiGroups":["*"],"apiVersions":["*"],"operations":["CREATE","UPDATE"],"resources":["*"],"scope":"Namespaced"}]` | [Rules](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-rules) | +| webhooks.hooks.generic.rules | list | `[{"apiGroups":["*"],"apiVersions":["*"],"operations":["CREATE","UPDATE"],"resources":["*/*"],"scope":"Namespaced"}]` | [Rules](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-rules) | | webhooks.hooks.globalcustomquotas.enabled | bool | `true` | Enable the Hook | | webhooks.hooks.globalcustomquotas.failurePolicy | string | `"Fail"` | [FailurePolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#failure-policy) | | webhooks.hooks.globalcustomquotas.matchConditions | list | `[]` | [MatchConditions](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) | @@ -419,6 +419,7 @@ The following Values have changed key or Value: | webhooks.hooks.tenants.opts | object | `{}` | Capsule Hook Options | | webhooks.hooks.tenants.reinvocationPolicy | string | `"Never"` | [ReinvocationPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#reinvocation-policy) | | webhooks.labels | object | `{}` | Additional Labels for all webhooks | +| webhooks.matchConditions | list | `[]` | MatchConditions for all webhooks | | webhooks.mutatingWebhooksTimeoutSeconds | int | `30` | Timeout in seconds for mutating webhooks | | webhooks.service.caBundle | string | `""` | CABundle for the webhook service | | webhooks.service.name | string | `""` | Custom service name for the webhook service | diff --git a/charts/capsule/crds/capsule.clastix.io_customquotas.yaml b/charts/capsule/crds/capsule.clastix.io_customquotas.yaml index e26cbc8a..ef513f58 100644 --- a/charts/capsule/crds/capsule.clastix.io_customquotas.yaml +++ b/charts/capsule/crds/capsule.clastix.io_customquotas.yaml @@ -154,6 +154,14 @@ spec: - "apps/v1" means the "apps/v1" API group/version. - "apps/*" means any version in the "apps" API group. type: string + cel: + description: |- + CEL expression evaluated against the source object. + The object is available as "object". + Must evaluate to kubernetes.Quantity or list. + Mutually exclusive with path and must be empty when op is "count". + maxLength: 4096 + type: string kind: description: |- Kind of the referent. @@ -182,6 +190,18 @@ spec: Allowing these selectors to make further selecting on the resulting subset. items: properties: + celExpressions: + description: |- + Additional CEL expressions evaluated against the selected object. + The object is available as "object". + All must evaluate to true for this selector to match. + CEL expressions and fieldSelectors may be used together. + items: + maxLength: 4096 + minLength: 1 + type: string + maxItems: 64 + type: array fieldSelectors: description: |- Additional boolean JSONPath expressions. @@ -237,10 +257,12 @@ spec: - kind type: object x-kubernetes-validations: - - message: path must be empty when op is 'count'; otherwise path - must be set and non-empty - rule: 'self.op == ''count'' ? !has(self.path) || size(self.path) - == 0 : has(self.path) && size(self.path) > 0' + - message: path and cel must be empty when op is 'count'; otherwise + exactly one of path or cel must be set and non-empty + rule: 'self.op == ''count'' ? ((!has(self.path) || size(self.path) + == 0) && (!has(self.cel) || size(self.cel) == 0)) : ((has(self.path) + && size(self.path) > 0) != (has(self.cel) && size(self.cel) + > 0))' type: array required: - limit @@ -354,6 +376,14 @@ spec: description: Targeting GVK items: properties: + cel: + description: |- + CEL expression evaluated against the source object. + The object is available as "object". + Must evaluate to kubernetes.Quantity or list. + Mutually exclusive with path and must be empty when op is "count". + maxLength: 4096 + type: string group: type: string kind: @@ -373,7 +403,7 @@ spec: Required and non-empty for all other operations. type: string scope: - description: Path on GVK where usage is evaluated + description: Scope of the GVK where usage is evaluated. type: string selectors: description: |- @@ -382,6 +412,18 @@ spec: Allowing these selectors to make further selecting on the resulting subset. items: properties: + celExpressions: + description: |- + Additional CEL expressions evaluated against the selected object. + The object is available as "object". + All must evaluate to true for this selector to match. + CEL expressions and fieldSelectors may be used together. + items: + maxLength: 4096 + minLength: 1 + type: string + maxItems: 64 + type: array fieldSelectors: description: |- Additional boolean JSONPath expressions. diff --git a/charts/capsule/crds/capsule.clastix.io_globalcustomquotas.yaml b/charts/capsule/crds/capsule.clastix.io_globalcustomquotas.yaml index 9b3398f5..6d46b150 100644 --- a/charts/capsule/crds/capsule.clastix.io_globalcustomquotas.yaml +++ b/charts/capsule/crds/capsule.clastix.io_globalcustomquotas.yaml @@ -204,6 +204,14 @@ spec: - "apps/v1" means the "apps/v1" API group/version. - "apps/*" means any version in the "apps" API group. type: string + cel: + description: |- + CEL expression evaluated against the source object. + The object is available as "object". + Must evaluate to kubernetes.Quantity or list. + Mutually exclusive with path and must be empty when op is "count". + maxLength: 4096 + type: string kind: description: |- Kind of the referent. @@ -232,6 +240,18 @@ spec: Allowing these selectors to make further selecting on the resulting subset. items: properties: + celExpressions: + description: |- + Additional CEL expressions evaluated against the selected object. + The object is available as "object". + All must evaluate to true for this selector to match. + CEL expressions and fieldSelectors may be used together. + items: + maxLength: 4096 + minLength: 1 + type: string + maxItems: 64 + type: array fieldSelectors: description: |- Additional boolean JSONPath expressions. @@ -287,10 +307,12 @@ spec: - kind type: object x-kubernetes-validations: - - message: path must be empty when op is 'count'; otherwise path - must be set and non-empty - rule: 'self.op == ''count'' ? !has(self.path) || size(self.path) - == 0 : has(self.path) && size(self.path) > 0' + - message: path and cel must be empty when op is 'count'; otherwise + exactly one of path or cel must be set and non-empty + rule: 'self.op == ''count'' ? ((!has(self.path) || size(self.path) + == 0) && (!has(self.cel) || size(self.cel) == 0)) : ((has(self.path) + && size(self.path) > 0) != (has(self.cel) && size(self.cel) + > 0))' type: array required: - limit @@ -409,6 +431,14 @@ spec: description: Targeting GVK items: properties: + cel: + description: |- + CEL expression evaluated against the source object. + The object is available as "object". + Must evaluate to kubernetes.Quantity or list. + Mutually exclusive with path and must be empty when op is "count". + maxLength: 4096 + type: string group: type: string kind: @@ -428,7 +458,7 @@ spec: Required and non-empty for all other operations. type: string scope: - description: Path on GVK where usage is evaluated + description: Scope of the GVK where usage is evaluated. type: string selectors: description: |- @@ -437,6 +467,18 @@ spec: Allowing these selectors to make further selecting on the resulting subset. items: properties: + celExpressions: + description: |- + Additional CEL expressions evaluated against the selected object. + The object is available as "object". + All must evaluate to true for this selector to match. + CEL expressions and fieldSelectors may be used together. + items: + maxLength: 4096 + minLength: 1 + type: string + maxItems: 64 + type: array fieldSelectors: description: |- Additional boolean JSONPath expressions. diff --git a/charts/capsule/crds/capsule.clastix.io_quantityledgers.yaml b/charts/capsule/crds/capsule.clastix.io_quantityledgers.yaml index 604db24d..b3c6963b 100644 --- a/charts/capsule/crds/capsule.clastix.io_quantityledgers.yaml +++ b/charts/capsule/crds/capsule.clastix.io_quantityledgers.yaml @@ -172,6 +172,11 @@ spec: createdAt: format: date-time type: string + id: + description: |- + ID identifies the admission request that added this hint. It allows a + failed multi-quota admission to roll back only its own hint. + type: string objectRef: description: |- QuotaLedgerObjectRef identifies the object for which a reservation exists. @@ -219,6 +224,20 @@ spec: description: Time the reservation was first created. format: date-time type: string + delta: + anyOf: + - type: integer + - type: string + description: |- + Delta is the additional amount held against the quota while the admitted + object is materializing. For creates this is normally equal to Usage. For + updates it is max(newUsage-oldUsage, 0), so admission never releases + capacity before the API server has persisted the update. + + A nil value is interpreted as Usage for backwards compatibility with + ledgers written before this field was introduced. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true expiresAt: description: Time after which the reservation may be considered stale. diff --git a/charts/capsule/diagnostics/admission-dashboard.json b/charts/capsule/diagnostics/admission-dashboard.json deleted file mode 100644 index d5f3a5ab..00000000 --- a/charts/capsule/diagnostics/admission-dashboard.json +++ /dev/null @@ -1,1083 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "description": "Profiling performance-related metrics based on controller-runtime.", - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 0, - "links": [], - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 19, - "options": { - "legend": { - "calcs": [ - "last" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true, - "sortBy": "Last", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.3.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "rate(controller_runtime_webhook_panics_total{job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "instant": false, - "legendFormat": "Panic", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum by (code) (rate(controller_runtime_webhook_requests_total{job=~\"$job\"}[$__rate_interval]))", - "hide": false, - "instant": false, - "legendFormat": "{{code}}", - "range": true, - "refId": "B" - } - ], - "title": "Requests", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 7 - }, - "id": 6, - "panels": [], - "title": "Overview", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 8 - }, - "id": 15, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.3.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum by (webhook) (rate(controller_runtime_webhook_requests_total{job=\"$job\"}[$__rate_interval]))", - "legendFormat": "{{webhook}}", - "range": true, - "refId": "A" - } - ], - "title": "Admission Requests", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 8 - }, - "id": 14, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.3.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum by (webhook) (rate(controller_runtime_webhook_requests_in_flight{job=\"$job\"}[$__rate_interval]))", - "legendFormat": "{{webhook}}", - "range": true, - "refId": "A" - } - ], - "title": "Requests In-Flight", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 18 - }, - "id": 13, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.3.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "histogram_quantile(\n 0.99,\n sum by (le, webhook) (rate(controller_runtime_webhook_latency_seconds_bucket{job=\"$job\"}[5m]))\n)", - "format": "time_series", - "instant": false, - "legendFormat": "{{webhook}}", - "range": true, - "refId": "A" - } - ], - "title": "Webhook Latency (P99)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 18 - }, - "id": 16, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.3.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "histogram_quantile(\n 0.95,\n sum by (le, webhook) (rate(controller_runtime_webhook_latency_seconds_bucket{job=\"$job\"}[$__rate_interval]))\n)", - "format": "time_series", - "instant": false, - "legendFormat": "{{webhook}}", - "range": true, - "refId": "A" - } - ], - "title": "Webhook Latency (P95)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 11, - "w": 12, - "x": 0, - "y": 28 - }, - "id": 17, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.3.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum by (webhook) (rate(controller_runtime_webhook_latency_seconds_count{job=\"$job\"}[$__rate_interval]))\n", - "format": "time_series", - "instant": false, - "legendFormat": "{{webhook}}", - "range": true, - "refId": "A" - } - ], - "title": "Webhook Latency Spent Total", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 11, - "w": 12, - "x": 12, - "y": 28 - }, - "id": 18, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.3.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "controller_runtime_webhook_requests_total{job=\"$job\"}\n", - "format": "time_series", - "instant": false, - "legendFormat": "{{webhook}}", - "range": true, - "refId": "A" - } - ], - "title": "Webhook Total Requests", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 39 - }, - "id": 8, - "panels": [], - "repeat": "Webhook", - "title": "Webhook \"$Webhook\" Status", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "percentage", - "steps": [ - { - "color": "purple", - "value": 0 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 24, - "x": 0, - "y": 40 - }, - "id": 10, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "none", - "sort": "none" - } - }, - "pluginVersion": "12.3.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "sum by (webhook) (rate(controller_runtime_webhook_requests_total{job=~\"$job\", webhook=\"$Webhook\"}[$__rate_interval]))", - "interval": "", - "legendFormat": "Request Rate", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum by (code) (rate(controller_runtime_webhook_requests_total{job=~\"$job\", webhook=\"$Webhook\"}[$__rate_interval]))", - "hide": false, - "instant": false, - "legendFormat": "{{code}}", - "range": true, - "refId": "B" - } - ], - "title": "Webhook Request Rate $Webhook", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 49 - }, - "id": 12, - "options": { - "calculate": false, - "calculation": {}, - "cellGap": 2, - "cellValues": {}, - "color": { - "exponent": 0.5, - "fill": "#b4ff00", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "Oranges", - "steps": 128 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": false - }, - "rowsFrame": { - "layout": "auto" - }, - "showValue": "never", - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": false - }, - "yAxis": { - "axisPlacement": "left", - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "12.3.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "rate(controller_runtime_webhook_latency_seconds_bucket{job=~\"$job\", webhook=\"$Webhook\"}[$__rate_interval])", - "format": "heatmap", - "interval": "", - "legendFormat": "{{le}}", - "range": true, - "refId": "A" - } - ], - "title": "Latency Second Buckets", - "type": "heatmap" - } - ], - "preload": false, - "refresh": "1m", - "schemaVersion": 42, - "tags": [], - "templating": { - "list": [ - { - "current": { - "text": "Prometheus", - "value": "prometheus" - }, - "description": "", - "hide": 1, - "name": "DS_PROMETHEUS", - "options": [], - "query": "prometheus", - "refresh": 1, - "regex": "", - "type": "datasource" - }, - { - "current": { - "text": "All", - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "definition": "label_values(controller_runtime_webhook_requests_total,job)", - "includeAll": true, - "label": "Job", - "multi": true, - "name": "job", - "options": [], - "query": { - "qryType": 1, - "query": "label_values(controller_runtime_webhook_requests_total,job)", - "refId": "PrometheusVariableQueryEditor-VariableQuery" - }, - "refresh": 2, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": "$__all" - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "definition": "label_values(controller_runtime_webhook_requests_total{job=~\"$job\"},webhook)", - "includeAll": true, - "multi": true, - "name": "Webhook", - "options": [], - "query": { - "qryType": 1, - "query": "label_values(controller_runtime_webhook_requests_total{job=~\"$job\"},webhook)", - "refId": "PrometheusVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "type": "query" - } - ] - }, - "time": { - "from": "now-5m", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "Capulse Admission Webhooks Diagnostics", - "uid": "0L6Y8KEnak", - "version": 4 -} diff --git a/charts/capsule/diagnostics/controller-runtime-dashboard.json b/charts/capsule/diagnostics/controller-runtime-dashboard.json index 721028c8..f117df44 100644 --- a/charts/capsule/diagnostics/controller-runtime-dashboard.json +++ b/charts/capsule/diagnostics/controller-runtime-dashboard.json @@ -4,17 +4,21 @@ "metadata": { "name": "5J4pyKEnka", "namespace": "default", - "uid": "78fa8eb4-e0ec-46a4-bac8-40497bc162ea", - "resourceVersion": "1784245790682001", + "uid": "f075f359-3350-4060-9ca9-8ea9694b89cc", + "resourceVersion": "1784812819751004", "generation": 1, - "creationTimestamp": "2026-07-16T23:49:48Z", + "creationTimestamp": "2026-07-23T13:20:19Z", "labels": { - "grafana.app/deprecatedInternalID": "1744130817093632" + "grafana.app/deprecatedInternalID": "4122431297622016" }, "annotations": { - "grafana.app/createdBy": "anonymous:0", - "grafana.app/folder": "", - "grafana.app/saved-from-ui": "Grafana v13.1.0 (b309c9bb3b)" + "grafana.app/createdBy": "access-policy:service", + "grafana.app/managedBy": "classic-file-provisioning", + "grafana.app/managerId": "sidecarProvider", + "grafana.app/sourceChecksum": "9327c8db4f041328c75cbd67e8675c9e", + "grafana.app/sourcePath": "/tmp/dashboards/namespace_capsule-system.configmap_capsule-controller-runtime-dashboard.controller-runtime-dashboard.json", + "grafana.app/sourceTimestamp": "1784812715000", + "grafana.app/folder": "" } }, "spec": { @@ -22,22 +26,22 @@ { "kind": "AnnotationQuery", "spec": { + "builtIn": true, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", "query": { - "kind": "DataQuery", "group": "grafana", - "version": "v0", + "kind": "DataQuery", "spec": { "limit": 100, "matchAny": false, "tags": [], "type": "dashboard" - } - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "builtIn": true + }, + "version": "v0" + } } } ], @@ -48,10 +52,6 @@ "panel-10": { "kind": "Panel", "spec": { - "id": 10, - "title": "Reconcile Rate", - "description": "", - "links": [], "data": { "kind": "QueryGroup", "spec": { @@ -59,13 +59,13 @@ { "kind": "PanelQuery", "spec": { + "hidden": false, "query": { - "kind": "DataQuery", - "group": "prometheus", - "version": "v0", "datasource": { "name": "prometheus" }, + "group": "prometheus", + "kind": "DataQuery", "spec": { "editorMode": "code", "exemplar": true, @@ -73,56 +73,27 @@ "interval": "", "legendFormat": "{{result}}", "range": true - } + }, + "version": "v0" }, - "refId": "A", - "hidden": false + "refId": "A" } } ], - "transformations": [], - "queryOptions": {} + "queryOptions": {}, + "transformations": [] } }, + "description": "", + "id": 10, + "links": [], + "title": "Reconcile Rate", "vizConfig": { - "kind": "VizConfig", "group": "timeseries", - "version": "13.1.0", + "kind": "VizConfig", "spec": { - "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, - "legend": { - "calcs": [], - "displayMode": "list", - "enableFacetedFilter": false, - "overflow": "ellipsis", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, "fieldConfig": { "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": 0, - "color": "green" - }, - { - "value": 80, - "color": "red" - } - ] - }, "color": { "mode": "palette-classic" }, @@ -159,21 +130,50 @@ "thresholdsStyle": { "mode": "off" } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] } }, "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } } - } + }, + "version": "13.1.0" } } }, "panel-12": { "kind": "Panel", "spec": { - "id": 12, - "title": "Reconcile Time Buckets", - "description": "", - "links": [], "data": { "kind": "QueryGroup", "spec": { @@ -181,13 +181,13 @@ { "kind": "PanelQuery", "spec": { + "hidden": false, "query": { - "kind": "DataQuery", - "group": "prometheus", - "version": "v0", "datasource": { "name": "prometheus" }, + "group": "prometheus", + "kind": "DataQuery", "spec": { "editorMode": "code", "exemplar": true, @@ -196,22 +196,40 @@ "interval": "", "legendFormat": "{{le}}", "range": true - } + }, + "version": "v0" }, - "refId": "A", - "hidden": false + "refId": "A" } } ], - "transformations": [], - "queryOptions": {} + "queryOptions": {}, + "transformations": [] } }, + "description": "", + "id": 12, + "links": [], + "title": "Reconcile Time Buckets", "vizConfig": { - "kind": "VizConfig", "group": "heatmap", - "version": "13.1.0", + "kind": "VizConfig", "spec": { + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, "options": { "annotations": { "clustering": -1, @@ -253,33 +271,15 @@ "reverse": false, "unit": "short" } - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] } - } + }, + "version": "13.1.0" } } }, "panel-13": { "kind": "Panel", "spec": { - "id": 13, - "title": "Workqueue Depth", - "description": "", - "links": [], "data": { "kind": "QueryGroup", "spec": { @@ -287,79 +287,39 @@ { "kind": "PanelQuery", "spec": { + "hidden": false, "query": { - "kind": "DataQuery", - "group": "prometheus", - "version": "v0", "datasource": { "name": "prometheus" }, + "group": "prometheus", + "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "sum by (name) (\n workqueue_depth{job=~\"$job\"}\n)", "legendFormat": "__auto", "range": true - } + }, + "version": "v0" }, - "refId": "A", - "hidden": false + "refId": "A" } } ], - "transformations": [], - "queryOptions": {} + "queryOptions": {}, + "transformations": [] } }, + "description": "", + "id": 13, + "links": [], + "title": "Workqueue Depth", "vizConfig": { - "kind": "VizConfig", "group": "timeseries", - "version": "13.1.0", + "kind": "VizConfig", "spec": { - "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, - "legend": { - "calcs": [ - "lastNotNull", - "max", - "mean" - ], - "displayMode": "table", - "enableFacetedFilter": false, - "overflow": "ellipsis", - "placement": "right", - "showLegend": true, - "sortBy": "Max", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, "fieldConfig": { "defaults": { - "unit": "short", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": 0, - "color": "green" - }, - { - "value": 100, - "color": "orange" - }, - { - "value": 500, - "color": "red" - } - ] - }, "color": { "mode": "palette-classic" }, @@ -396,21 +356,61 @@ "thresholdsStyle": { "mode": "dashed" } - } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "orange", + "value": 100 + }, + { + "color": "red", + "value": 500 + } + ] + }, + "unit": "short" }, "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "max", + "mean" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "right", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } } - } + }, + "version": "13.1.0" } } }, "panel-14": { "kind": "Panel", "spec": { - "id": 14, - "title": "Workqueue Depth Max", - "description": "Max Queue Depth, 15m", - "links": [], "data": { "kind": "QueryGroup", "spec": { @@ -418,78 +418,39 @@ { "kind": "PanelQuery", "spec": { + "hidden": false, "query": { - "kind": "DataQuery", - "group": "prometheus", - "version": "v0", "datasource": { "name": "prometheus" }, + "group": "prometheus", + "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "max by (name) (\n max_over_time(workqueue_depth{job=~\"$job\"}[15m])\n)", "legendFormat": "__auto", "range": true - } + }, + "version": "v0" }, - "refId": "A", - "hidden": false + "refId": "A" } } ], - "transformations": [], - "queryOptions": {} + "queryOptions": {}, + "transformations": [] } }, + "description": "Max Queue Depth, 15m", + "id": 14, + "links": [], + "title": "Workqueue Depth Max", "vizConfig": { - "kind": "VizConfig", "group": "timeseries", - "version": "13.1.0", + "kind": "VizConfig", "spec": { - "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, - "legend": { - "calcs": [ - "lastNotNull", - "max", - "mean" - ], - "displayMode": "table", - "enableFacetedFilter": false, - "overflow": "ellipsis", - "placement": "right", - "showLegend": true, - "sortBy": "Max", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, "fieldConfig": { "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": 0, - "color": "green" - }, - { - "value": 100, - "color": "orange" - }, - { - "value": 500, - "color": "red" - } - ] - }, "color": { "mode": "palette-classic" }, @@ -526,21 +487,60 @@ "thresholdsStyle": { "mode": "dashed" } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "orange", + "value": 100 + }, + { + "color": "red", + "value": 500 + } + ] } }, "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "max", + "mean" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "right", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } } - } + }, + "version": "13.1.0" } } }, "panel-15": { "kind": "Panel", "spec": { - "id": 15, - "title": "Queue Wait Duration (p95)", - "description": "Queue Wait Duration p95", - "links": [], "data": { "kind": "QueryGroup", "spec": { @@ -548,79 +548,39 @@ { "kind": "PanelQuery", "spec": { + "hidden": false, "query": { - "kind": "DataQuery", - "group": "prometheus", - "version": "v0", "datasource": { "name": "prometheus" }, + "group": "prometheus", + "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "histogram_quantile(\n 0.95,\n sum by (name, le) (\n rate(workqueue_queue_duration_seconds_bucket{job=~\"$job\"}[$__rate_interval])\n )\n)", "legendFormat": "__auto", "range": true - } + }, + "version": "v0" }, - "refId": "A", - "hidden": false + "refId": "A" } } ], - "transformations": [], - "queryOptions": {} + "queryOptions": {}, + "transformations": [] } }, + "description": "Queue Wait Duration p95", + "id": 15, + "links": [], + "title": "Queue Wait Duration (p95)", "vizConfig": { - "kind": "VizConfig", "group": "timeseries", - "version": "13.1.0", + "kind": "VizConfig", "spec": { - "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, - "legend": { - "calcs": [ - "lastNotNull", - "max", - "mean" - ], - "displayMode": "table", - "enableFacetedFilter": false, - "overflow": "ellipsis", - "placement": "right", - "showLegend": true, - "sortBy": "Max", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, "fieldConfig": { "defaults": { - "unit": "s", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": 0, - "color": "green" - }, - { - "value": 30, - "color": "orange" - }, - { - "value": 300, - "color": "red" - } - ] - }, "color": { "mode": "palette-classic" }, @@ -657,7 +617,25 @@ "thresholdsStyle": { "mode": "dashed" } - } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "orange", + "value": 30 + }, + { + "color": "red", + "value": 300 + } + ] + }, + "unit": "s" }, "overrides": [ { @@ -685,18 +663,40 @@ ] } ] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "max", + "mean" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "right", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } } - } + }, + "version": "13.1.0" } } }, "panel-16": { "kind": "Panel", "spec": { - "id": 16, - "title": "Queue Wait Duration (p99)", - "description": "", - "links": [], "data": { "kind": "QueryGroup", "spec": { @@ -704,79 +704,39 @@ { "kind": "PanelQuery", "spec": { + "hidden": false, "query": { - "kind": "DataQuery", - "group": "prometheus", - "version": "v0", "datasource": { "name": "prometheus" }, + "group": "prometheus", + "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "histogram_quantile(\n 0.99,\n sum by (name, le) (\n rate(workqueue_queue_duration_seconds_bucket{job=~\"$job\"}[$__rate_interval])\n )\n)", "legendFormat": "__auto", "range": true - } + }, + "version": "v0" }, - "refId": "A", - "hidden": false + "refId": "A" } } ], - "transformations": [], - "queryOptions": {} + "queryOptions": {}, + "transformations": [] } }, + "description": "", + "id": 16, + "links": [], + "title": "Queue Wait Duration (p99)", "vizConfig": { - "kind": "VizConfig", "group": "timeseries", - "version": "13.1.0", + "kind": "VizConfig", "spec": { - "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, - "legend": { - "calcs": [ - "lastNotNull", - "max", - "mean" - ], - "displayMode": "table", - "enableFacetedFilter": false, - "overflow": "ellipsis", - "placement": "right", - "showLegend": true, - "sortBy": "Max", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, "fieldConfig": { "defaults": { - "unit": "s", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": 0, - "color": "green" - }, - { - "value": 60, - "color": "orange" - }, - { - "value": 300, - "color": "red" - } - ] - }, "color": { "mode": "palette-classic" }, @@ -813,21 +773,61 @@ "thresholdsStyle": { "mode": "dashed" } - } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "orange", + "value": 60 + }, + { + "color": "red", + "value": 300 + } + ] + }, + "unit": "s" }, "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "max", + "mean" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "right", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } } - } + }, + "version": "13.1.0" } } }, "panel-17": { "kind": "Panel", "spec": { - "id": 17, - "title": "Work Duration p95", - "description": "", - "links": [], "data": { "kind": "QueryGroup", "spec": { @@ -835,79 +835,39 @@ { "kind": "PanelQuery", "spec": { + "hidden": false, "query": { - "kind": "DataQuery", - "group": "prometheus", - "version": "v0", "datasource": { "name": "prometheus" }, + "group": "prometheus", + "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "histogram_quantile(\n 0.95,\n sum by (name, le) (\n rate(workqueue_work_duration_seconds_bucket{job=~\"$job\"}[$__rate_interval])\n )\n)", "legendFormat": "__auto", "range": true - } + }, + "version": "v0" }, - "refId": "A", - "hidden": false + "refId": "A" } } ], - "transformations": [], - "queryOptions": {} + "queryOptions": {}, + "transformations": [] } }, + "description": "", + "id": 17, + "links": [], + "title": "Work Duration p95", "vizConfig": { - "kind": "VizConfig", "group": "timeseries", - "version": "13.1.0", + "kind": "VizConfig", "spec": { - "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, - "legend": { - "calcs": [ - "lastNotNull", - "max", - "mean" - ], - "displayMode": "table", - "enableFacetedFilter": false, - "overflow": "ellipsis", - "placement": "right", - "showLegend": true, - "sortBy": "Max", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, "fieldConfig": { "defaults": { - "unit": "s", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": 0, - "color": "green" - }, - { - "value": 5, - "color": "orange" - }, - { - "value": 15, - "color": "red" - } - ] - }, "color": { "mode": "palette-classic" }, @@ -944,21 +904,61 @@ "thresholdsStyle": { "mode": "dashed" } - } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "orange", + "value": 5 + }, + { + "color": "red", + "value": 15 + } + ] + }, + "unit": "s" }, "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "max", + "mean" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "right", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } } - } + }, + "version": "13.1.0" } } }, "panel-18": { "kind": "Panel", "spec": { - "id": 18, - "title": "Work Duration p99", - "description": "", - "links": [], "data": { "kind": "QueryGroup", "spec": { @@ -966,79 +966,39 @@ { "kind": "PanelQuery", "spec": { + "hidden": false, "query": { - "kind": "DataQuery", - "group": "prometheus", - "version": "v0", "datasource": { "name": "prometheus" }, + "group": "prometheus", + "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "histogram_quantile(\n 0.99,\n sum by (name, le) (\n rate(workqueue_work_duration_seconds_bucket{job=~\"$job\"}[$__rate_interval])\n )\n)", "legendFormat": "__auto", "range": true - } + }, + "version": "v0" }, - "refId": "A", - "hidden": false + "refId": "A" } } ], - "transformations": [], - "queryOptions": {} + "queryOptions": {}, + "transformations": [] } }, + "description": "", + "id": 18, + "links": [], + "title": "Work Duration p99", "vizConfig": { - "kind": "VizConfig", "group": "timeseries", - "version": "13.1.0", + "kind": "VizConfig", "spec": { - "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, - "legend": { - "calcs": [ - "lastNotNull", - "max", - "mean" - ], - "displayMode": "table", - "enableFacetedFilter": false, - "overflow": "ellipsis", - "placement": "right", - "showLegend": true, - "sortBy": "Max", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, "fieldConfig": { "defaults": { - "unit": "s", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": 0, - "color": "green" - }, - { - "value": 5, - "color": "orange" - }, - { - "value": 15, - "color": "red" - } - ] - }, "color": { "mode": "palette-classic" }, @@ -1075,21 +1035,61 @@ "thresholdsStyle": { "mode": "dashed" } - } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "orange", + "value": 5 + }, + { + "color": "red", + "value": 15 + } + ] + }, + "unit": "s" }, "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "max", + "mean" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "right", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } } - } + }, + "version": "13.1.0" } } }, "panel-19": { "kind": "Panel", "spec": { - "id": 19, - "title": "Work Duration Avg", - "description": "", - "links": [], "data": { "kind": "QueryGroup", "spec": { @@ -1097,79 +1097,39 @@ { "kind": "PanelQuery", "spec": { + "hidden": false, "query": { - "kind": "DataQuery", - "group": "prometheus", - "version": "v0", "datasource": { "name": "prometheus" }, + "group": "prometheus", + "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "sum by (name) (\n rate(workqueue_work_duration_seconds_sum{job=~\"$job\"}[$__rate_interval])\n)\n/\nsum by (name) (\n rate(workqueue_work_duration_seconds_count{job=~\"$job\"}[$__rate_interval])\n)", "legendFormat": "__auto", "range": true - } + }, + "version": "v0" }, - "refId": "A", - "hidden": false + "refId": "A" } } ], - "transformations": [], - "queryOptions": {} + "queryOptions": {}, + "transformations": [] } }, + "description": "", + "id": 19, + "links": [], + "title": "Work Duration Avg", "vizConfig": { - "kind": "VizConfig", "group": "timeseries", - "version": "13.1.0", + "kind": "VizConfig", "spec": { - "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, - "legend": { - "calcs": [ - "lastNotNull", - "max", - "mean" - ], - "displayMode": "table", - "enableFacetedFilter": false, - "overflow": "ellipsis", - "placement": "right", - "showLegend": true, - "sortBy": "Max", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, "fieldConfig": { "defaults": { - "unit": "s", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": 0, - "color": "green" - }, - { - "value": 5, - "color": "orange" - }, - { - "value": 15, - "color": "red" - } - ] - }, "color": { "mode": "palette-classic" }, @@ -1206,21 +1166,61 @@ "thresholdsStyle": { "mode": "dashed" } - } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "orange", + "value": 5 + }, + { + "color": "red", + "value": 15 + } + ] + }, + "unit": "s" }, "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "max", + "mean" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "right", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } } - } + }, + "version": "13.1.0" } } }, "panel-2": { "kind": "Panel", "spec": { - "id": 2, - "title": "Active Workers", - "description": "", - "links": [], "data": { "kind": "QueryGroup", "spec": { @@ -1228,13 +1228,13 @@ { "kind": "PanelQuery", "spec": { + "hidden": false, "query": { - "kind": "DataQuery", - "group": "prometheus", - "version": "v0", "datasource": { "name": "prometheus" }, + "group": "prometheus", + "kind": "DataQuery", "spec": { "editorMode": "code", "exemplar": true, @@ -1242,84 +1242,49 @@ "interval": "", "legendFormat": "{{controller}}", "range": true - } + }, + "version": "v0" }, - "refId": "A", - "hidden": false + "refId": "A" } }, { "kind": "PanelQuery", "spec": { + "hidden": false, "query": { - "kind": "DataQuery", - "group": "prometheus", - "version": "v0", "datasource": { "name": "prometheus" }, + "group": "prometheus", + "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "sum (controller_runtime_active_workers{job=~\"$job\"})", "instant": false, "legendFormat": "Total", "range": true - } + }, + "version": "v0" }, - "refId": "B", - "hidden": false + "refId": "B" } } ], - "transformations": [], - "queryOptions": {} + "queryOptions": {}, + "transformations": [] } }, + "description": "", + "id": 2, + "links": [], + "title": "Active Workers", "vizConfig": { - "kind": "VizConfig", "group": "timeseries", - "version": "13.1.0", + "kind": "VizConfig", "spec": { - "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, - "legend": { - "calcs": [ - "lastNotNull", - "max", - "mean" - ], - "displayMode": "table", - "enableFacetedFilter": false, - "overflow": "ellipsis", - "placement": "right", - "showLegend": true, - "sortBy": "Max", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, "fieldConfig": { "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": 0, - "color": "green" - }, - { - "value": 80, - "color": "red" - } - ] - }, "color": { "mode": "palette-classic" }, @@ -1356,21 +1321,56 @@ "thresholdsStyle": { "mode": "off" } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] } }, "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "max", + "mean" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "right", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } } - } + }, + "version": "13.1.0" } } }, "panel-20": { "kind": "Panel", "spec": { - "id": 20, - "title": "Processed Work Items / Second", - "description": "", - "links": [], "data": { "kind": "QueryGroup", "spec": { @@ -1378,75 +1378,39 @@ { "kind": "PanelQuery", "spec": { + "hidden": false, "query": { - "kind": "DataQuery", - "group": "prometheus", - "version": "v0", "datasource": { "name": "prometheus" }, + "group": "prometheus", + "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "sum by (name) (\n rate(workqueue_work_duration_seconds_count{job=~\"$job\"}[$__rate_interval])\n)", "legendFormat": "__auto", "range": true - } + }, + "version": "v0" }, - "refId": "A", - "hidden": false + "refId": "A" } } ], - "transformations": [], - "queryOptions": {} + "queryOptions": {}, + "transformations": [] } }, + "description": "", + "id": 20, + "links": [], + "title": "Processed Work Items / Second", "vizConfig": { - "kind": "VizConfig", "group": "timeseries", - "version": "13.1.0", + "kind": "VizConfig", "spec": { - "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, - "legend": { - "calcs": [ - "lastNotNull", - "max", - "mean" - ], - "displayMode": "table", - "enableFacetedFilter": false, - "overflow": "ellipsis", - "placement": "right", - "showLegend": true, - "sortBy": "Max", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, "fieldConfig": { "defaults": { - "unit": "ops", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": 0, - "color": "green" - }, - { - "value": 80, - "color": "red" - } - ] - }, "color": { "mode": "palette-classic" }, @@ -1483,21 +1447,57 @@ "thresholdsStyle": { "mode": "off" } - } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" }, "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "max", + "mean" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "right", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } } - } + }, + "version": "13.1.0" } } }, "panel-21": { "kind": "Panel", "spec": { - "id": 21, - "title": "Queue Adds / Second", - "description": "", - "links": [], "data": { "kind": "QueryGroup", "spec": { @@ -1505,75 +1505,39 @@ { "kind": "PanelQuery", "spec": { + "hidden": false, "query": { - "kind": "DataQuery", - "group": "prometheus", - "version": "v0", "datasource": { "name": "prometheus" }, + "group": "prometheus", + "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "sum by (name) (\n rate(workqueue_adds_total{job=~\"$job\"}[$__rate_interval])\n)", "legendFormat": "__auto", "range": true - } + }, + "version": "v0" }, - "refId": "A", - "hidden": false + "refId": "A" } } ], - "transformations": [], - "queryOptions": {} + "queryOptions": {}, + "transformations": [] } }, + "description": "", + "id": 21, + "links": [], + "title": "Queue Adds / Second", "vizConfig": { - "kind": "VizConfig", "group": "timeseries", - "version": "13.1.0", + "kind": "VizConfig", "spec": { - "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, - "legend": { - "calcs": [ - "lastNotNull", - "max", - "mean" - ], - "displayMode": "table", - "enableFacetedFilter": false, - "overflow": "ellipsis", - "placement": "right", - "showLegend": true, - "sortBy": "Max", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, "fieldConfig": { "defaults": { - "unit": "ops", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": 0, - "color": "green" - }, - { - "value": 80, - "color": "red" - } - ] - }, "color": { "mode": "palette-classic" }, @@ -1610,21 +1574,57 @@ "thresholdsStyle": { "mode": "off" } - } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" }, "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "max", + "mean" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "right", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } } - } + }, + "version": "13.1.0" } } }, "panel-22": { "kind": "Panel", "spec": { - "id": 22, - "title": "Backlog Pressure", - "description": "Positive means the queue is growing. Negative means the controller is draining backlog.", - "links": [], "data": { "kind": "QueryGroup", "spec": { @@ -1632,79 +1632,39 @@ { "kind": "PanelQuery", "spec": { + "hidden": false, "query": { - "kind": "DataQuery", - "group": "prometheus", - "version": "v0", "datasource": { "name": "prometheus" }, + "group": "prometheus", + "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "sum by (name) (\n rate(workqueue_adds_total{job=~\"$job\"}[$__rate_interval])\n)\n-\nsum by (name) (\n rate(workqueue_work_duration_seconds_count{job=~\"$job\"}[$__rate_interval])\n)", "legendFormat": "__auto", "range": true - } + }, + "version": "v0" }, - "refId": "A", - "hidden": false + "refId": "A" } } ], - "transformations": [], - "queryOptions": {} + "queryOptions": {}, + "transformations": [] } }, + "description": "Positive means the queue is growing. Negative means the controller is draining backlog.", + "id": 22, + "links": [], + "title": "Backlog Pressure", "vizConfig": { - "kind": "VizConfig", "group": "timeseries", - "version": "13.1.0", + "kind": "VizConfig", "spec": { - "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, - "legend": { - "calcs": [ - "lastNotNull", - "max", - "mean" - ], - "displayMode": "table", - "enableFacetedFilter": false, - "overflow": "ellipsis", - "placement": "right", - "showLegend": true, - "sortBy": "Max", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, "fieldConfig": { "defaults": { - "unit": "ops", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": 0, - "color": "green" - }, - { - "value": 0, - "color": "orange" - }, - { - "value": 1, - "color": "red" - } - ] - }, "color": { "mode": "palette-classic" }, @@ -1741,21 +1701,61 @@ "thresholdsStyle": { "mode": "dashed" } - } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "orange", + "value": 0 + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "ops" }, "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "max", + "mean" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "right", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } } - } + }, + "version": "13.1.0" } } }, "panel-23": { "kind": "Panel", "spec": { - "id": 23, - "title": "Average Queue Wait Duration", - "description": "Good next to p95/p99 to distinguish general slowness from spikes.", - "links": [], "data": { "kind": "QueryGroup", "spec": { @@ -1763,79 +1763,39 @@ { "kind": "PanelQuery", "spec": { + "hidden": false, "query": { - "kind": "DataQuery", - "group": "prometheus", - "version": "v0", "datasource": { "name": "prometheus" }, + "group": "prometheus", + "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "sum by (name) (\n rate(workqueue_queue_duration_seconds_sum{job=~\"$job\"}[$__rate_interval])\n)\n/\nsum by (name) (\n rate(workqueue_queue_duration_seconds_count{job=~\"$job\"}[$__rate_interval])\n)", "legendFormat": "__auto", "range": true - } + }, + "version": "v0" }, - "refId": "A", - "hidden": false + "refId": "A" } } ], - "transformations": [], - "queryOptions": {} + "queryOptions": {}, + "transformations": [] } }, + "description": "Good next to p95/p99 to distinguish general slowness from spikes.", + "id": 23, + "links": [], + "title": "Average Queue Wait Duration", "vizConfig": { - "kind": "VizConfig", "group": "timeseries", - "version": "13.1.0", + "kind": "VizConfig", "spec": { - "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, - "legend": { - "calcs": [ - "lastNotNull", - "max", - "mean" - ], - "displayMode": "table", - "enableFacetedFilter": false, - "overflow": "ellipsis", - "placement": "right", - "showLegend": true, - "sortBy": "Max", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, "fieldConfig": { "defaults": { - "unit": "s", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": 0, - "color": "green" - }, - { - "value": 30, - "color": "orange" - }, - { - "value": 120, - "color": "red" - } - ] - }, "color": { "mode": "palette-classic" }, @@ -1872,21 +1832,61 @@ "thresholdsStyle": { "mode": "dashed" } - } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "orange", + "value": 30 + }, + { + "color": "red", + "value": 120 + } + ] + }, + "unit": "s" }, "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "max", + "mean" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "right", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } } - } + }, + "version": "13.1.0" } } }, "panel-24": { "kind": "Panel", "spec": { - "id": 24, - "title": "Queue Retries / Second", - "description": "Spikes here usually mean reconcile errors, conflicts, or rate-limited requeues.", - "links": [], "data": { "kind": "QueryGroup", "spec": { @@ -1894,34 +1894,96 @@ { "kind": "PanelQuery", "spec": { + "hidden": false, "query": { - "kind": "DataQuery", - "group": "prometheus", - "version": "v0", "datasource": { "name": "prometheus" }, + "group": "prometheus", + "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "sum by (name) (\n rate(workqueue_retries_total{job=~\"$job\"}[$__rate_interval])\n)", "legendFormat": "__auto", "range": true - } + }, + "version": "v0" }, - "refId": "A", - "hidden": false + "refId": "A" } } ], - "transformations": [], - "queryOptions": {} + "queryOptions": {}, + "transformations": [] } }, + "description": "Spikes here usually mean reconcile errors, conflicts, or rate-limited requeues.", + "id": 24, + "links": [], + "title": "Queue Retries / Second", "vizConfig": { - "kind": "VizConfig", "group": "timeseries", - "version": "13.1.0", + "kind": "VizConfig", "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "dashed" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "orange", + "value": 0.1 + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, "options": { "annotations": { "clustering": -1, @@ -1946,77 +2008,15 @@ "mode": "single", "sort": "none" } - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": 0, - "color": "green" - }, - { - "value": 0.1, - "color": "orange" - }, - { - "value": 1, - "color": "red" - } - ] - }, - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "dashed" - } - } - }, - "overrides": [] } - } + }, + "version": "13.1.0" } } }, "panel-25": { "kind": "Panel", "spec": { - "id": 25, - "title": "Reconcile Duration p95", - "description": "", - "links": [], "data": { "kind": "QueryGroup", "spec": { @@ -2024,79 +2024,39 @@ { "kind": "PanelQuery", "spec": { + "hidden": false, "query": { - "kind": "DataQuery", - "group": "prometheus", - "version": "v0", "datasource": { "name": "prometheus" }, + "group": "prometheus", + "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "histogram_quantile(\n 0.95,\n sum by (controller, le) (\n rate(controller_runtime_reconcile_time_seconds_bucket{job=~\"$job\"}[$__rate_interval])\n )\n)", "legendFormat": "__auto", "range": true - } + }, + "version": "v0" }, - "refId": "A", - "hidden": false + "refId": "A" } } ], - "transformations": [], - "queryOptions": {} + "queryOptions": {}, + "transformations": [] } }, + "description": "", + "id": 25, + "links": [], + "title": "Reconcile Duration p95", "vizConfig": { - "kind": "VizConfig", "group": "timeseries", - "version": "13.1.0", + "kind": "VizConfig", "spec": { - "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, - "legend": { - "calcs": [ - "lastNotNull", - "max", - "mean" - ], - "displayMode": "table", - "enableFacetedFilter": false, - "overflow": "ellipsis", - "placement": "right", - "showLegend": true, - "sortBy": "Max", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, "fieldConfig": { "defaults": { - "unit": "s", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": 0, - "color": "green" - }, - { - "value": 5, - "color": "orange" - }, - { - "value": 15, - "color": "red" - } - ] - }, "color": { "mode": "palette-classic" }, @@ -2133,21 +2093,61 @@ "thresholdsStyle": { "mode": "dashed" } - } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "orange", + "value": 5 + }, + { + "color": "red", + "value": 15 + } + ] + }, + "unit": "s" }, "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "max", + "mean" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "right", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } } - } + }, + "version": "13.1.0" } } }, "panel-26": { "kind": "Panel", "spec": { - "id": 26, - "title": "Reconcile Duration Avg", - "description": "", - "links": [], "data": { "kind": "QueryGroup", "spec": { @@ -2155,79 +2155,39 @@ { "kind": "PanelQuery", "spec": { + "hidden": false, "query": { - "kind": "DataQuery", - "group": "prometheus", - "version": "v0", "datasource": { "name": "prometheus" }, + "group": "prometheus", + "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "sum by (controller) (\n rate(controller_runtime_reconcile_time_seconds_sum{job=~\"$job\"}[$__rate_interval])\n)\n/\nsum by (controller) (\n rate(controller_runtime_reconcile_time_seconds_count{job=~\"$job\"}[$__rate_interval])\n)", "legendFormat": "__auto", "range": true - } + }, + "version": "v0" }, - "refId": "A", - "hidden": false + "refId": "A" } } ], - "transformations": [], - "queryOptions": {} + "queryOptions": {}, + "transformations": [] } }, + "description": "", + "id": 26, + "links": [], + "title": "Reconcile Duration Avg", "vizConfig": { - "kind": "VizConfig", "group": "timeseries", - "version": "13.1.0", + "kind": "VizConfig", "spec": { - "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, - "legend": { - "calcs": [ - "lastNotNull", - "max", - "mean" - ], - "displayMode": "table", - "enableFacetedFilter": false, - "overflow": "ellipsis", - "placement": "right", - "showLegend": true, - "sortBy": "Max", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, "fieldConfig": { "defaults": { - "unit": "s", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": 0, - "color": "green" - }, - { - "value": 5, - "color": "orange" - }, - { - "value": 15, - "color": "red" - } - ] - }, "color": { "mode": "palette-classic" }, @@ -2264,56 +2224,28 @@ "thresholdsStyle": { "mode": "dashed" } - } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "orange", + "value": 5 + }, + { + "color": "red", + "value": 15 + } + ] + }, + "unit": "s" }, "overrides": [] - } - } - } - } - }, - "panel-27": { - "kind": "Panel", - "spec": { - "id": 27, - "title": "Max Workers", - "description": "Max Concurrent Reconciles", - "links": [], - "data": { - "kind": "QueryGroup", - "spec": { - "queries": [ - { - "kind": "PanelQuery", - "spec": { - "query": { - "kind": "DataQuery", - "group": "prometheus", - "version": "v0", - "datasource": { - "name": "prometheus" - }, - "spec": { - "editorMode": "code", - "expr": "controller_runtime_max_concurrent_reconciles{job=~\"$job\"}", - "legendFormat": "{{controller}}", - "range": true - } - }, - "refId": "A", - "hidden": false - } - } - ], - "transformations": [], - "queryOptions": {} - } - }, - "vizConfig": { - "kind": "VizConfig", - "group": "timeseries", - "version": "13.1.0", - "spec": { + }, "options": { "annotations": { "clustering": -1, @@ -2321,36 +2253,72 @@ }, "legend": { "calcs": [ - "lastNotNull" + "lastNotNull", + "max", + "mean" ], "displayMode": "table", "enableFacetedFilter": false, "overflow": "ellipsis", "placement": "right", - "showLegend": true + "showLegend": true, + "sortBy": "Max", + "sortDesc": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } - }, + } + }, + "version": "13.1.0" + } + } + }, + "panel-27": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "controller_runtime_max_concurrent_reconciles{job=~\"$job\"}", + "legendFormat": "{{controller}}", + "range": true + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Max Concurrent Reconciles", + "id": 27, + "links": [], + "title": "Max Workers", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { "fieldConfig": { "defaults": { - "unit": "short", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": 0, - "color": "green" - }, - { - "value": 80, - "color": "red" - } - ] - }, "color": { "mode": "palette-classic" }, @@ -2387,56 +2355,24 @@ "thresholdsStyle": { "mode": "off" } - } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" }, "overrides": [] - } - } - } - } - }, - "panel-28": { - "kind": "Panel", - "spec": { - "id": 28, - "title": "Worker Saturation", - "description": "If this is close to 1 while queue depth and queue wait rise, the controller is worker-saturated.", - "links": [], - "data": { - "kind": "QueryGroup", - "spec": { - "queries": [ - { - "kind": "PanelQuery", - "spec": { - "query": { - "kind": "DataQuery", - "group": "prometheus", - "version": "v0", - "datasource": { - "name": "prometheus" - }, - "spec": { - "editorMode": "code", - "expr": "controller_runtime_active_workers{job=~\"$job\"}\n/\ncontroller_runtime_max_concurrent_reconciles{job=~\"$job\"}", - "legendFormat": "{{controller}}", - "range": true - } - }, - "refId": "A", - "hidden": false - } - } - ], - "transformations": [], - "queryOptions": {} - } - }, - "vizConfig": { - "kind": "VizConfig", - "group": "timeseries", - "version": "13.1.0", - "spec": { + }, "options": { "annotations": { "clustering": -1, @@ -2444,44 +2380,68 @@ }, "legend": { "calcs": [ - "lastNotNull", - "max", - "mean" + "lastNotNull" ], "displayMode": "table", "enableFacetedFilter": false, "overflow": "ellipsis", "placement": "right", - "showLegend": true, - "sortBy": "Max", - "sortDesc": true + "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } - }, + } + }, + "version": "13.1.0" + } + } + }, + "panel-28": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "controller_runtime_active_workers{job=~\"$job\"}\n/\ncontroller_runtime_max_concurrent_reconciles{job=~\"$job\"}", + "legendFormat": "{{controller}}", + "range": true + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "If this is close to 1 while queue depth and queue wait rise, the controller is worker-saturated.", + "id": 28, + "links": [], + "title": "Worker Saturation", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { "fieldConfig": { "defaults": { - "unit": "percentunit", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": 0, - "color": "green" - }, - { - "value": 0.8, - "color": "orange" - }, - { - "value": 0.95, - "color": "red" - } - ] - }, "color": { "mode": "palette-classic" }, @@ -2520,21 +2480,61 @@ "thresholdsStyle": { "mode": "dashed" } - } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "orange", + "value": 0.8 + }, + { + "color": "red", + "value": 0.95 + } + ] + }, + "unit": "percentunit" }, "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "max", + "mean" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "right", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } } - } + }, + "version": "13.1.0" } } }, "panel-29": { "kind": "Panel", "spec": { - "id": 29, - "title": "Longest Running Processor", - "description": "This grows when work is still in-flight. If it grows continuously, suspect stuck reconciles or long API calls.", - "links": [], "data": { "kind": "QueryGroup", "spec": { @@ -2542,79 +2542,39 @@ { "kind": "PanelQuery", "spec": { + "hidden": false, "query": { - "kind": "DataQuery", - "group": "prometheus", - "version": "v0", "datasource": { "name": "prometheus" }, + "group": "prometheus", + "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "workqueue_longest_running_processor_seconds{job=~\"$job\"}", "legendFormat": "{{controller}}", "range": true - } + }, + "version": "v0" }, - "refId": "A", - "hidden": false + "refId": "A" } } ], - "transformations": [], - "queryOptions": {} + "queryOptions": {}, + "transformations": [] } }, + "description": "This grows when work is still in-flight. If it grows continuously, suspect stuck reconciles or long API calls.", + "id": 29, + "links": [], + "title": "Longest Running Processor", "vizConfig": { - "kind": "VizConfig", "group": "timeseries", - "version": "13.1.0", + "kind": "VizConfig", "spec": { - "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, - "legend": { - "calcs": [ - "lastNotNull", - "max", - "mean" - ], - "displayMode": "table", - "enableFacetedFilter": false, - "overflow": "ellipsis", - "placement": "right", - "showLegend": true, - "sortBy": "Max", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, "fieldConfig": { "defaults": { - "unit": "s", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": 0, - "color": "green" - }, - { - "value": 60, - "color": "orange" - }, - { - "value": 300, - "color": "red" - } - ] - }, "color": { "mode": "palette-classic" }, @@ -2651,21 +2611,61 @@ "thresholdsStyle": { "mode": "dashed" } - } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "orange", + "value": 60 + }, + { + "color": "red", + "value": 300 + } + ] + }, + "unit": "s" }, "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "max", + "mean" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "right", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } } - } + }, + "version": "13.1.0" } } }, "panel-30": { "kind": "Panel", "spec": { - "id": 30, - "title": "Unfinished Work Seconds", - "description": "This grows when work is still in-flight. If it grows continuously, suspect stuck reconciles or long API calls.", - "links": [], "data": { "kind": "QueryGroup", "spec": { @@ -2673,79 +2673,39 @@ { "kind": "PanelQuery", "spec": { + "hidden": false, "query": { - "kind": "DataQuery", - "group": "prometheus", - "version": "v0", "datasource": { "name": "prometheus" }, + "group": "prometheus", + "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "workqueue_unfinished_work_seconds{job=~\"$job\"}", "legendFormat": "{{controller}}", "range": true - } + }, + "version": "v0" }, - "refId": "A", - "hidden": false + "refId": "A" } } ], - "transformations": [], - "queryOptions": {} + "queryOptions": {}, + "transformations": [] } }, + "description": "This grows when work is still in-flight. If it grows continuously, suspect stuck reconciles or long API calls.", + "id": 30, + "links": [], + "title": "Unfinished Work Seconds", "vizConfig": { - "kind": "VizConfig", "group": "timeseries", - "version": "13.1.0", + "kind": "VizConfig", "spec": { - "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, - "legend": { - "calcs": [ - "lastNotNull", - "max", - "mean" - ], - "displayMode": "table", - "enableFacetedFilter": false, - "overflow": "ellipsis", - "placement": "right", - "showLegend": true, - "sortBy": "Max", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, "fieldConfig": { "defaults": { - "unit": "s", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": 0, - "color": "green" - }, - { - "value": 300, - "color": "orange" - }, - { - "value": 500, - "color": "red" - } - ] - }, "color": { "mode": "palette-classic" }, @@ -2782,21 +2742,61 @@ "thresholdsStyle": { "mode": "dashed" } - } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "orange", + "value": 300 + }, + { + "color": "red", + "value": 500 + } + ] + }, + "unit": "s" }, "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "max", + "mean" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "right", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } } - } + }, + "version": "13.1.0" } } }, "panel-31": { "kind": "Panel", "spec": { - "id": 31, - "title": "Kubernetes API Client Requests", - "description": "This is useful to see whether Capsule is generating a high API call rate.", - "links": [], "data": { "kind": "QueryGroup", "spec": { @@ -2804,34 +2804,93 @@ { "kind": "PanelQuery", "spec": { + "hidden": false, "query": { - "kind": "DataQuery", - "group": "prometheus", - "version": "v0", "datasource": { "name": "prometheus" }, + "group": "prometheus", + "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "sum by (method, code) (\n rate(rest_client_requests_total{job=~\"$job\"}[$__rate_interval])\n)", "legendFormat": "{{method}} {{code}}", "range": true - } + }, + "version": "v0" }, - "refId": "A", - "hidden": false + "refId": "A" } } ], - "transformations": [], - "queryOptions": {} + "queryOptions": {}, + "transformations": [] } }, + "description": "This is useful to see whether Capsule is generating a high API call rate.", + "id": 31, + "links": [], + "title": "Kubernetes API Client Requests", "vizConfig": { - "kind": "VizConfig", "group": "timeseries", - "version": "13.1.0", + "kind": "VizConfig", "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, "options": { "annotations": { "clustering": -1, @@ -2856,74 +2915,15 @@ "mode": "single", "sort": "none" } - }, - "fieldConfig": { - "defaults": { - "unit": "ops", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": 0, - "color": "green" - }, - { - "value": 80, - "color": "red" - } - ] - }, - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - } - }, - "overrides": [] } - } + }, + "version": "13.1.0" } } }, "panel-32": { "kind": "Panel", "spec": { - "id": 32, - "title": "Kubernetes API Client Errors", - "description": "", - "links": [], "data": { "kind": "QueryGroup", "spec": { @@ -2931,73 +2931,39 @@ { "kind": "PanelQuery", "spec": { + "hidden": false, "query": { - "kind": "DataQuery", - "group": "prometheus", - "version": "v0", "datasource": { "name": "prometheus" }, + "group": "prometheus", + "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "sum by (method, code) (\n rate(rest_client_requests_total{\n job=~\"$job\",\n code=\"429\"\n }[$__rate_interval])\n)", "legendFormat": "{{method}} {{code}}", "range": true - } + }, + "version": "v0" }, - "refId": "A", - "hidden": false + "refId": "A" } } ], - "transformations": [], - "queryOptions": {} + "queryOptions": {}, + "transformations": [] } }, + "description": "", + "id": 32, + "links": [], + "title": "Kubernetes API Client Errors", "vizConfig": { - "kind": "VizConfig", "group": "timeseries", - "version": "13.1.0", + "kind": "VizConfig", "spec": { - "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, - "legend": { - "calcs": [ - "lastNotNull", - "max", - "mean" - ], - "displayMode": "table", - "enableFacetedFilter": false, - "overflow": "ellipsis", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, "fieldConfig": { "defaults": { - "unit": "ops", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": 0, - "color": "green" - }, - { - "value": 80, - "color": "red" - } - ] - }, "color": { "mode": "palette-classic" }, @@ -3034,21 +3000,55 @@ "thresholdsStyle": { "mode": "off" } - } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" }, "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "max", + "mean" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } } - } + }, + "version": "13.1.0" } } }, "panel-33": { "kind": "Panel", "spec": { - "id": 33, - "title": "Kubernetes API Client Request Duration p95", - "description": "", - "links": [], "data": { "kind": "QueryGroup", "spec": { @@ -3056,73 +3056,39 @@ { "kind": "PanelQuery", "spec": { + "hidden": false, "query": { - "kind": "DataQuery", - "group": "prometheus", - "version": "v0", "datasource": { "name": "prometheus" }, + "group": "prometheus", + "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "histogram_quantile(\n 0.95,\n sum by (method, le) (\n rate(rest_client_request_duration_seconds_bucket{job=~\"$job\"}[$__rate_interval])\n )\n)", "legendFormat": "{{method}}", "range": true - } + }, + "version": "v0" }, - "refId": "A", - "hidden": false + "refId": "A" } } ], - "transformations": [], - "queryOptions": {} + "queryOptions": {}, + "transformations": [] } }, + "description": "", + "id": 33, + "links": [], + "title": "Kubernetes API Client Request Duration p95", "vizConfig": { - "kind": "VizConfig", "group": "timeseries", - "version": "13.1.0", + "kind": "VizConfig", "spec": { - "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, - "legend": { - "calcs": [ - "lastNotNull", - "max", - "mean" - ], - "displayMode": "table", - "enableFacetedFilter": false, - "overflow": "ellipsis", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, "fieldConfig": { "defaults": { - "unit": "s", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": 0, - "color": "green" - }, - { - "value": 80, - "color": "red" - } - ] - }, "color": { "mode": "palette-classic" }, @@ -3159,21 +3125,55 @@ "thresholdsStyle": { "mode": "off" } - } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" }, "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "max", + "mean" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } } - } + }, + "version": "13.1.0" } } }, "panel-34": { "kind": "Panel", "spec": { - "id": 34, - "title": "Reoncile Rate", - "description": "", - "links": [], "data": { "kind": "QueryGroup", "spec": { @@ -3181,13 +3181,13 @@ { "kind": "PanelQuery", "spec": { + "hidden": false, "query": { - "kind": "DataQuery", - "group": "prometheus", - "version": "v0", "datasource": { "name": "prometheus" }, + "group": "prometheus", + "kind": "DataQuery", "spec": { "editorMode": "code", "exemplar": false, @@ -3196,22 +3196,46 @@ "interval": "", "legendFormat": "{{controller}}", "range": true - } + }, + "version": "v0" }, - "refId": "A", - "hidden": false + "refId": "A" } } ], - "transformations": [], - "queryOptions": {} + "queryOptions": {}, + "transformations": [] } }, + "description": "", + "id": 34, + "links": [], + "title": "Reoncile Rate", "vizConfig": { - "kind": "VizConfig", "group": "bargauge", - "version": "13.1.0", + "kind": "VizConfig", "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, "options": { "displayMode": "lcd", "legend": { @@ -3236,39 +3260,15 @@ "showUnfilled": true, "sizing": "auto", "valueMode": "text" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": 0, - "color": "green" - }, - { - "value": 80, - "color": "red" - } - ] - }, - "color": { - "mode": "continuous-GrYlRd" - } - }, - "overrides": [] } - } + }, + "version": "13.1.0" } } }, "panel-35": { "kind": "Panel", "spec": { - "id": 35, - "title": "APF Current In-Queue Requests", - "description": "", - "links": [], "data": { "kind": "QueryGroup", "spec": { @@ -3276,68 +3276,39 @@ { "kind": "PanelQuery", "spec": { + "hidden": false, "query": { - "kind": "DataQuery", - "group": "prometheus", - "version": "v0", "datasource": { "name": "prometheus" }, + "group": "prometheus", + "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "sum by (priority_level, flow_schema) (\n apiserver_flowcontrol_current_inqueue_requests{\n flow_schema=~\".*capsule.*\"\n }\n)", "legendFormat": "__auto", "range": true - } + }, + "version": "v0" }, - "refId": "A", - "hidden": false + "refId": "A" } } ], - "transformations": [], - "queryOptions": {} + "queryOptions": {}, + "transformations": [] } }, + "description": "", + "id": 35, + "links": [], + "title": "APF Current In-Queue Requests", "vizConfig": { - "kind": "VizConfig", "group": "timeseries", - "version": "13.1.0", + "kind": "VizConfig", "spec": { - "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, - "legend": { - "calcs": [], - "displayMode": "list", - "enableFacetedFilter": false, - "overflow": "ellipsis", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, "fieldConfig": { "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": 0, - "color": "green" - }, - { - "value": 80, - "color": "red" - } - ] - }, "color": { "mode": "palette-classic" }, @@ -3374,21 +3345,50 @@ "thresholdsStyle": { "mode": "off" } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] } }, "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } } - } + }, + "version": "13.1.0" } } }, - "panel-4": { + "panel-37": { "kind": "Panel", "spec": { - "id": 4, - "title": "Reoncile Rate", - "description": "", - "links": [], "data": { "kind": "QueryGroup", "spec": { @@ -3396,13 +3396,392 @@ { "kind": "PanelQuery", "spec": { + "hidden": false, "query": { - "kind": "DataQuery", - "group": "prometheus", - "version": "v0", "datasource": { "name": "prometheus" }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "sum by (webhook) (rate(controller_runtime_webhook_requests_in_flight{job=\"$job\"}[$__rate_interval]))", + "legendFormat": "__auto", + "range": true + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "", + "id": 37, + "links": [], + "title": "Requests In-Flight", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "last", + "max", + "mean" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "right", + "showLegend": true, + "sortBy": "Last", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.1.0" + } + } + }, + "panel-38": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "sum by (webhook) (rate(controller_runtime_webhook_requests_total{job=\"$job\"}[$__rate_interval]))", + "legendFormat": "__auto", + "range": true + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "", + "id": 38, + "links": [], + "title": "Admission Requests", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "last", + "max", + "mean" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "right", + "showLegend": true, + "sortBy": "Last", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.1.0" + } + } + }, + "panel-39": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "sum by (webhook) (\n rate(controller_runtime_webhook_latency_seconds_sum{job=\"$job\"}[5m])\n)\n/\nsum by (webhook) (\n rate(controller_runtime_webhook_latency_seconds_count{job=\"$job\"}[5m])\n)", + "legendFormat": "__auto", + "range": true + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "", + "id": 39, + "links": [], + "title": "Webhook Latency Average", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "last", + "max", + "mean" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "right", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.1.0" + } + } + }, + "panel-4": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", "spec": { "editorMode": "code", "exemplar": false, @@ -3411,82 +3790,49 @@ "interval": "", "legendFormat": "{{result}}", "range": true - } + }, + "version": "v0" }, - "refId": "A", - "hidden": false + "refId": "A" } }, { "kind": "PanelQuery", "spec": { + "hidden": false, "query": { - "kind": "DataQuery", - "group": "prometheus", - "version": "v0", "datasource": { "name": "prometheus" }, + "group": "prometheus", + "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "sum(\n increase(controller_runtime_reconcile_total{job=~\"$job\"}[$__rate_interval])\n)", "instant": false, "legendFormat": "Total", "range": true - } + }, + "version": "v0" }, - "refId": "B", - "hidden": false + "refId": "B" } } ], - "transformations": [], - "queryOptions": {} + "queryOptions": {}, + "transformations": [] } }, + "description": "", + "id": 4, + "links": [], + "title": "Reoncile Rate", "vizConfig": { - "kind": "VizConfig", "group": "timeseries", - "version": "13.1.0", + "kind": "VizConfig", "spec": { - "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "enableFacetedFilter": false, - "overflow": "ellipsis", - "placement": "right", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, "fieldConfig": { "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": 0, - "color": "green" - }, - { - "value": 80, - "color": "red" - } - ] - }, "color": { "mode": "palette-classic" }, @@ -3523,11 +3869,623 @@ "thresholdsStyle": { "mode": "off" } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] } }, "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "right", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } } + }, + "version": "13.1.0" + } + } + }, + "panel-40": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "histogram_quantile(\n 0.99,\n sum by (le, webhook) (rate(controller_runtime_webhook_latency_seconds_bucket{job=\"$job\"}[5m]))\n)", + "legendFormat": "__auto", + "range": true + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] } + }, + "description": "", + "id": 40, + "links": [], + "title": "Webhook Latency (P99)", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "last", + "max", + "mean" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "right", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.1.0" + } + } + }, + "panel-41": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "histogram_quantile(\n 0.95,\n sum by (le, webhook) (rate(controller_runtime_webhook_latency_seconds_bucket{job=\"$job\"}[5m]))\n)", + "legendFormat": "__auto", + "range": true + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "", + "id": 41, + "links": [], + "title": "Webhook Latency (P95)", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "last", + "max", + "mean" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "right", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.1.0" + } + } + }, + "panel-42": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "sum by (webhook) (\n rate(controller_runtime_webhook_requests_total{job=\"$job\"}[$__rate_interval])\n)", + "legendFormat": "{{webhook}}", + "range": true + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "", + "id": 42, + "links": [], + "title": "Webhook Requests / Second", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "last", + "max", + "mean" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "right", + "showLegend": true, + "sortBy": "Last", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.1.0" + } + } + }, + "panel-43": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "topk(\n 10,\n histogram_quantile(\n 0.99,\n sum by (name, type, le) (\n rate(apiserver_admission_webhook_admission_duration_seconds_bucket[$__rate_interval])\n )\n )\n)", + "legendFormat": "__auto", + "range": true + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "", + "id": 43, + "links": [], + "title": "Slowest Admission Webhooks p99", + "vizConfig": { + "group": "bargauge", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "orange", + "value": 1 + }, + { + "color": "red", + "value": 5 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "options": { + "displayMode": "lcd", + "legend": { + "calcs": [], + "displayMode": "list", + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "top", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "text" + } + }, + "version": "13.1.0" + } + } + }, + "panel-44": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "topk(\n 10,\n sum by (name, type) (\n rate(apiserver_admission_webhook_admission_duration_seconds_sum[$__rate_interval])\n )\n /\n sum by (name, type) (\n rate(apiserver_admission_webhook_admission_duration_seconds_count[$__rate_interval])\n )\n)", + "legendFormat": "__auto", + "range": true + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "", + "id": 44, + "links": [], + "title": "New panel", + "vizConfig": { + "group": "bargauge", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "#EAB839", + "value": 0.5 + }, + { + "color": "red", + "value": 2 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "options": { + "displayMode": "lcd", + "legend": { + "calcs": [], + "displayMode": "list", + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "text" + } + }, + "version": "13.1.0" } } } @@ -3539,7 +4497,6 @@ { "kind": "RowsLayoutRow", "spec": { - "title": "Overview", "collapse": false, "layout": { "kind": "RowsLayout", @@ -3548,7 +4505,6 @@ { "kind": "RowsLayoutRow", "spec": { - "title": "Workers", "collapse": false, "layout": { "kind": "GridLayout", @@ -3557,103 +4513,103 @@ { "kind": "GridLayoutItem", "spec": { - "x": 0, - "y": 0, - "width": 12, - "height": 8, "element": { "kind": "ElementReference", "name": "panel-2" - } + }, + "height": 8, + "width": 12, + "x": 0, + "y": 0 } }, { "kind": "GridLayoutItem", "spec": { - "x": 12, - "y": 0, - "width": 12, - "height": 8, "element": { "kind": "ElementReference", "name": "panel-28" - } + }, + "height": 8, + "width": 12, + "x": 12, + "y": 0 } }, { "kind": "GridLayoutItem", "spec": { - "x": 0, - "y": 8, - "width": 12, - "height": 8, "element": { "kind": "ElementReference", "name": "panel-27" - } + }, + "height": 8, + "width": 12, + "x": 0, + "y": 8 } }, { "kind": "GridLayoutItem", "spec": { - "x": 12, - "y": 8, - "width": 12, - "height": 8, "element": { "kind": "ElementReference", "name": "panel-4" - } + }, + "height": 8, + "width": 12, + "x": 12, + "y": 8 } }, { "kind": "GridLayoutItem", "spec": { - "x": 0, - "y": 16, - "width": 12, - "height": 8, "element": { "kind": "ElementReference", "name": "panel-25" - } + }, + "height": 8, + "width": 12, + "x": 0, + "y": 16 } }, { "kind": "GridLayoutItem", "spec": { - "x": 12, - "y": 16, - "width": 12, - "height": 8, "element": { "kind": "ElementReference", "name": "panel-26" - } + }, + "height": 8, + "width": 12, + "x": 12, + "y": 16 } }, { "kind": "GridLayoutItem", "spec": { - "x": 0, - "y": 24, - "width": 24, - "height": 12, "element": { "kind": "ElementReference", "name": "panel-34" - } + }, + "height": 12, + "width": 24, + "x": 0, + "y": 24 } } ] } - } + }, + "title": "Workers" } }, { "kind": "RowsLayoutRow", "spec": { - "title": "Workqueue", "collapse": false, "layout": { "kind": "RowsLayout", @@ -3662,14 +4618,147 @@ { "kind": "RowsLayoutRow", "spec": { - "title": "Depth", "collapse": false, "layout": { "kind": "AutoGridLayout", "spec": { - "maxColumnCount": 3, "columnWidthMode": "standard", - "rowHeightMode": "standard", + "items": [ + { + "kind": "AutoGridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-38" + } + } + }, + { + "kind": "AutoGridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-42" + } + } + }, + { + "kind": "AutoGridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-37" + } + } + }, + { + "kind": "AutoGridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-39" + } + } + }, + { + "kind": "AutoGridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-40" + } + } + }, + { + "kind": "AutoGridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-41" + } + } + } + ], + "maxColumnCount": 2, + "rowHeightMode": "standard" + } + }, + "title": "Webhooks" + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "collapse": false, + "layout": { + "kind": "AutoGridLayout", + "spec": { + "columnWidthMode": "standard", + "items": [ + { + "kind": "AutoGridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-43" + } + } + }, + { + "kind": "AutoGridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-44" + } + } + } + ], + "maxColumnCount": 2, + "rowHeightMode": "tall" + } + }, + "title": "Comparison" + } + } + ] + } + }, + "title": "Admission" + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "collapse": false, + "layout": { + "kind": "AutoGridLayout", + "spec": { + "columnWidthMode": "standard", + "items": [], + "maxColumnCount": 3, + "rowHeightMode": "standard" + } + }, + "title": "New row" + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "collapse": true, + "layout": { + "kind": "RowsLayout", + "spec": { + "rows": [ + { + "kind": "RowsLayoutRow", + "spec": { + "collapse": false, + "layout": { + "kind": "AutoGridLayout", + "spec": { + "columnWidthMode": "standard", "items": [ { "kind": "AutoGridLayoutItem", @@ -3725,22 +4814,22 @@ } } } - ] + ], + "maxColumnCount": 3, + "rowHeightMode": "standard" } - } + }, + "title": "Depth" } }, { "kind": "RowsLayoutRow", "spec": { - "title": "Work Duration", "collapse": false, "layout": { "kind": "AutoGridLayout", "spec": { - "maxColumnCount": 3, "columnWidthMode": "standard", - "rowHeightMode": "standard", "items": [ { "kind": "AutoGridLayoutItem", @@ -3796,22 +4885,22 @@ } } } - ] + ], + "maxColumnCount": 3, + "rowHeightMode": "standard" } - } + }, + "title": "Work Duration" } }, { "kind": "RowsLayoutRow", "spec": { - "title": "Unfinished Work", "collapse": false, "layout": { "kind": "AutoGridLayout", "spec": { - "maxColumnCount": 3, "columnWidthMode": "standard", - "rowHeightMode": "standard", "items": [ { "kind": "AutoGridLayoutItem", @@ -3831,21 +4920,24 @@ } } } - ] + ], + "maxColumnCount": 3, + "rowHeightMode": "standard" } - } + }, + "title": "Unfinished Work" } } ] } - } + }, + "title": "Workqueue" } }, { "kind": "RowsLayoutRow", "spec": { - "title": "Kubernetes API", - "collapse": false, + "collapse": true, "layout": { "kind": "RowsLayout", "spec": { @@ -3853,14 +4945,11 @@ { "kind": "RowsLayoutRow", "spec": { - "title": "Controller Client", "collapse": false, "layout": { "kind": "AutoGridLayout", "spec": { - "maxColumnCount": 3, "columnWidthMode": "standard", - "rowHeightMode": "standard", "items": [ { "kind": "AutoGridLayoutItem", @@ -3889,22 +4978,22 @@ } } } - ] + ], + "maxColumnCount": 3, + "rowHeightMode": "standard" } - } + }, + "title": "Controller Client" } }, { "kind": "RowsLayoutRow", "spec": { - "title": "Flowschema", "collapse": false, "layout": { "kind": "AutoGridLayout", "spec": { - "maxColumnCount": 3, "columnWidthMode": "standard", - "rowHeightMode": "standard", "items": [ { "kind": "AutoGridLayoutItem", @@ -3915,30 +5004,30 @@ } } } - ] + ], + "maxColumnCount": 3, + "rowHeightMode": "standard" } - } + }, + "title": "Flowschema" } } ] } - } + }, + "title": "Kubernetes API" } } ] } - } + }, + "title": "Overview" } }, { "kind": "RowsLayoutRow", "spec": { - "title": "Controller \"$Controller\" Status", "collapse": false, - "repeat": { - "mode": "variable", - "value": "Controller" - }, "layout": { "kind": "GridLayout", "spec": { @@ -3946,32 +5035,37 @@ { "kind": "GridLayoutItem", "spec": { - "x": 0, - "y": 0, - "width": 24, - "height": 9, "element": { "kind": "ElementReference", "name": "panel-10" - } + }, + "height": 9, + "width": 24, + "x": 0, + "y": 0 } }, { "kind": "GridLayoutItem", "spec": { - "x": 0, - "y": 9, - "width": 24, - "height": 10, "element": { "kind": "ElementReference", "name": "panel-12" - } + }, + "height": 10, + "width": 24, + "x": 0, + "y": 9 } } ] } - } + }, + "repeat": { + "mode": "variable", + "value": "Controller" + }, + "title": "Controller \"$Controller\" Status" } } ] @@ -3982,9 +5076,6 @@ "preload": false, "tags": [], "timeSettings": { - "timezone": "browser", - "from": "now-15m", - "to": "now", "autoRefresh": "10s", "autoRefreshIntervals": [ "5s", @@ -3998,84 +5089,87 @@ "2h", "1d" ], + "fiscalYearStartMonth": 0, + "from": "now-15m", "hideTimepicker": false, - "fiscalYearStartMonth": 0 + "timezone": "browser", + "to": "now" }, "title": "Controller Runtime Controllers", "variables": [ { "kind": "QueryVariable", "spec": { - "name": "job", + "allowCustomValue": true, "current": { "text": "All", "value": [ "$__all" ] }, - "label": "Job", + "definition": "label_values(controller_runtime_active_workers,job)", "hide": "dontHide", - "refresh": "onTimeRangeChanged", - "skipUrlSync": false, + "includeAll": true, + "label": "Job", + "multi": true, + "name": "job", + "options": [], "query": { - "kind": "DataQuery", - "group": "prometheus", - "version": "v0", "datasource": { "name": "prometheus" }, + "group": "prometheus", + "kind": "DataQuery", "spec": { "qryType": 1, "query": "label_values(controller_runtime_active_workers,job)", "refId": "PrometheusVariableQueryEditor-VariableQuery" - } + }, + "version": "v0" }, + "refresh": "onTimeRangeChanged", "regex": "", "regexApplyTo": "value", - "sort": "disabled", - "definition": "label_values(controller_runtime_active_workers,job)", - "options": [], - "multi": true, - "includeAll": true, - "allowCustomValue": true + "skipUrlSync": false, + "sort": "disabled" } }, { "kind": "QueryVariable", "spec": { - "name": "Controller", + "allowCustomValue": true, "current": { "text": "All", "value": [ "$__all" ] }, + "definition": "label_values(controller_runtime_active_workers{job=~\"$job\"},controller)", "hide": "dontHide", - "refresh": "onDashboardLoad", - "skipUrlSync": false, + "includeAll": true, + "multi": true, + "name": "Controller", + "options": [], "query": { - "kind": "DataQuery", - "group": "prometheus", - "version": "v0", "datasource": { "name": "prometheus" }, + "group": "prometheus", + "kind": "DataQuery", "spec": { "qryType": 1, "query": "label_values(controller_runtime_active_workers{job=~\"$job\"},controller)", "refId": "PrometheusVariableQueryEditor-VariableQuery" - } + }, + "version": "v0" }, + "refresh": "onDashboardLoad", "regex": "", "regexApplyTo": "value", - "sort": "disabled", - "definition": "label_values(controller_runtime_active_workers{job=~\"$job\"},controller)", - "options": [], - "multi": true, - "includeAll": true, - "allowCustomValue": true + "skipUrlSync": false, + "sort": "disabled" } } ] } -} \ No newline at end of file +} diff --git a/charts/capsule/templates/configuration.yaml b/charts/capsule/templates/configuration.yaml index c708327a..bf9dcdd5 100644 --- a/charts/capsule/templates/configuration.yaml +++ b/charts/capsule/templates/configuration.yaml @@ -86,8 +86,13 @@ spec: objectSelector: {{- toYaml . | nindent 10 }} {{- end }} - {{- with .matchConditions }} + {{- if or .matchConditions $.Values.webhooks.matchConditions }} matchConditions: + {{- end }} + {{- with .matchConditions }} + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with $.Values.webhooks.matchConditions }} {{- toYaml . | nindent 10 }} {{- end }} rules: @@ -162,8 +167,13 @@ spec: objectSelector: {{- toYaml . | nindent 10 }} {{- end }} - {{- with .matchConditions }} + {{- if or .matchConditions $.Values.webhooks.matchConditions }} matchConditions: + {{- end }} + {{- with .matchConditions }} + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with $.Values.webhooks.matchConditions }} {{- toYaml . | nindent 10 }} {{- end }} rules: @@ -204,8 +214,13 @@ spec: objectSelector: {{- toYaml . | nindent 10 }} {{- end }} - {{- with .matchConditions }} + {{- if or .matchConditions $.Values.webhooks.matchConditions }} matchConditions: + {{- end }} + {{- with .matchConditions }} + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with $.Values.webhooks.matchConditions }} {{- toYaml . | nindent 10 }} {{- end }} rules: @@ -246,8 +261,13 @@ spec: objectSelector: {{- toYaml . | nindent 10 }} {{- end }} - {{- with .matchConditions }} + {{- if or .matchConditions $.Values.webhooks.matchConditions }} matchConditions: + {{- end }} + {{- with .matchConditions }} + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with $.Values.webhooks.matchConditions }} {{- toYaml . | nindent 10 }} {{- end }} {{- with .rules }} @@ -280,8 +300,13 @@ spec: objectSelector: {{- toYaml . | nindent 10 }} {{- end }} - {{- with .matchConditions }} + {{- if or .matchConditions $.Values.webhooks.matchConditions }} matchConditions: + {{- end }} + {{- with .matchConditions }} + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with $.Values.webhooks.matchConditions }} {{- toYaml . | nindent 10 }} {{- end }} rules: @@ -322,8 +347,13 @@ spec: objectSelector: {{- toYaml . | nindent 10 }} {{- end }} - {{- with .matchConditions }} + {{- if or .matchConditions $.Values.webhooks.matchConditions }} matchConditions: + {{- end }} + {{- with .matchConditions }} + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with $.Values.webhooks.matchConditions }} {{- toYaml . | nindent 10 }} {{- end }} rules: @@ -363,8 +393,13 @@ spec: objectSelector: {{- toYaml . | nindent 10 }} {{- end }} - {{- with .matchConditions }} + {{- if or .matchConditions $.Values.webhooks.matchConditions }} matchConditions: + {{- end }} + {{- with .matchConditions }} + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with $.Values.webhooks.matchConditions }} {{- toYaml . | nindent 10 }} {{- end }} rules: @@ -406,8 +441,13 @@ spec: objectSelector: {{- toYaml . | nindent 10 }} {{- end }} - {{- with .matchConditions }} + {{- if or .matchConditions $.Values.webhooks.matchConditions }} matchConditions: + {{- end }} + {{- with .matchConditions }} + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with $.Values.webhooks.matchConditions }} {{- toYaml . | nindent 10 }} {{- end }} rules: @@ -445,8 +485,13 @@ spec: objectSelector: {{- toYaml . | nindent 10 }} {{- end }} - {{- with .matchConditions }} + {{- if or .matchConditions $.Values.webhooks.matchConditions }} matchConditions: + {{- end }} + {{- with .matchConditions }} + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with $.Values.webhooks.matchConditions }} {{- toYaml . | nindent 10 }} {{- end }} rules: @@ -487,10 +532,18 @@ spec: objectSelector: {{- toYaml . | nindent 10 }} {{- end }} - {{- with .matchConditions }} matchConditions: + {{- with .matchConditions }} {{- toYaml . | nindent 10 }} {{- end }} + {{- with $.Values.webhooks.matchConditions }} + {{- toYaml . | nindent 10 }} + {{- end }} + - name: requires-pvc-spec-validation + expression: > + request.operation != "UPDATE" || + !has(object.metadata.deletionTimestamp) || + object.spec != oldObject.spec rules: - apiGroups: - "" @@ -528,8 +581,13 @@ spec: objectSelector: {{- toYaml . | nindent 10 }} {{- end }} - {{- with .matchConditions }} + {{- if or .matchConditions $.Values.webhooks.matchConditions }} matchConditions: + {{- end }} + {{- with .matchConditions }} + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with $.Values.webhooks.matchConditions }} {{- toYaml . | nindent 10 }} {{- end }} rules: @@ -568,8 +626,13 @@ spec: objectSelector: {{- toYaml . | nindent 10 }} {{- end }} - {{- with .matchConditions }} + {{- if or .matchConditions $.Values.webhooks.matchConditions }} matchConditions: + {{- end }} + {{- with .matchConditions }} + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with $.Values.webhooks.matchConditions }} {{- toYaml . | nindent 10 }} {{- end }} rules: @@ -599,8 +662,13 @@ spec: objectSelector: {{- toYaml . | nindent 10 }} {{- end }} - {{- with .matchConditions }} + {{- if or .matchConditions $.Values.webhooks.matchConditions }} matchConditions: + {{- end }} + {{- with .matchConditions }} + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with $.Values.webhooks.matchConditions }} {{- toYaml . | nindent 10 }} {{- end }} rules: @@ -631,8 +699,13 @@ spec: objectSelector: {{- toYaml . | nindent 10 }} {{- end }} - {{- with .matchConditions }} + {{- if or .matchConditions $.Values.webhooks.matchConditions }} matchConditions: + {{- end }} + {{- with .matchConditions }} + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with $.Values.webhooks.matchConditions }} {{- toYaml . | nindent 10 }} {{- end }} rules: @@ -673,8 +746,13 @@ spec: objectSelector: {{- toYaml . | nindent 10 }} {{- end }} - {{- with .matchConditions }} + {{- if or .matchConditions $.Values.webhooks.matchConditions }} matchConditions: + {{- end }} + {{- with .matchConditions }} + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with $.Values.webhooks.matchConditions }} {{- toYaml . | nindent 10 }} {{- end }} rules: @@ -715,8 +793,13 @@ spec: objectSelector: {{- toYaml . | nindent 10 }} {{- end }} - {{- with .matchConditions }} + {{- if or .matchConditions $.Values.webhooks.matchConditions }} matchConditions: + {{- end }} + {{- with .matchConditions }} + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with $.Values.webhooks.matchConditions }} {{- toYaml . | nindent 10 }} {{- end }} rules: @@ -756,8 +839,13 @@ spec: objectSelector: {{- toYaml . | nindent 10 }} {{- end }} - {{- with .matchConditions }} + {{- if or .matchConditions $.Values.webhooks.matchConditions }} matchConditions: + {{- end }} + {{- with .matchConditions }} + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with $.Values.webhooks.matchConditions }} {{- toYaml . | nindent 10 }} {{- end }} rules: @@ -798,10 +886,20 @@ spec: objectSelector: {{- toYaml . | nindent 10 }} {{- end }} - {{- with .matchConditions }} matchConditions: + {{- with .matchConditions }} {{- toYaml . | nindent 10 }} {{- end }} + {{- with $.Values.webhooks.matchConditions }} + {{- toYaml . | nindent 10 }} + {{- end }} + - name: requires-promotion-label + expression: > + has(object.metadata.labels) && + ( + "projectcapsule.dev/promote" in object.metadata.labels || + "owner.projectcapsule.dev/promote" in object.metadata.labels + ) rules: - apiGroups: - '*' @@ -839,8 +937,13 @@ spec: objectSelector: {{- toYaml . | nindent 10 }} {{- end }} - {{- with .matchConditions }} + {{- if or .matchConditions $.Values.webhooks.matchConditions }} matchConditions: + {{- end }} + {{- with .matchConditions }} + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with $.Values.webhooks.matchConditions }} {{- toYaml . | nindent 10 }} {{- end }} rules: @@ -880,8 +983,13 @@ spec: objectSelector: {{- toYaml . | nindent 10 }} {{- end }} - {{- with .matchConditions }} + {{- if or .matchConditions $.Values.webhooks.matchConditions }} matchConditions: + {{- end }} + {{- with .matchConditions }} + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with $.Values.webhooks.matchConditions }} {{- toYaml . | nindent 10 }} {{- end }} rules: @@ -912,8 +1020,13 @@ spec: objectSelector: {{- toYaml . | nindent 10 }} {{- end }} - {{- with .matchConditions }} + {{- if or .matchConditions $.Values.webhooks.matchConditions }} matchConditions: + {{- end }} + {{- with .matchConditions }} + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with $.Values.webhooks.matchConditions }} {{- toYaml . | nindent 10 }} {{- end }} rules: @@ -950,7 +1063,7 @@ spec: {{- end }} rules: {{- toYaml .rules | nindent 10 }} - sideEffects: None + sideEffects: NoneOnDryRun timeoutSeconds: {{ $.Values.webhooks.validatingWebhooksTimeoutSeconds }} {{- end }} {{- end }} @@ -995,10 +1108,15 @@ spec: objectSelector: {{- toYaml . | nindent 10 }} {{- end }} - {{- with .matchConditions }} matchConditions: - {{- toYaml . | nindent 10 }} + {{- with .matchConditions }} + {{- toYaml . | nindent 10 }} {{- end }} + {{- with $.Values.webhooks.matchConditions }} + {{- toYaml . | nindent 10 }} + {{- end }} + - name: mutations-ignore-subresources + expression: '!has(request.subResource) || request.subResource == ""' rules: {{- toYaml .rules | nindent 10 }} sideEffects: None @@ -1028,8 +1146,13 @@ spec: objectSelector: {{- toYaml . | nindent 10 }} {{- end }} - {{- with .matchConditions }} + {{- if or .matchConditions $.Values.webhooks.matchConditions }} matchConditions: + {{- end }} + {{- with .matchConditions }} + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with $.Values.webhooks.matchConditions }} {{- toYaml . | nindent 10 }} {{- end }} rules: @@ -1071,10 +1194,26 @@ spec: objectSelector: {{- toYaml . | nindent 10 }} {{- end }} - {{- with .matchConditions }} matchConditions: + {{- with .matchConditions }} {{- toYaml . | nindent 10 }} {{- end }} + {{- with $.Values.webhooks.matchConditions }} + {{- toYaml . | nindent 10 }} + {{- end }} + - name: requires-pvc-spec-validation + expression: > + request.operation != "UPDATE" || + !has(object.metadata.deletionTimestamp) || + object.spec != oldObject.spec + - name: requires-static-binding + expression: > + has(object.spec.selector) || + ( + request.operation == "CREATE" && + has(object.spec.volumeName) && + object.spec.volumeName != "" + ) rules: - apiGroups: - "" @@ -1113,8 +1252,13 @@ spec: objectSelector: {{- toYaml . | nindent 10 }} {{- end }} - {{- with .matchConditions }} + {{- if or .matchConditions $.Values.webhooks.matchConditions }} matchConditions: + {{- end }} + {{- with .matchConditions }} + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with $.Values.webhooks.matchConditions }} {{- toYaml . | nindent 10 }} {{- end }} rules: @@ -1153,8 +1297,13 @@ spec: objectSelector: {{- toYaml . | nindent 10 }} {{- end }} - {{- with .matchConditions }} + {{- if or .matchConditions $.Values.webhooks.matchConditions }} matchConditions: + {{- end }} + {{- with .matchConditions }} + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with $.Values.webhooks.matchConditions }} {{- toYaml . | nindent 10 }} {{- end }} rules: @@ -1193,8 +1342,13 @@ spec: objectSelector: {{- toYaml . | nindent 10 }} {{- end }} - {{- with .matchConditions }} + {{- if or .matchConditions $.Values.webhooks.matchConditions }} matchConditions: + {{- end }} + {{- with .matchConditions }} + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with $.Values.webhooks.matchConditions }} {{- toYaml . | nindent 10 }} {{- end }} rules: @@ -1235,8 +1389,13 @@ spec: objectSelector: {{- toYaml . | nindent 10 }} {{- end }} - {{- with .matchConditions }} + {{- if or .matchConditions $.Values.webhooks.matchConditions }} matchConditions: + {{- end }} + {{- with .matchConditions }} + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with $.Values.webhooks.matchConditions }} {{- toYaml . | nindent 10 }} {{- end }} rules: @@ -1277,8 +1436,13 @@ spec: objectSelector: {{- toYaml . | nindent 10 }} {{- end }} - {{- with .matchConditions }} + {{- if or .matchConditions $.Values.webhooks.matchConditions }} matchConditions: + {{- end }} + {{- with .matchConditions }} + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with $.Values.webhooks.matchConditions }} {{- toYaml . | nindent 10 }} {{- end }} rules: @@ -1319,8 +1483,13 @@ spec: objectSelector: {{- toYaml . | nindent 10 }} {{- end }} - {{- with .matchConditions }} + {{- if or .matchConditions $.Values.webhooks.matchConditions }} matchConditions: + {{- end }} + {{- with .matchConditions }} + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with $.Values.webhooks.matchConditions }} {{- toYaml . | nindent 10 }} {{- end }} rules: @@ -1355,15 +1524,20 @@ spec: reinvocationPolicy: {{ .reinvocationPolicy }} {{- with .namespaceSelector }} namespaceSelector: - {{- toYaml . | nindent 4 }} + {{- toYaml . | nindent 10 }} {{- end }} {{- with .objectSelector }} objectSelector: - {{- toYaml . | nindent 4 }} + {{- toYaml . | nindent 10 }} + {{- end }} + {{- if or .matchConditions $.Values.webhooks.matchConditions }} + matchConditions: {{- end }} {{- with .matchConditions }} - matchConditions: - {{- toYaml . | nindent 4 }} + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with $.Values.webhooks.matchConditions }} + {{- toYaml . | nindent 10 }} {{- end }} rules: - apiGroups: @@ -1404,8 +1578,13 @@ spec: objectSelector: {{- toYaml . | nindent 10 }} {{- end }} - {{- with .matchConditions }} + {{- if or .matchConditions $.Values.webhooks.matchConditions }} matchConditions: + {{- end }} + {{- with .matchConditions }} + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with $.Values.webhooks.matchConditions }} {{- toYaml . | nindent 10 }} {{- end }} rules: diff --git a/charts/capsule/values.schema.json b/charts/capsule/values.schema.json index c72502b6..b5ac898c 100644 --- a/charts/capsule/values.schema.json +++ b/charts/capsule/values.schema.json @@ -1147,7 +1147,7 @@ "additionalProperties": true }, "rules": { - "description": "[Rules](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-rules)", + "description": "[Rules](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-rules) Include an explicit status subresource (for example, pods/status) when CustomQuota field selectors depend on status fields.", "type": "array" } } @@ -1552,7 +1552,7 @@ } }, "generic": { - "description": "Generic Rules API", + "description": "Generic Rules API, including the rule-engine validation for Pods and Services", "type": "object", "properties": { "enabled": { @@ -2556,6 +2556,10 @@ "description": "Additional Labels for all webhooks", "type": "object" }, + "matchConditions": { + "description": "MatchConditions for all webhooks", + "type": "array" + }, "mutatingWebhooksTimeoutSeconds": { "description": "Timeout in seconds for mutating webhooks", "type": "integer" diff --git a/charts/capsule/values.yaml b/charts/capsule/values.yaml index 341668b7..ade64bae 100644 --- a/charts/capsule/values.yaml +++ b/charts/capsule/values.yaml @@ -851,6 +851,17 @@ webhooks: # -- (integer, null) Custom service port for the webhook service port: # @schema type:[integer, null] + # -- MatchConditions for all webhooks + matchConditions: [] + # - name: "exclude-privileged-users" + # expression: > + # request.resource.group == "capsule.clastix.io" || + # !( + # request.userInfo.username in [ + # "kubernetes-admin" + # ] + # ) + # Admission Webhook Configuration hooks: customquotas: @@ -936,6 +947,8 @@ webhooks: # -- [MatchConditions](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) matchConditions: [] # -- [Rules](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-rules) + # Include an explicit status subresource (for example, pods/status) when + # CustomQuota field selectors depend on status fields. rules: [] # -- Webhook for Rule Status ([Read More](https://projectcapsule.dev/docs/resource-management/customquotas/#admission)) @@ -1002,7 +1015,7 @@ webhooks: resources: - '*' scope: Namespaced - # -- Generic Rules API + # -- Generic Rules API, including the rule-engine validation for Pods and Services generic: # -- Enable the Hook enabled: true @@ -1025,8 +1038,6 @@ webhooks: operator: Exists # -- [MatchConditions](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) matchConditions: - - name: ignore-subresources - expression: '!has(request.subResource) || request.subResource == ""' - name: ignore-events expression: 'request.resource.resource != "events"' @@ -1042,7 +1053,7 @@ webhooks: - CREATE - UPDATE resources: - - '*' + - '*/*' scope: Namespaced resourcepools: diff --git a/cmd/controller/main.go b/cmd/controller/main.go index 9374d367..adfb972c 100644 --- a/cmd/controller/main.go +++ b/cmd/controller/main.go @@ -32,7 +32,6 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/certwatcher" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/metrics/filters" @@ -629,16 +628,18 @@ func main() { setupLog.Info("initializing caches") - // Initialize Notifiers (Channels) - customQuotaCh := make(chan event.TypedGenericEvent[*capsulev1beta2.CustomQuota], 1024) - globalCustomQuotaCh := make(chan event.TypedGenericEvent[*capsulev1beta2.GlobalCustomQuota], 1024) - // Initialize Caches impersonationCache := cache.NewImpersonationCache() regexCache := cache.NewRegexCache() registryCache := cache.NewRegistryRuleSetCache(regexCache) - customQuotaQuantityCache := cache.NewQuantityCache[string]() jsonPathCache := cache.NewJSONPathCache() + + celCache, err := cache.NewCELCache() + if err != nil { + setupLog.Error(err, "unable to initialize Kubernetes CEL cache") + os.Exit(1) + } + targetsCache := cache.NewCompiledTargetsCache[string]() if directCfg.EnableTLSConfiguration() { @@ -687,12 +688,27 @@ func main() { webhooksList := append( make([]handlers.Webhook, 0), rulesgenericmutation.Register(cfg), - rulesgenericvalidation.Register(regexCache, cfg), + rulesgenericvalidation.Register( + regexCache, + cfg, + rulesgenericvalidation.ForKind( + corev1.SchemeGroupVersion.WithKind("Pod").GroupKind(), + pod.Handler(cfg, + podrules.PodRules(regexCache, registryCache), + ), + "ephemeralcontainers", + ), + rulesgenericvalidation.ForKind( + corev1.SchemeGroupVersion.WithKind("Service").GroupKind(), + service.Handler(cfg, + servicerules.ServiceRules(regexCache), + ), + ), + ), route.GenericReplicasHandler(), route.GenericManagedHandler(cfg), route.Pod( pod.Handler(cfg, - podrules.PodRules(regexCache, registryCache), pod.ImagePullPolicy(), pod.ContainerRegistryLegacy(cfg), pod.PriorityClass(), @@ -707,13 +723,12 @@ func main() { ), ), route.PVCMutating( - pvc.Handler( + pvc.MutatingHandler( pvc.PersistentVolumeMutatingVolume(), ), ), route.Service( service.Handler(cfg, - servicerules.ServiceRules(regexCache), service.Validating(), ), ), @@ -782,15 +797,18 @@ func main() { route.CustomQuotaValidation(customquotavalidation.CustomQuotaValidationHandler( targetsCache, jsonPathCache, + celCache, )), route.GlobalCustomQuotaValidation(customquotavalidation.GlobalCustomQuotaValidationHandler( targetsCache, jsonPathCache, + celCache, )), route.CalculationCustomQuotas( customquotavalidation.ObjectCalculationHandler( targetsCache, jsonPathCache, + celCache, ), ), route.GenericTenantAssignment( @@ -909,6 +927,7 @@ func main() { ImpersonationCache: impersonationCache, RegistryCache: registryCache, JSONPathCache: jsonPathCache, + CELCache: celCache, TargetsCache: targetsCache, RegexCache: regexCache, } @@ -954,11 +973,9 @@ func main() { manager, manager.GetEventRecorder("customquotas-ctrl"), controllerConfig, - customQuotaQuantityCache, jsonPathCache, + celCache, targetsCache, - customQuotaCh, - globalCustomQuotaCh, ); err != nil { setupLog.Error(err, "unable to create controller", "controller", "customquotas") os.Exit(1) diff --git a/e2e/customquota_global_test.go b/e2e/customquota_global_test.go index eb9a63fb..2f33eee7 100644 --- a/e2e/customquota_global_test.go +++ b/e2e/customquota_global_test.go @@ -297,11 +297,22 @@ var _ = Describe("when GlobalCustomQuota uses ledger-backed reconciliation", Ord }) AfterEach(func() { - ForceDeleteNamespace(ctx, testNamespace) - req, err := labels.NewRequirement("e2e.capsule.dev/test-suite", selection.Equals, []string{"globalcustomquota-ledger"}) Expect(err).NotTo(HaveOccurred()) + var customQuotaList capsulev1beta2.CustomQuotaList + Expect(k8sClient.List( + context.TODO(), + &customQuotaList, + client.MatchingLabelsSelector{ + Selector: labels.NewSelector().Add(*req), + }, + )).Should(Succeed()) + + for i := range customQuotaList.Items { + EventuallyDeletion(&customQuotaList.Items[i]) + } + var list capsulev1beta2.GlobalCustomQuotaList Expect(k8sClient.List( context.TODO(), @@ -314,6 +325,8 @@ var _ = Describe("when GlobalCustomQuota uses ledger-backed reconciliation", Ord for i := range list.Items { EventuallyDeletion(&list.Items[i]) } + + ForceDeleteNamespace(ctx, testNamespace) }) It("aggregates a custom pod quantity path and settles the corresponding ledger", Label("skip-on-openshift"), func() { diff --git a/e2e/customquota_namespaced_test.go b/e2e/customquota_namespaced_test.go index f833e5d5..26ab76ca 100644 --- a/e2e/customquota_namespaced_test.go +++ b/e2e/customquota_namespaced_test.go @@ -55,8 +55,8 @@ var _ = Describe("when CustomQuota uses ledger-backed reconciliation", Ordered, }) AfterEach(func() { - ForceDeleteNamespace(ctx, testNamespace) - // delete all global quotas used by tests + // Delete quota coordination resources while the namespace still accepts + // updates. Namespace termination must not race ledger settlement. quotaList := &capsulev1beta2.CustomQuotaList{} if err := k8sClient.List(ctx, quotaList); err == nil { for i := range quotaList.Items { @@ -70,7 +70,6 @@ var _ = Describe("when CustomQuota uses ledger-backed reconciliation", Ordered, } } - // delete all global quotas used by tests gquotaList := &capsulev1beta2.GlobalCustomQuotaList{} if err := k8sClient.List(ctx, gquotaList); err == nil { for i := range gquotaList.Items { @@ -83,6 +82,8 @@ var _ = Describe("when CustomQuota uses ledger-backed reconciliation", Ordered, } } } + + ForceDeleteNamespace(ctx, testNamespace) }) It("marks the quota not ready when an existing matching object has no value at the configured quantity path", func() { @@ -729,6 +730,84 @@ var _ = Describe("when CustomQuota uses ledger-backed reconciliation", Ordered, expectCustomQuotaUsedAndClaims(ctx, testNamespace, q.GetName(), "0", 0) }) + It("calculates usage with CEL while combining JSONPath and CEL selectors", func() { + q := &capsulev1beta2.CustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cq-cel-usage-mixed-selectors", + Namespace: testNamespace, + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "customquota-ledger", + }, + }, + Spec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("500m"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: runtime.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpAdd, + CEL: `object.spec.containers` + + `.map(c, quantity(c.resources.requests["cpu"]))`, + Selectors: []selectors.SelectorWithFields{ + { + FieldSelectors: []string{ + ".spec.restartPolicy=Always", + }, + CELExpressions: []string{ + `object.spec.containers.exists(c, c.image == "nginx:1.27.0")`, + }, + }, + }, + }, + }, + }, + }, + } + + EventuallyCreation(func() error { + return k8sClient.Create(ctx, q) + }).Should(Succeed()) + awaitCustomQuotaReady(ctx, testNamespace, q.GetName()) + + matching := MakePod( + testNamespace, + "cel-usage-matching", + nil, + nil, + "nginx:1.27.0", + "100m", + "", + ) + EventuallyCreation(func() error { + matching.ResourceVersion = "" + return k8sClient.Create(ctx, matching) + }).Should(Succeed()) + + ignored := MakePod( + testNamespace, + "cel-usage-ignored", + nil, + nil, + "nginx:1.26.0", + "200m", + "", + ) + EventuallyCreation(func() error { + ignored.ResourceVersion = "" + return k8sClient.Create(ctx, ignored) + }).Should(Succeed()) + + expectCustomQuotaUsedAndClaims(ctx, testNamespace, q.GetName(), "100m", 1) + expectLedgerSettled(ctx, testNamespace, q.GetName()) + + UpdatePodImage(ctx, testNamespace, "cel-usage-matching", "nginx:1.26.0") + expectLedgerSettled(ctx, testNamespace, q.GetName()) + expectCustomQuotaUsedAndClaims(ctx, testNamespace, q.GetName(), "0", 0) + }) + It("aggregates multiple sources across pod emptyDir size and pvc storage size", func() { q := &capsulev1beta2.CustomQuota{ ObjectMeta: metav1.ObjectMeta{ diff --git a/e2e/gateway_class_test.go b/e2e/gateway_class_test.go index 8245e3c2..34f1d4c9 100644 --- a/e2e/gateway_class_test.go +++ b/e2e/gateway_class_test.go @@ -27,6 +27,12 @@ import ( "github.com/projectcapsule/capsule/pkg/utils" ) +func gatewayAdmissionClient(user string) client.Client { + // Keep the tenant-owner username visible to admission while granting the + // authorization needed for Gateway API resources. + return impersonationClient(user, []string{"system:masters"}) +} + var _ = Describe("when Tenant handles Gateway classes", Ordered, Label("tenant", "classes", "gatewayclass"), func() { authorized := &gatewayv1.GatewayClass{ ObjectMeta: metav1.ObjectMeta{ @@ -240,6 +246,8 @@ var _ = Describe("when Tenant handles Gateway classes", Ordered, Label("tenant", NamespaceCreation(ns, tntNoRestrictions.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) NamespaceIsPartOfTenant(tntNoRestrictions, ns).Should(Succeed()) + admissionClient := gatewayAdmissionClient(tntNoRestrictions.Spec.Owners[0].UserSpec.Name) + By("providing any storageclass", func() { for _, class := range []*gatewayv1.GatewayClass{authorized, unauthorized, exact, exactU} { c := class.GetName() @@ -261,7 +269,7 @@ var _ = Describe("when Tenant handles Gateway classes", Ordered, Label("tenant", }, } - err = k8sClient.Create(context.TODO(), g) + err = admissionClient.Create(context.TODO(), g) return }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) } @@ -285,7 +293,7 @@ var _ = Describe("when Tenant handles Gateway classes", Ordered, Label("tenant", GatewayClassName: gatewayv1.ObjectName("very-unauthorized-and-nonexistent-class"), }, } - err = k8sClient.Create(context.TODO(), g) + err = admissionClient.Create(context.TODO(), g) return }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) }) @@ -342,6 +350,8 @@ var _ = Describe("when Tenant handles Gateway classes", Ordered, Label("tenant", NamespaceCreation(ns, tntWithDefault.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) NamespaceIsPartOfTenant(tntWithDefault, ns).Should(Succeed()) + admissionClient := gatewayAdmissionClient(tntWithDefault.Spec.Owners[0].UserSpec.Name) + By("providing unauthorized gatewayClassName", func() { Eventually(func() (err error) { g := &gatewayv1.Gateway{ @@ -360,7 +370,7 @@ var _ = Describe("when Tenant handles Gateway classes", Ordered, Label("tenant", GatewayClassName: gatewayv1.ObjectName("unauthorized-class"), }, } - err = k8sClient.Create(context.TODO(), g) + err = admissionClient.Create(context.TODO(), g) return }, defaultTimeoutInterval, defaultPollInterval).ShouldNot(Succeed()) }) @@ -383,7 +393,7 @@ var _ = Describe("when Tenant handles Gateway classes", Ordered, Label("tenant", GatewayClassName: gatewayv1.ObjectName("very-unauthorized-and-nonexistent-class"), }, } - err = k8sClient.Create(context.TODO(), g) + err = admissionClient.Create(context.TODO(), g) return }, defaultTimeoutInterval, defaultPollInterval).ShouldNot(Succeed()) }) @@ -460,6 +470,8 @@ var _ = Describe("when Tenant handles Gateway classes", Ordered, Label("tenant", NamespaceCreation(ns, tntWithDefault.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) NamespaceIsPartOfTenant(tntWithDefault, ns).Should(Succeed()) + admissionClient := gatewayAdmissionClient(tntWithDefault.Spec.Owners[0].UserSpec.Name) + By("providing authorized class", func() { Eventually(func() (err error) { g := &gatewayv1.Gateway{ @@ -478,7 +490,7 @@ var _ = Describe("when Tenant handles Gateway classes", Ordered, Label("tenant", GatewayClassName: gatewayv1.ObjectName("customer-class"), }, } - err = k8sClient.Create(context.TODO(), g) + err = admissionClient.Create(context.TODO(), g) return }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) }) @@ -501,7 +513,7 @@ var _ = Describe("when Tenant handles Gateway classes", Ordered, Label("tenant", GatewayClassName: gatewayv1.ObjectName("legacy-2"), }, } - err = k8sClient.Create(context.TODO(), g) + err = admissionClient.Create(context.TODO(), g) return }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) }) @@ -522,7 +534,7 @@ var _ = Describe("when Tenant handles Gateway classes", Ordered, Label("tenant", }, }, } - Expect(k8sClient.Create(context.TODO(), g)).Should(Succeed()) + Expect(admissionClient.Create(context.TODO(), g)).Should(Succeed()) gw := &gatewayv1.Gateway{} Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: g.GetName(), Namespace: g.Namespace}, gw)).Should(Succeed()) Expect(gw.Spec.GatewayClassName).Should(Equal(gatewayv1.ObjectName("customer-class"))) @@ -561,6 +573,8 @@ var _ = Describe("when Tenant handles Gateway classes", Ordered, Label("tenant", NamespaceCreation(ns, tntWithoutDefault.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) NamespaceIsPartOfTenant(tntWithoutDefault, ns).Should(Succeed()) + admissionClient := gatewayAdmissionClient(tntWithoutDefault.Spec.Owners[0].UserSpec.Name) + By("providing empty GatewayClassName", func() { Eventually(func() (err error) { g := &gatewayv1.Gateway{ @@ -578,7 +592,7 @@ var _ = Describe("when Tenant handles Gateway classes", Ordered, Label("tenant", }, }, } - err = k8sClient.Create(context.TODO(), g) + err = admissionClient.Create(context.TODO(), g) return }, defaultTimeoutInterval, defaultPollInterval).ShouldNot(Succeed()) }) diff --git a/e2e/rules_enforce_metadata_test.go b/e2e/rules_enforce_metadata_test.go index ff875880..8fcfd357 100644 --- a/e2e/rules_enforce_metadata_test.go +++ b/e2e/rules_enforce_metadata_test.go @@ -616,8 +616,15 @@ var _ = Describe("enforcing generic metadata namespace rules", Ordered, Label("t } createDeploymentAndExpectAllowed := func(nsName string, deploy *appsv1.Deployment) { + owner := impersonationClient( + tnt.Spec.Owners[0].UserSpec.Name, + withDefaultGroups(nil), + ) + + deploy.Namespace = nsName + EventuallyCreation(func() error { - return k8sClient.Create(context.Background(), deploy, &client.CreateOptions{}) + return owner.Create(context.Background(), deploy, &client.CreateOptions{}) }).Should(Succeed()) EventuallyDeletion(deploy) @@ -630,6 +637,11 @@ var _ = Describe("enforcing generic metadata namespace rules", Ordered, Label("t baseName = "deployment" } + owner := impersonationClient( + tnt.Spec.Owners[0].UserSpec.Name, + withDefaultGroups(nil), + ) + Eventually(func() error { candidate := base.DeepCopy() candidate.Name = fmt.Sprintf("%s-%d", baseName, time.Now().UnixNano()%1e6) @@ -637,9 +649,9 @@ var _ = Describe("enforcing generic metadata namespace rules", Ordered, Label("t candidate.Spec.Selector.MatchLabels["app"] = candidate.Name candidate.Spec.Template.Labels["app"] = candidate.Name - err := k8sClient.Create(context.Background(), candidate, &client.CreateOptions{}) + err := owner.Create(context.Background(), candidate, &client.CreateOptions{}) if err == nil { - _ = k8sClient.Delete(context.Background(), candidate) + _ = owner.Delete(context.Background(), candidate) return fmt.Errorf("expected deployment create to be denied, but it succeeded") } diff --git a/e2e/suite_test.go b/e2e/suite_test.go index 099783b2..aae410bd 100644 --- a/e2e/suite_test.go +++ b/e2e/suite_test.go @@ -82,31 +82,6 @@ var _ = SynchronizedAfterSuite( // Keep this empty, or put per-worker cleanup here. }, func() { - Eventually(func() error { - var tnts capsulev1beta2.TenantList - - if err := k8sClient.List( - context.TODO(), - &tnts, - client.MatchingLabels{"env": "e2e"}, - ); err != nil { - return err - } - - if len(tnts.Items) == 0 { - return nil - } - - for i := range tnts.Items { - ns := &tnts.Items[i] - if err := k8sClient.Delete(context.TODO(), ns); err != nil && !apierrors.IsNotFound(err) { - return err - } - } - - return fmt.Errorf("still have %d tenants with env=e2e", len(tnts.Items)) - }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) - Eventually(func() error { var nsList corev1.NamespaceList @@ -132,6 +107,31 @@ var _ = SynchronizedAfterSuite( return fmt.Errorf("still have %d namespaces with env=e2e", len(nsList.Items)) }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + Eventually(func() error { + var tnts capsulev1beta2.TenantList + + if err := k8sClient.List( + context.TODO(), + &tnts, + client.MatchingLabels{"env": "e2e"}, + ); err != nil { + return err + } + + if len(tnts.Items) == 0 { + return nil + } + + for i := range tnts.Items { + ns := &tnts.Items[i] + if err := k8sClient.Delete(context.TODO(), ns); err != nil && !apierrors.IsNotFound(err) { + return err + } + } + + return fmt.Errorf("still have %d tenants with env=e2e", len(tnts.Items)) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + By("tearing down the test environment") Expect(testEnv.Stop()).ToNot(HaveOccurred()) diff --git a/e2e/utils_test.go b/e2e/utils_test.go index 87848611..98934562 100644 --- a/e2e/utils_test.go +++ b/e2e/utils_test.go @@ -139,9 +139,10 @@ func NamespaceDeletionAdmin(ns *corev1.Namespace, timeout time.Duration) AsyncAs } func ForceDeleteNamespace(ctx context.Context, name string) { + cs := clusterAdminClient() + Eventually(func() error { - ns := &corev1.Namespace{} - err := k8sClient.Get(ctx, types.NamespacedName{Name: name}, ns) + ns, err := cs.CoreV1().Namespaces().Get(ctx, name, metav1.GetOptions{}) if apierrors.IsNotFound(err) { return nil } @@ -149,27 +150,30 @@ func ForceDeleteNamespace(ctx context.Context, name string) { return err } - // Trigger deletion if not already happening - if ns.DeletionTimestamp.IsZero() { - if err := k8sClient.Delete(ctx, ns); err != nil && !apierrors.IsNotFound(err) { + if controllerutil.RemoveFinalizer(ns, namespaceTerminationHoldFinalizer) { + if _, err = cs.CoreV1().Namespaces().Update(ctx, ns, metav1.UpdateOptions{}); err != nil { return err } + + return fmt.Errorf("removed E2E termination hold finalizer from namespace %s", name) + } + + // Trigger deletion if not already happening + if ns.DeletionTimestamp.IsZero() { + if err := cs.CoreV1().Namespaces().Delete(ctx, name, metav1.DeleteOptions{}); err != nil && + !apierrors.IsNotFound(err) { + return err + } + return fmt.Errorf("namespace %s deletion triggered", name) } - // Force-remove finalizers (THIS is the key part) - if len(ns.Finalizers) > 0 { - ns.Finalizers = nil - if err := k8sClient.Update(ctx, ns); err != nil { - return err - } - return fmt.Errorf("namespace %s finalizers removed", name) - } - - // wait until fully gone + // Let the namespace controller remove the kubernetes finalizer. Forcing + // finalization can orphan content that becomes visible if a namespace + // with the same name is recreated. return fmt.Errorf("namespace %s still terminating", name) }, defaultTerminationTimeoutInterval, defaultPollInterval).Should(Succeed(), - "failed to force delete namespace %s", name) + "failed to delete namespace %s", name) } func NamespaceCreation(ns *corev1.Namespace, owner rbac.UserSpec, timeout time.Duration) AsyncAssertion { diff --git a/hack/distro/capsule/example-setup/tenants.yaml b/hack/distro/capsule/example-setup/tenants.yaml index a876e16d..378d303d 100644 --- a/hack/distro/capsule/example-setup/tenants.yaml +++ b/hack/distro/capsule/example-setup/tenants.yaml @@ -10,7 +10,39 @@ spec: - name: alice kind: User rules: - - permissions: + - quota: + - hard: + limits.cpu: "8" + limits.memory: 16Gi + requests.cpu: "8" + requests.memory: 16Gi + classes: + gateway: + - matchLabels: + team: platform + ingress: + - matchLabels: + team: platform + storage: + - matchLabels: + team: platform + priority: + - matchLabels: + team: platform + runtime: + - matchLabels: + team: platform + cluster: + - matchLabels: + team: platform + namespaceSelector: + matchExpressions: + - key: env + operator: In + values: + - "test" + + permissions: bindings: - clusterRoleName: 'custom:proxy-viewer' subjects: diff --git a/internal/cache/cel.go b/internal/cache/cel.go new file mode 100644 index 00000000..04a88c78 --- /dev/null +++ b/internal/cache/cel.go @@ -0,0 +1,137 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package cache + +import ( + "fmt" + "sync" + + "k8s.io/apiserver/pkg/cel/environment" + + celruntime "github.com/projectcapsule/capsule/pkg/runtime/cel" +) + +type celCacheKey struct { + expression string + resultType celruntime.ResultType + mode environment.Type +} + +type CELCache struct { + mu sync.RWMutex + compiler *celruntime.Compiler + data map[celCacheKey]*celruntime.CompiledExpression +} + +func NewCELCache() (*CELCache, error) { + compiler, err := celruntime.NewCompiler() + if err != nil { + return nil, err + } + + return &CELCache{ + compiler: compiler, + data: make(map[celCacheKey]*celruntime.CompiledExpression), + }, nil +} + +func (c *CELCache) GetOrCompileBoolean( + expression string, + mode environment.Type, +) (*celruntime.CompiledExpression, error) { + return c.getOrCompile(expression, celruntime.ResultTypeBoolean, mode) +} + +func (c *CELCache) GetOrCompileQuantity( + expression string, + mode environment.Type, +) (*celruntime.CompiledExpression, error) { + return c.getOrCompile(expression, celruntime.ResultTypeQuantity, mode) +} + +func (c *CELCache) DeleteMany(expressions ...string) int { + if c == nil { + return 0 + } + + c.mu.Lock() + defer c.mu.Unlock() + + deleted := 0 + + for key := range c.data { + for _, expression := range expressions { + if expression != "" && key.expression == expression { + delete(c.data, key) + + deleted++ + + break + } + } + } + + return deleted +} + +func (c *CELCache) Stats() int { + if c == nil { + return 0 + } + + c.mu.RLock() + defer c.mu.RUnlock() + + return len(c.data) +} + +func (c *CELCache) getOrCompile( + expression string, + resultType celruntime.ResultType, + mode environment.Type, +) (*celruntime.CompiledExpression, error) { + if c == nil || c.compiler == nil { + return nil, fmt.Errorf("CEL cache is nil") + } + + key := celCacheKey{ + expression: expression, + resultType: resultType, + mode: mode, + } + + c.mu.RLock() + compiled, ok := c.data[key] + c.mu.RUnlock() + + if ok { + return compiled, nil + } + + c.mu.Lock() + defer c.mu.Unlock() + + if compiled, ok = c.data[key]; ok { + return compiled, nil + } + + var err error + + switch resultType { + case celruntime.ResultTypeBoolean: + compiled, err = c.compiler.CompileBoolean(expression, mode) + case celruntime.ResultTypeQuantity: + compiled, err = c.compiler.CompileQuantity(expression, mode) + default: + err = fmt.Errorf("unsupported CEL result type %q", resultType) + } + + if err != nil { + return nil, err + } + + c.data[key] = compiled + + return compiled, nil +} diff --git a/internal/cache/compiled_targets.go b/internal/cache/compiled_targets.go index fac38c4e..d110b9a7 100644 --- a/internal/cache/compiled_targets.go +++ b/internal/cache/compiled_targets.go @@ -7,6 +7,7 @@ import ( "sync" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + celruntime "github.com/projectcapsule/capsule/pkg/runtime/cel" "github.com/projectcapsule/capsule/pkg/runtime/jsonpath" "github.com/projectcapsule/capsule/pkg/runtime/selectors" ) @@ -15,6 +16,7 @@ type CompiledTarget struct { capsulev1beta2.CustomQuotaStatusTarget CompiledPath *jsonpath.CompiledJSONPath + CompiledCEL *celruntime.CompiledExpression CompiledSelectors []selectors.CompiledSelectorWithFields CompiledConditions []*jsonpath.CompiledJSONPath } diff --git a/internal/controllers/cfg/invalidator/manager.go b/internal/controllers/cfg/invalidator/manager.go index 4a01a389..5ca6ad50 100644 --- a/internal/controllers/cfg/invalidator/manager.go +++ b/internal/controllers/cfg/invalidator/manager.go @@ -44,6 +44,7 @@ type CacheInvalidator struct { RegistryCache *cache.RegistryRuleSetCache TargetsCache *cache.CompiledTargetsCache[string] JSONPathCache *cache.JSONPathCache + CELCache *cache.CELCache ImpersonationCache *cache.ImpersonationCache RegexCache *cache.RegexCache } diff --git a/internal/controllers/cfg/invalidator/targets.go b/internal/controllers/cfg/invalidator/targets.go index 69dc3dd7..458921b5 100644 --- a/internal/controllers/cfg/invalidator/targets.go +++ b/internal/controllers/cfg/invalidator/targets.go @@ -46,7 +46,7 @@ func (r *CacheInvalidator) rebuildTargetsCache(ctx context.Context, log logr.Log } for key, targets := range targetsByKey { - compiled, err := customquotas.CompileTargets(r.JSONPathCache, targets) + compiled, err := customquotas.CompileTargets(r.JSONPathCache, r.CELCache, targets) if err != nil { return fmt.Errorf("compile targets for cache key %q: %w", key, err) } diff --git a/internal/controllers/customquotas/calculation.go b/internal/controllers/customquotas/calculation.go index fe655d90..8766e3d9 100644 --- a/internal/controllers/customquotas/calculation.go +++ b/internal/controllers/customquotas/calculation.go @@ -31,6 +31,7 @@ type quotaUsageReconcileInput struct { Mapper k8smeta.RESTMapper JSONPathCache *cache.JSONPathCache + CELCache *cache.CELCache Sources []capsulev1beta2.CustomQuotaSpecSource ScopeSelectors []metav1.LabelSelector @@ -95,7 +96,7 @@ func reconcileQuotaUsage( }) } - targets, err := CompileTargets(in.JSONPathCache, out.Targets) + targets, err := CompileTargets(in.JSONPathCache, in.CELCache, out.Targets) if err != nil { return out, err } @@ -118,7 +119,14 @@ func reconcileQuotaUsage( items, ok := itemsByGVK[gvk] if !ok { - items, err = getResourcesByGVK(ctx, gvk, in.Client, in.ScopeSelectors, in.Namespaces...) + items, err = getResourcesByGVK( + ctx, + gvk, + in.Client, + target.Scope == k8smeta.RESTScopeNameNamespace, + in.ScopeSelectors, + in.Namespaces..., + ) if err != nil { errs = append(errs, fmt.Errorf("list resources for %s: %w", gvk.String(), err)) @@ -136,7 +144,7 @@ func reconcileQuotaUsage( ) for _, item := range items { - matches, err := MatchesCompiledSelectorsWithFields(item, target.CompiledSelectors) + matches, err := MatchesCompiledSelectorsWithFields(ctx, item, target.CompiledSelectors) if err != nil { errs = append(errs, fmt.Errorf( "evaluate selectors for %s/%s (%s): %w", @@ -153,7 +161,7 @@ func reconcileQuotaUsage( continue } - rawUsage, err := usageForTarget(item, target) + rawUsage, err := usageForTarget(ctx, item, target) if err != nil { errs = append(errs, err) @@ -256,6 +264,7 @@ func reconcileQuotaUsage( } func usageForTarget( + ctx context.Context, item unstructured.Unstructured, target cache.CompiledTarget, ) (resource.Quantity, error) { @@ -264,14 +273,28 @@ func usageForTarget( return *resource.NewQuantity(1, resource.DecimalSI), nil case quota.OpAdd, quota.OpSub: - usage, err := quota.ParseQuantityFromUnstructured(item, target.CompiledPath) + var ( + usage resource.Quantity + err error + ) + + switch { + case target.CompiledCEL != nil: + usage, err = target.CompiledCEL.EvaluateQuantity(ctx, item) + case target.CompiledPath != nil: + usage, err = quota.ParseQuantityFromUnstructured(item, target.CompiledPath) + default: + err = fmt.Errorf("compiled usage expression is missing") + } + if err != nil { return resource.Quantity{}, fmt.Errorf( - "get usage from %s/%s (%s) path %q op %q: %w", + "get usage from %s/%s (%s) path %q cel %q op %q: %w", item.GetNamespace(), item.GetName(), item.GetObjectKind().GroupVersionKind().String(), target.Path, + target.CEL, target.Operation, err, ) diff --git a/internal/controllers/customquotas/custom_quota_controller.go b/internal/controllers/customquotas/custom_quota_controller.go index 3512471d..8ff8c546 100644 --- a/internal/controllers/customquotas/custom_quota_controller.go +++ b/internal/controllers/customquotas/custom_quota_controller.go @@ -46,6 +46,7 @@ type customQuotaClaimController struct { mapper k8smeta.RESTMapper jsonPathCache *cache.JSONPathCache + celCache *cache.CELCache targetsCache *cache.CompiledTargetsCache[string] } @@ -98,7 +99,8 @@ func (r *customQuotaClaimController) Reconcile(ctx context.Context, request ctrl return reconcile.Result{}, err } - if err := r.ensureQuotaLedger(ctx, instance); err != nil { + ledger, err := r.ensureQuotaLedger(ctx, instance) + if err != nil { if instance.DeletionTimestamp != nil || shouldIgnoreLedgerEnsureError(err) { log.V(4).Info("skipping QuantityLedger ensure because CustomQuota or namespace is terminating", "customQuota", request.String(), @@ -111,7 +113,19 @@ func (r *customQuotaClaimController) Reconcile(ctx context.Context, request ctrl return reconcile.Result{}, err } + if hasWork, delay := quantityLedgerWorkDelay(time.Now(), ledger); hasWork && delay > 0 { + log.V(5).Info("debouncing QuantityLedger work", + "customQuota", request.String(), + "after", delay.String(), + ) + + return ctrl.Result{RequeueAfter: delay}, nil + } + reconcileErr := r.reconcile(ctx, log, instance) + if reconcileErr == nil { + meta.RemoveReconcileTriggerAnnotation(instance) + } requeueAfter, ledgerErr := r.reconcileLedger(ctx, log, instance) @@ -135,6 +149,10 @@ func (r *customQuotaClaimController) Reconcile(ctx context.Context, request ctrl return reconcile.Result{}, fmt.Errorf("cannot patch: %w", err) } + if ledgerErr != nil { + return ctrl.Result{}, fmt.Errorf("reconcile QuantityLedger: %w", ledgerErr) + } + if requeueAfter != nil { log.V(5).Info("ledger still has pending work, requeueing", "customQuota", instance.Name, @@ -160,6 +178,7 @@ func (r *customQuotaClaimController) reconcile( Mapper: r.mapper, JSONPathCache: r.jsonPathCache, + CELCache: r.celCache, Sources: instance.Spec.Sources, ScopeSelectors: instance.Spec.ScopeSelectors, @@ -192,6 +211,7 @@ func (r *customQuotaClaimController) reconcileLedger( return reconcileQuantityLedgerAllocation( ctx, r.Client, + r.reader, log, key, instance.Status.Usage.Used.DeepCopy(), @@ -202,7 +222,7 @@ func (r *customQuotaClaimController) reconcileLedger( func (r *customQuotaClaimController) ensureQuotaLedger( ctx context.Context, instance *capsulev1beta2.CustomQuota, -) error { +) (*capsulev1beta2.QuantityLedger, error) { ledger := &capsulev1beta2.QuantityLedger{ ObjectMeta: metav1.ObjectMeta{ Name: instance.GetName(), @@ -228,11 +248,11 @@ func (r *customQuotaClaimController) ensureQuotaLedger( return controllerutil.SetControllerReference(instance, ledger, r.Scheme()) }) if err != nil { - return fmt.Errorf("create or update QuantityLedger %s/%s for CustomQuota %s/%s: %w", + return nil, fmt.Errorf("create or update QuantityLedger %s/%s for CustomQuota %s/%s: %w", ledger.Namespace, ledger.Name, instance.Namespace, instance.Name, err) } - return nil + return ledger, nil } func (r *customQuotaClaimController) emitMetrics( diff --git a/internal/controllers/customquotas/global_custom_quota_controller.go b/internal/controllers/customquotas/global_custom_quota_controller.go index e7a2ba4a..c7a4e7cf 100644 --- a/internal/controllers/customquotas/global_custom_quota_controller.go +++ b/internal/controllers/customquotas/global_custom_quota_controller.go @@ -16,6 +16,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" k8smeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/events" "k8s.io/client-go/util/retry" @@ -50,6 +51,7 @@ type clusterCustomQuotaClaimController struct { mapper k8smeta.RESTMapper jsonPathCache *cache.JSONPathCache + celCache *cache.CELCache targetsCache *cache.CompiledTargetsCache[string] } @@ -101,7 +103,8 @@ func (r *clusterCustomQuotaClaimController) Reconcile(ctx context.Context, reque return reconcile.Result{}, err } - if err := r.ensureQuotaLedger(ctx, instance); err != nil { + ledger, err := r.ensureQuotaLedger(ctx, instance) + if err != nil { if instance.DeletionTimestamp != nil || shouldIgnoreLedgerEnsureError(err) { log.V(4).Info("skipping QuantityLedger ensure because CustomQuota or namespace is terminating", "customQuota", request.String(), @@ -114,7 +117,19 @@ func (r *clusterCustomQuotaClaimController) Reconcile(ctx context.Context, reque return reconcile.Result{}, err } + if hasWork, delay := quantityLedgerWorkDelay(time.Now(), ledger); hasWork && delay > 0 { + log.V(5).Info("debouncing QuantityLedger work", + "customQuota", request.String(), + "after", delay.String(), + ) + + return ctrl.Result{RequeueAfter: delay}, nil + } + reconcileErr := r.reconcile(ctx, log, instance) + if reconcileErr == nil { + meta.RemoveReconcileTriggerAnnotation(instance) + } requeueAfter, ledgerErr := r.reconcileLedger(ctx, log, instance) @@ -138,6 +153,10 @@ func (r *clusterCustomQuotaClaimController) Reconcile(ctx context.Context, reque return reconcile.Result{}, fmt.Errorf("failed to patch: %w", err) } + if ledgerErr != nil { + return ctrl.Result{}, fmt.Errorf("reconcile QuantityLedger: %w", ledgerErr) + } + if requeueAfter != nil { log.V(5).Info("ledger still has pending work, requeueing", "customQuota", instance.Name, @@ -172,7 +191,17 @@ func (r *clusterCustomQuotaClaimController) mapNamespaceToGlobalCustomQuotas( for i := range quotaList.Items { gcq := "aList.Items[i] - if shouldReconcileForNamespaceEvent(gcq, ns.Name) { + shouldReconcile, err := shouldReconcileForNamespaceEvent(gcq, ns) + if err != nil { + r.log.Error(err, "cannot evaluate GlobalCustomQuota namespace selector", + "globalCustomQuota", gcq.Name, + "namespace", ns.Name, + ) + + shouldReconcile = true + } + + if shouldReconcile { requests = append(requests, reconcile.Request{ NamespacedName: types.NamespacedName{ Name: gcq.Name, @@ -186,13 +215,38 @@ func (r *clusterCustomQuotaClaimController) mapNamespaceToGlobalCustomQuotas( func shouldReconcileForNamespaceEvent( instance *capsulev1beta2.GlobalCustomQuota, - namespace string, -) bool { - if len(instance.Spec.NamespaceSelectors) > 0 { - return true + namespace *corev1.Namespace, +) (bool, error) { + if len(instance.Spec.NamespaceSelectors) == 0 { + return false, nil } - return slices.Contains(instance.Status.Namespaces, namespace) + selected := false + + if namespace.DeletionTimestamp == nil { + namespaceLabels := labels.Set(namespace.Labels) + + for _, rawSelector := range instance.Spec.NamespaceSelectors { + if rawSelector.LabelSelector == nil { + continue + } + + selector, err := metav1.LabelSelectorAsSelector(rawSelector.LabelSelector) + if err != nil { + return false, err + } + + if selector.Matches(namespaceLabels) { + selected = true + + break + } + } + } + + wasSelected := slices.Contains(instance.Status.Namespaces, namespace.Name) + + return selected != wasSelected, nil } func (r *clusterCustomQuotaClaimController) reconcile( @@ -226,6 +280,7 @@ func (r *clusterCustomQuotaClaimController) reconcile( Mapper: r.mapper, JSONPathCache: r.jsonPathCache, + CELCache: r.celCache, Sources: instance.Spec.Sources, ScopeSelectors: instance.Spec.ScopeSelectors, @@ -258,6 +313,7 @@ func (r *clusterCustomQuotaClaimController) reconcileLedger( return reconcileQuantityLedgerAllocation( ctx, r.Client, + r.reader, log, key, instance.Status.Usage.Used.DeepCopy(), @@ -268,7 +324,7 @@ func (r *clusterCustomQuotaClaimController) reconcileLedger( func (r *clusterCustomQuotaClaimController) ensureQuotaLedger( ctx context.Context, instance *capsulev1beta2.GlobalCustomQuota, -) error { +) (*capsulev1beta2.QuantityLedger, error) { ledger := &capsulev1beta2.QuantityLedger{ ObjectMeta: metav1.ObjectMeta{ Name: instance.GetName(), @@ -293,11 +349,11 @@ func (r *clusterCustomQuotaClaimController) ensureQuotaLedger( return controllerutil.SetControllerReference(instance, ledger, r.Scheme()) }) if err != nil { - return fmt.Errorf("create or update QuantityLedger %s/%s for GlobalCustomQuota %s: %w", + return nil, fmt.Errorf("create or update QuantityLedger %s/%s for GlobalCustomQuota %s: %w", ledger.Namespace, ledger.Name, instance.GetName(), err) } - return nil + return ledger, nil } func (r *clusterCustomQuotaClaimController) emitMetrics( diff --git a/internal/controllers/customquotas/manager.go b/internal/controllers/customquotas/manager.go index f1c2dd6a..23916636 100644 --- a/internal/controllers/customquotas/manager.go +++ b/internal/controllers/customquotas/manager.go @@ -8,10 +8,8 @@ import ( "github.com/go-logr/logr" "k8s.io/client-go/tools/events" - "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/manager" - capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" "github.com/projectcapsule/capsule/internal/cache" "github.com/projectcapsule/capsule/internal/controllers/utils" "github.com/projectcapsule/capsule/internal/metrics" @@ -22,11 +20,9 @@ func Add( mgr manager.Manager, recorder events.EventRecorder, cfg utils.ControllerOptions, - quantityCache *cache.QuantityCache[string], jsonPathCache *cache.JSONPathCache, + celCache *cache.CELCache, targetsCache *cache.CompiledTargetsCache[string], - namespaceNotifier chan event.TypedGenericEvent[*capsulev1beta2.CustomQuota], - globalNotifier chan event.TypedGenericEvent[*capsulev1beta2.GlobalCustomQuota], ) (err error) { if err = (&customQuotaClaimController{ Client: mgr.GetClient(), @@ -34,6 +30,7 @@ func Add( recorder: recorder, metrics: metrics.MustMakeCustomQuotaRecorder(), jsonPathCache: jsonPathCache, + celCache: celCache, targetsCache: targetsCache, }).SetupWithManager(mgr, cfg); err != nil { return fmt.Errorf("unable to create custom quota controller: %w", err) @@ -45,6 +42,7 @@ func Add( recorder: recorder, metrics: metrics.MustMakeGlobalCustomQuotaRecorder(), jsonPathCache: jsonPathCache, + celCache: celCache, targetsCache: targetsCache, }).SetupWithManager(mgr, cfg); err != nil { return fmt.Errorf("unable to create cluster custom quota controller: %w", err) diff --git a/internal/controllers/customquotas/utils.go b/internal/controllers/customquotas/utils.go index 17f3df52..4d8a4e40 100644 --- a/internal/controllers/customquotas/utils.go +++ b/internal/controllers/customquotas/utils.go @@ -21,18 +21,20 @@ import ( "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" + "k8s.io/apiserver/pkg/cel/environment" "k8s.io/client-go/util/retry" "sigs.k8s.io/controller-runtime/pkg/client" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" "github.com/projectcapsule/capsule/internal/cache" + celruntime "github.com/projectcapsule/capsule/pkg/runtime/cel" "github.com/projectcapsule/capsule/pkg/runtime/jsonpath" "github.com/projectcapsule/capsule/pkg/runtime/quota" "github.com/projectcapsule/capsule/pkg/runtime/selectors" "github.com/projectcapsule/capsule/pkg/utils" ) -const immediatePendingDeleteRequeue = 500 * time.Millisecond +const immediatePendingDeleteRequeue = 2 * time.Second func usagePercentage(used, limit resource.Quantity) float64 { if limit.MilliValue() <= 0 { @@ -51,11 +53,13 @@ type CompiledTarget struct { capsulev1beta2.CustomQuotaStatusTarget CompiledPath *jsonpath.CompiledJSONPath + CompiledCEL *celruntime.CompiledExpression CompiledSelectors []selectors.CompiledSelectorWithFields } func CompileTargets( jcache *cache.JSONPathCache, + ccache *cache.CELCache, targets []capsulev1beta2.CustomQuotaStatusTarget, ) ([]cache.CompiledTarget, error) { out := make([]cache.CompiledTarget, 0, len(targets)) @@ -67,27 +71,51 @@ func CompileTargets( switch target.Operation { case quota.OpCount: - // no usage path needed + // no usage expression needed case quota.OpAdd, quota.OpSub: - compiledPath, err := jcache.GetOrCompile(target.Path) - if err != nil { + switch { + case target.Path != "" && target.CEL == "": + compiledPath, err := jcache.GetOrCompile(target.Path) + if err != nil { + return nil, fmt.Errorf( + "compile usage path %q for %s %q: %w", + target.Path, + target.String(), + target.Operation, + err, + ) + } + + pt.CompiledPath = compiledPath + + case target.CEL != "" && target.Path == "": + compiledCEL, err := ccache.GetOrCompileQuantity(target.CEL, environment.StoredExpressions) + if err != nil { + return nil, fmt.Errorf( + "compile usage CEL expression %q for %s %q: %w", + target.CEL, + target.String(), + target.Operation, + err, + ) + } + + pt.CompiledCEL = compiledCEL + + default: return nil, fmt.Errorf( - "compile usage path %q for %s %q: %w", - target.Path, + "exactly one of path or cel must be set for %s %q", target.String(), target.Operation, - err, ) } - pt.CompiledPath = compiledPath - default: return nil, fmt.Errorf("unsupported operation %q for %s", target.Operation, target.String()) } - compiledSelectors, err := CompileSelectorsWithFields(jcache, target.Selectors) + compiledSelectors, err := CompileSelectorsWithFields(jcache, ccache, target.Selectors) if err != nil { return nil, fmt.Errorf( "compile selectors for %s: %w", @@ -105,6 +133,7 @@ func CompileTargets( } func MatchesCompiledSelectorsWithFields( + ctx context.Context, u unstructured.Unstructured, selectors []selectors.CompiledSelectorWithFields, ) (bool, error) { @@ -134,6 +163,23 @@ func MatchesCompiledSelectorsWithFields( } } + if !allFieldsMatch { + continue + } + + for _, matcher := range sel.CELMatchers { + ok, err := matcher.EvaluateBoolean(ctx, u) + if err != nil { + return false, err + } + + if !ok { + allFieldsMatch = false + + break + } + } + if allFieldsMatch { return true, nil } @@ -180,7 +226,8 @@ func MakeGlobalCustomQuotaCacheKey(name string) string { } func CompileSelectorsWithFields( - cache *cache.JSONPathCache, + jcache *cache.JSONPathCache, + ccache *cache.CELCache, in []selectors.SelectorWithFields, ) ([]selectors.CompiledSelectorWithFields, error) { if len(in) == 0 { @@ -204,7 +251,7 @@ func CompileSelectorsWithFields( fieldMatchers := make([]selectors.CompiledFieldSelector, 0, len(selector.FieldSelectors)) for _, raw := range selector.FieldSelectors { - compiledSelector, err := utils.CompileFieldSelector(cache, raw) + compiledSelector, err := utils.CompileFieldSelector(jcache, raw) if err != nil { return nil, fmt.Errorf("compile field selector %q: %w", raw, err) } @@ -212,9 +259,21 @@ func CompileSelectorsWithFields( fieldMatchers = append(fieldMatchers, compiledSelector) } + celMatchers := make([]*celruntime.CompiledExpression, 0, len(selector.CELExpressions)) + + for _, expression := range selector.CELExpressions { + compiledExpression, err := ccache.GetOrCompileBoolean(expression, environment.StoredExpressions) + if err != nil { + return nil, fmt.Errorf("compile CEL selector %q: %w", expression, err) + } + + celMatchers = append(celMatchers, compiledExpression) + } + out = append(out, selectors.CompiledSelectorWithFields{ LabelSelector: lblSel, FieldMatchers: fieldMatchers, + CELMatchers: celMatchers, }) } @@ -249,95 +308,42 @@ func getResourcesByGVK( ctx context.Context, gvk schema.GroupVersionKind, kubeClient client.Reader, + namespaced bool, scopeSelectors []metav1.LabelSelector, namespaces ...string, ) ([]unstructured.Unstructured, error) { - compiledSelectors := make([]labels.Selector, 0, len(scopeSelectors)) + compiledSelectors, err := compileScopeSelectors(scopeSelectors) + if err != nil { + return nil, err + } - for _, selector := range scopeSelectors { - sel, err := metav1.LabelSelectorAsSelector(&selector) + filterByNamespace, namespaceSet := namespaceFilter(namespaces) + listNamespaces := resourceListNamespaces(namespaced, filterByNamespace, namespaceSet) + + items := make([]unstructured.Unstructured, 0) + seen := make(map[string]struct{}) + + for _, namespace := range listNamespaces { + candidates, err := listResourcesForGVK(ctx, gvk, kubeClient, namespace, compiledSelectors) if err != nil { return nil, err } - compiledSelectors = append(compiledSelectors, sel) - } - - filterByNamespace := true - namespaceSet := make(map[string]struct{}, len(namespaces)) - - for _, ns := range namespaces { - if ns == "*" { - filterByNamespace = false - namespaceSet = nil - - break - } - - namespaceSet[ns] = struct{}{} - } - - list := &unstructured.UnstructuredList{} - list.SetGroupVersionKind(schema.GroupVersionKind{ - Group: gvk.Group, - Version: gvk.Version, - Kind: gvk.Kind + "List", - }) - - if err := kubeClient.List(ctx, list); err != nil { - return nil, err - } - - items := make([]unstructured.Unstructured, 0, len(list.Items)) - seen := make(map[string]struct{}, len(list.Items)) - - for i := range list.Items { - item := list.Items[i] - - // Skip objects that are already definitely deleting: - // deletionTimestamp is set and there are no finalizers left. - if item.GetDeletionTimestamp() != nil && len(item.GetFinalizers()) == 0 { - continue - } - - // Namespace filter - if filterByNamespace { - if _, ok := namespaceSet[item.GetNamespace()]; !ok { + for i := range candidates { + item := candidates[i] + if !resourceMatchesQuotaScope(item, filterByNamespace, namespaceSet, compiledSelectors) { continue } - } - // Label selector filter (OR semantics) - if len(compiledSelectors) > 0 { - itemLabels := labels.Set(item.GetLabels()) - - matched := false - - for _, sel := range compiledSelectors { - if sel.Matches(itemLabels) { - matched = true - - break - } - } - - if !matched { + key := client.ObjectKeyFromObject(&item).String() + if _, exists := seen[key]; exists { continue } + + seen[key] = struct{}{} + + items = append(items, item) } - - key := item.GetNamespace() + "/" + item.GetName() - if item.GetNamespace() == "" { - key = item.GetName() - } - - if _, exists := seen[key]; exists { - continue - } - - seen[key] = struct{}{} - - items = append(items, item) } // Sort by oldest first @@ -348,6 +354,120 @@ func getResourcesByGVK( return items, nil } +func compileScopeSelectors(scopeSelectors []metav1.LabelSelector) ([]labels.Selector, error) { + compiledSelectors := make([]labels.Selector, 0, len(scopeSelectors)) + + for _, selector := range scopeSelectors { + compiled, err := metav1.LabelSelectorAsSelector(&selector) + if err != nil { + return nil, err + } + + compiledSelectors = append(compiledSelectors, compiled) + } + + return compiledSelectors, nil +} + +func namespaceFilter(namespaces []string) (filter bool, namespaceSet map[string]struct{}) { + namespaceSet = make(map[string]struct{}, len(namespaces)) + + for _, namespace := range namespaces { + if namespace == "*" { + return false, nil + } + + namespaceSet[namespace] = struct{}{} + } + + return true, namespaceSet +} + +func resourceListNamespaces( + namespaced bool, + filterByNamespace bool, + namespaceSet map[string]struct{}, +) []string { + if !namespaced || !filterByNamespace { + return []string{""} + } + + orderedNamespaces := make([]string, 0, len(namespaceSet)) + for namespace := range namespaceSet { + orderedNamespaces = append(orderedNamespaces, namespace) + } + + sort.Strings(orderedNamespaces) + + return orderedNamespaces +} + +func listResourcesForGVK( + ctx context.Context, + gvk schema.GroupVersionKind, + kubeClient client.Reader, + namespace string, + compiledSelectors []labels.Selector, +) ([]unstructured.Unstructured, error) { + list := &unstructured.UnstructuredList{} + list.SetGroupVersionKind(schema.GroupVersionKind{ + Group: gvk.Group, + Version: gvk.Version, + Kind: gvk.Kind + "List", + }) + + options := make([]client.ListOption, 0, 2) + if namespace != "" { + options = append(options, client.InNamespace(namespace)) + } + + // A single scope selector can be pushed into the informer/API list. + // Multiple selectors have OR semantics and are filtered below. + if len(compiledSelectors) == 1 { + options = append(options, client.MatchingLabelsSelector{Selector: compiledSelectors[0]}) + } + + if err := kubeClient.List(ctx, list, options...); err != nil { + return nil, err + } + + return list.Items, nil +} + +func resourceMatchesQuotaScope( + item unstructured.Unstructured, + filterByNamespace bool, + namespaceSet map[string]struct{}, + compiledSelectors []labels.Selector, +) bool { + // Skip objects that are already definitely deleting: + // deletionTimestamp is set and there are no finalizers left. + if item.GetDeletionTimestamp() != nil && len(item.GetFinalizers()) == 0 { + return false + } + + // Namespace filtering remains necessary for cluster-wide lists and + // preserves the previous behavior for cluster-scoped targets. + if filterByNamespace { + if _, ok := namespaceSet[item.GetNamespace()]; !ok { + return false + } + } + + if len(compiledSelectors) == 0 { + return true + } + + itemLabels := labels.Set(item.GetLabels()) + for _, selector := range compiledSelectors { + if selector.Matches(itemLabels) { + return true + } + } + + return false +} + func minDurationPtr(cur *time.Duration, cand time.Duration) *time.Duration { if cand < 0 { cand = 0 @@ -365,15 +485,7 @@ func pendingDeleteStillPresent( claims []capsulev1beta2.CustomQuotaClaimItem, ) bool { for _, claim := range claims { - if pd.ObjectRef.UID != "" && claim.UID != "" && pd.ObjectRef.UID == claim.UID { - return true - } - - if pd.ObjectRef.APIGroup == claim.Group && - pd.ObjectRef.APIVersion == claim.Version && - pd.ObjectRef.Kind == claim.Kind && - pd.ObjectRef.Namespace == string(claim.Namespace) && - pd.ObjectRef.Name == claim.Name { + if sameLedgerObject(pd.ObjectRef, claim) { return true } } @@ -381,7 +493,66 @@ func pendingDeleteStillPresent( return false } -const unresolvedReservationRequeue = 250 * time.Millisecond +const ( + unresolvedReservationInitialRequeue = 2 * time.Second + unresolvedReservationRequeue = 5 * time.Second + unresolvedReservationLongRequeue = 15 * time.Second + ledgerWorkDebounce = 500 * time.Millisecond + ledgerWorkMaximumDelay = 2 * time.Second +) + +func quantityLedgerWorkDelay( + now time.Time, + ledger *capsulev1beta2.QuantityLedger, +) (hasWork bool, delay time.Duration) { + var oldest, newest time.Time + + record := func(ts time.Time) { + hasWork = true + + if ts.IsZero() { + return + } + + if oldest.IsZero() || ts.Before(oldest) { + oldest = ts + } + + if newest.IsZero() || ts.After(newest) { + newest = ts + } + } + + for _, reservation := range ledger.Status.Reservations { + ts := reservation.UpdatedAt.Time + if ts.IsZero() { + ts = reservation.CreatedAt.Time + } + + record(ts) + } + + for _, pendingDelete := range ledger.Status.PendingDeletes { + record(pendingDelete.CreatedAt.Time) + } + + if !hasWork || oldest.IsZero() || newest.IsZero() { + return hasWork, 0 + } + + readyAt := newest.Add(ledgerWorkDebounce) + + maximumAt := oldest.Add(ledgerWorkMaximumDelay) + if maximumAt.Before(readyAt) { + readyAt = maximumAt + } + + if !now.Before(readyAt) { + return true, 0 + } + + return true, readyAt.Sub(now) +} func nextReservationMaterializationRequeue( now metav1.Time, @@ -391,21 +562,35 @@ func nextReservationMaterializationRequeue( return unresolvedReservationRequeue } - untilExpiry := time.Until(res.ExpiresAt.Time) + untilExpiry := res.ExpiresAt.Sub(now.Time) if untilExpiry <= 0 { return 0 } - if untilExpiry < unresolvedReservationRequeue { + age := now.Sub(res.UpdatedAt.Time) + + var candidate time.Duration + + switch { + case age < 5*time.Second: + candidate = unresolvedReservationInitialRequeue + case age < 30*time.Second: + candidate = unresolvedReservationRequeue + default: + candidate = unresolvedReservationLongRequeue + } + + if untilExpiry < candidate { return untilExpiry } - return unresolvedReservationRequeue + return candidate } func reconcileQuantityLedgerAllocation( ctx context.Context, c client.Client, + reader client.Reader, log logr.Logger, key types.NamespacedName, observedUsed resource.Quantity, @@ -415,7 +600,10 @@ func reconcileQuantityLedgerAllocation( err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { ledger := &capsulev1beta2.QuantityLedger{} - if err := c.Get(ctx, key, ledger); err != nil { + // Conflicts here are normally caused by admission updating the ledger. + // The informer cache can keep returning the same stale resourceVersion, + // making RetryOnConflict ineffective, so retries must read directly. + if err := reader.Get(ctx, key, ledger); err != nil { if apierrors.IsNotFound(err) { return nil } @@ -426,10 +614,25 @@ func reconcileQuantityLedgerAllocation( now := metav1.Now() pendingDeleteTTL := 30 * time.Second - activeReservations := make([]capsulev1beta2.QuantityLedgerReservation, 0, len(ledger.Status.Reservations)) + pendingDeletePresent := make([]bool, len(ledger.Status.PendingDeletes)) + confirmedTransitions := make(map[string][]capsulev1beta2.QuantityLedgerObjectRef) - for _, res := range ledger.Status.Reservations { - materialized := reservationMaterializedLedger(res, claims) + for i, pendingDelete := range ledger.Status.PendingDeletes { + pendingDeletePresent[i] = pendingDeleteStillPresent(pendingDelete, claims) + if pendingDeletePresent[i] { + continue + } + + key := ledgerReservationObjectKey(pendingDelete.ObjectRef) + confirmedTransitions[key] = append(confirmedTransitions[key], pendingDelete.ObjectRef) + } + + activeReservations := make([]capsulev1beta2.QuantityLedgerReservation, 0, len(ledger.Status.Reservations)) + materializedThrough := materializedReservationPositions(ledger.Status.Reservations, claims) + + for i, res := range ledger.Status.Reservations { + materialized := materializedThrough[ledgerReservationObjectKey(res.ObjectRef)] > i + transitioned := reservationHasConfirmedTransition(res, confirmedTransitions) expired := res.ExpiresAt != nil && res.ExpiresAt.Before(&now) log.V(5).Info("evaluating ledger reservation", @@ -443,6 +646,7 @@ func reconcileQuantityLedgerAllocation( "namespace", res.ObjectRef.Namespace, "name", res.ObjectRef.Name, "materialized", materialized, + "transitioned", transitioned, "expired", expired, ) @@ -450,6 +654,14 @@ func reconcileQuantityLedgerAllocation( case materialized: continue + case transitioned: + // A persisted update/delete moved this object out of the + // matching claims before reconciliation observed its earlier + // state. Its pending-delete hint confirms that the admission + // operation materialized, so older reservations for the same + // object can be released without waiting for their TTL. + continue + case expired: continue @@ -465,8 +677,8 @@ func reconcileQuantityLedgerAllocation( activeDeletes := make([]capsulev1beta2.QuantityLedgerPendingDelete, 0, len(ledger.Status.PendingDeletes)) - for _, pd := range ledger.Status.PendingDeletes { - stillPresent := pendingDeleteStillPresent(pd, claims) + for i, pd := range ledger.Status.PendingDeletes { + stillPresent := pendingDeletePresent[i] expired := now.Sub(pd.CreatedAt.Time) >= pendingDeleteTTL log.V(5).Info("evaluating pending delete", @@ -481,15 +693,22 @@ func reconcileQuantityLedgerAllocation( "expired", expired, ) - if stillPresent { - activeDeletes = append(activeDeletes, pd) - requeueAfter = minDurationPtr(requeueAfter, immediatePendingDeleteRequeue) + // Pending deletes are admission hints, not durable desired state. + // The admitted update/delete may fail after the webhook returns, + // and a transient policy snapshot must not leave a hint that + // requeues this quota forever. Once the hint expires, the current + // observed claims are authoritative. + if !stillPresent || expired { + continue } + + activeDeletes = append(activeDeletes, pd) + requeueAfter = minDurationPtr(requeueAfter, immediatePendingDeleteRequeue) } reserved := resource.MustParse("0") for _, res := range activeReservations { - reserved.Add(res.Usage) + reserved.Add(quantityLedgerReservationDelta(res)) } allocated := observedUsed.DeepCopy() @@ -516,6 +735,58 @@ func reconcileQuantityLedgerAllocation( return requeueAfter, nil } +func reservationHasConfirmedTransition( + reservation capsulev1beta2.QuantityLedgerReservation, + confirmed map[string][]capsulev1beta2.QuantityLedgerObjectRef, +) bool { + for _, ref := range confirmed[ledgerReservationObjectKey(reservation.ObjectRef)] { + if sameLedgerObjectRef(reservation.ObjectRef, ref) { + return true + } + } + + return false +} + +func quantityLedgerReservationDelta( + res capsulev1beta2.QuantityLedgerReservation, +) resource.Quantity { + if res.Delta == nil { + return res.Usage.DeepCopy() + } + + return res.Delta.DeepCopy() +} + +func materializedReservationPositions( + reservations []capsulev1beta2.QuantityLedgerReservation, + claims []capsulev1beta2.CustomQuotaClaimItem, +) map[string]int { + positions := make(map[string]int) + + // Reservations are kept in ledger update order. When a later update for + // the same object is observed, its persisted usage also supersedes every + // earlier reservation for that object. Clearing the prefix avoids holding + // already-materialized deltas until their TTL after rapid updates. + for i, reservation := range reservations { + if reservationMaterializedLedger(reservation, claims) { + positions[ledgerReservationObjectKey(reservation.ObjectRef)] = i + 1 + } + } + + return positions +} + +func ledgerReservationObjectKey(ref capsulev1beta2.QuantityLedgerObjectRef) string { + return strings.Join([]string{ + ref.APIGroup, + ref.APIVersion, + ref.Kind, + ref.Namespace, + ref.Name, + }, "\x00") +} + func reservationMaterializedLedger( res capsulev1beta2.QuantityLedgerReservation, claims []capsulev1beta2.CustomQuotaClaimItem, @@ -542,17 +813,31 @@ func sameLedgerObject( ref capsulev1beta2.QuantityLedgerObjectRef, claim capsulev1beta2.CustomQuotaClaimItem, ) bool { - if ref.APIGroup != claim.Group || - ref.APIVersion != claim.Version || - ref.Kind != claim.Kind || - ref.Namespace != string(claim.Namespace) || - ref.Name != claim.Name { + return sameLedgerObjectRef(ref, capsulev1beta2.QuantityLedgerObjectRef{ + APIGroup: claim.Group, + APIVersion: claim.Version, + Kind: claim.Kind, + Namespace: string(claim.Namespace), + Name: claim.Name, + UID: claim.UID, + }) +} + +func sameLedgerObjectRef( + a capsulev1beta2.QuantityLedgerObjectRef, + b capsulev1beta2.QuantityLedgerObjectRef, +) bool { + if a.APIGroup != b.APIGroup || + a.APIVersion != b.APIVersion || + a.Kind != b.Kind || + a.Namespace != b.Namespace || + a.Name != b.Name { return false } // CREATE admissions often do not have a UID yet. - if ref.UID != "" && claim.UID != "" { - return ref.UID == claim.UID + if a.UID != "" && b.UID != "" { + return a.UID == b.UID } return true diff --git a/internal/controllers/customquotas/utils_test.go b/internal/controllers/customquotas/utils_test.go index 34d63b8b..dcf438a9 100644 --- a/internal/controllers/customquotas/utils_test.go +++ b/internal/controllers/customquotas/utils_test.go @@ -4,11 +4,160 @@ package customquotas import ( + "context" + "errors" "testing" + "time" + "github.com/go-logr/logr" "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/internal/cache" + capsulemeta "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/runtime/quota" + "github.com/projectcapsule/capsule/pkg/runtime/selectors" ) +func TestCompiledTargetsSupportMixedJSONPathAndCELSelectors(t *testing.T) { + t.Parallel() + + celCache, err := cache.NewCELCache() + if err != nil { + t.Fatalf("NewCELCache() error = %v", err) + } + + targets, err := CompileTargets( + cache.NewJSONPathCache(), + celCache, + []capsulev1beta2.CustomQuotaStatusTarget{ + { + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpCount, + Selectors: []selectors.SelectorWithFields{ + { + FieldSelectors: []string{".spec.restartPolicy=Always"}, + CELExpressions: []string{ + `object.spec.containers.exists(c, c.image == "nginx:1.27.0")`, + }, + }, + }, + }, + }, + }, + ) + if err != nil { + t.Fatalf("CompileTargets() error = %v", err) + } + + object := unstructured.Unstructured{Object: map[string]any{ + "spec": map[string]any{ + "restartPolicy": "Always", + "containers": []any{ + map[string]any{"image": "nginx:1.27.0"}, + }, + }, + }} + + matched, err := MatchesCompiledSelectorsWithFields( + context.Background(), + object, + targets[0].CompiledSelectors, + ) + if err != nil { + t.Fatalf("MatchesCompiledSelectorsWithFields() error = %v", err) + } + if !matched { + t.Fatal("mixed JSONPath and CEL selectors did not match") + } + + object.Object["spec"].(map[string]any)["restartPolicy"] = "Never" + matched, err = MatchesCompiledSelectorsWithFields( + context.Background(), + object, + targets[0].CompiledSelectors, + ) + if err != nil { + t.Fatalf("MatchesCompiledSelectorsWithFields() JSONPath mismatch error = %v", err) + } + if matched { + t.Fatal("selector matched when its JSONPath condition was false") + } + + object.Object["spec"].(map[string]any)["restartPolicy"] = "Always" + object.Object["spec"].(map[string]any)["containers"] = []any{ + map[string]any{"image": "nginx:1.26.0"}, + } + matched, err = MatchesCompiledSelectorsWithFields( + context.Background(), + object, + targets[0].CompiledSelectors, + ) + if err != nil { + t.Fatalf("MatchesCompiledSelectorsWithFields() CEL mismatch error = %v", err) + } + if matched { + t.Fatal("selector matched when its CEL condition was false") + } +} + +func TestUsageForTargetSupportsCELQuantityLists(t *testing.T) { + t.Parallel() + + celCache, err := cache.NewCELCache() + if err != nil { + t.Fatalf("NewCELCache() error = %v", err) + } + + targets, err := CompileTargets( + cache.NewJSONPathCache(), + celCache, + []capsulev1beta2.CustomQuotaStatusTarget{ + { + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpAdd, + CEL: `object.spec.containers` + + `.map(c, quantity(c.resources.requests["cpu"]))`, + }, + }, + }, + ) + if err != nil { + t.Fatalf("CompileTargets() error = %v", err) + } + + object := unstructured.Unstructured{Object: map[string]any{ + "spec": map[string]any{ + "containers": []any{ + map[string]any{ + "resources": map[string]any{ + "requests": map[string]any{"cpu": "250m"}, + }, + }, + map[string]any{ + "resources": map[string]any{ + "requests": map[string]any{"cpu": "500m"}, + }, + }, + }, + }, + }} + + usage, err := usageForTarget(context.Background(), object, targets[0]) + if err != nil { + t.Fatalf("usageForTarget() error = %v", err) + } + if usage.Cmp(resource.MustParse("750m")) != 0 { + t.Fatalf("usageForTarget() = %s, want 750m", usage.String()) + } +} + func TestUsagePercentage(t *testing.T) { t.Parallel() @@ -49,3 +198,378 @@ func TestUsagePercentage(t *testing.T) { }) } } + +func TestQuantityLedgerWorkDelay(t *testing.T) { + t.Parallel() + + now := time.Date(2026, time.July, 23, 12, 0, 0, 0, time.UTC) + + t.Run("settled ledger", func(t *testing.T) { + t.Parallel() + + hasWork, delay := quantityLedgerWorkDelay(now, &capsulev1beta2.QuantityLedger{}) + if hasWork || delay != 0 { + t.Fatalf("quantityLedgerWorkDelay() = (%v, %s), want (false, 0)", hasWork, delay) + } + }) + + t.Run("fresh work is debounced", func(t *testing.T) { + t.Parallel() + + updated := metav1.NewTime(now.Add(-100 * time.Millisecond)) + ledger := &capsulev1beta2.QuantityLedger{ + Status: capsulev1beta2.QuantityLedgerStatus{ + Reservations: []capsulev1beta2.QuantityLedgerReservation{ + {ID: "request", UpdatedAt: updated}, + }, + }, + } + + hasWork, delay := quantityLedgerWorkDelay(now, ledger) + if !hasWork { + t.Fatal("quantityLedgerWorkDelay() did not detect work") + } + if delay != 400*time.Millisecond { + t.Fatalf("quantityLedgerWorkDelay() delay = %s, want 400ms", delay) + } + }) + + t.Run("maximum batch delay bounds continuous work", func(t *testing.T) { + t.Parallel() + + old := metav1.NewTime(now.Add(-3 * time.Second)) + fresh := metav1.NewTime(now.Add(-100 * time.Millisecond)) + ledger := &capsulev1beta2.QuantityLedger{ + Status: capsulev1beta2.QuantityLedgerStatus{ + Reservations: []capsulev1beta2.QuantityLedgerReservation{ + {ID: "old", UpdatedAt: old}, + {ID: "fresh", UpdatedAt: fresh}, + }, + }, + } + + hasWork, delay := quantityLedgerWorkDelay(now, ledger) + if !hasWork || delay != 0 { + t.Fatalf("quantityLedgerWorkDelay() = (%v, %s), want ready work", hasWork, delay) + } + }) +} + +func TestQuantityLedgerReservationDelta(t *testing.T) { + t.Parallel() + + usage := resource.MustParse("5") + legacy := quantityLedgerReservationDelta(capsulev1beta2.QuantityLedgerReservation{Usage: usage}) + if legacy.Cmp(usage) != 0 { + t.Fatalf("legacy delta = %s, want %s", legacy.String(), usage.String()) + } + + zero := resource.MustParse("0") + explicit := quantityLedgerReservationDelta(capsulev1beta2.QuantityLedgerReservation{ + Usage: usage, + Delta: &zero, + }) + if !explicit.IsZero() { + t.Fatalf("explicit delta = %s, want 0", explicit.String()) + } +} + +func TestMaterializedReservationPositionsSupersedesOlderUpdates(t *testing.T) { + t.Parallel() + + ref := capsulev1beta2.QuantityLedgerObjectRef{ + APIVersion: "v1", + Kind: "Pod", + Namespace: "tenant-a", + Name: "pod-a", + UID: "pod-uid", + } + reservations := []capsulev1beta2.QuantityLedgerReservation{ + {ID: "older", ObjectRef: ref, Usage: resource.MustParse("8")}, + {ID: "newer", ObjectRef: ref, Usage: resource.MustParse("9")}, + } + claims := []capsulev1beta2.CustomQuotaClaimItem{ + { + GroupVersionKind: metav1.GroupVersionKind{Version: "v1", Kind: "Pod"}, + NamespacedObjectWithUIDReference: capsulemeta.NamespacedObjectWithUIDReference{ + Name: "pod-a", + Namespace: "tenant-a", + UID: types.UID("pod-uid"), + }, + Usage: resource.MustParse("9"), + }, + } + + positions := materializedReservationPositions(reservations, claims) + if got := positions[ledgerReservationObjectKey(ref)]; got != 2 { + t.Fatalf("materialized position = %d, want 2 reservations cleared through the newer update", got) + } +} + +func TestReconcileQuantityLedgerAllocationHandlesFastObjectTransition(t *testing.T) { + t.Parallel() + + scheme := runtime.NewScheme() + if err := capsulev1beta2.AddToScheme(scheme); err != nil { + t.Fatalf("add Capsule scheme: %v", err) + } + + key := types.NamespacedName{Namespace: "capsule-system", Name: "pods"} + ref := capsulev1beta2.QuantityLedgerObjectRef{ + APIVersion: "v1", + Kind: "Pod", + Namespace: "tenant-a", + Name: "pod-a", + UID: "pod-uid", + } + now := metav1.Now() + expiresAt := metav1.NewTime(now.Add(time.Minute)) + delta := resource.MustParse("1") + + t.Run("releases an earlier reservation after a confirmed transition out of claims", func(t *testing.T) { + t.Parallel() + + ledger := &capsulev1beta2.QuantityLedger{ + ObjectMeta: metav1.ObjectMeta{Name: key.Name, Namespace: key.Namespace}, + Status: capsulev1beta2.QuantityLedgerStatus{ + Allocated: resource.MustParse("1"), + Reserved: resource.MustParse("1"), + Reservations: []capsulev1beta2.QuantityLedgerReservation{ + { + ID: "create", + Usage: resource.MustParse("1"), + Delta: &delta, + ObjectRef: ref, + CreatedAt: now, + UpdatedAt: now, + ExpiresAt: &expiresAt, + }, + }, + PendingDeletes: []capsulev1beta2.QuantityLedgerPendingDelete{ + {ID: "status-update", ObjectRef: ref, CreatedAt: now}, + }, + }, + } + kubeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithStatusSubresource(&capsulev1beta2.QuantityLedger{}). + WithObjects(ledger). + Build() + + requeueAfter, err := reconcileQuantityLedgerAllocation( + context.Background(), + kubeClient, + kubeClient, + logr.Discard(), + key, + resource.MustParse("0"), + nil, + ) + if err != nil { + t.Fatalf("reconcileQuantityLedgerAllocation() error = %v", err) + } + if requeueAfter != nil { + t.Fatalf("requeueAfter = %s, want nil", requeueAfter.String()) + } + + got := &capsulev1beta2.QuantityLedger{} + if err := kubeClient.Get(context.Background(), key, got); err != nil { + t.Fatalf("get ledger: %v", err) + } + if len(got.Status.Reservations) != 0 || len(got.Status.PendingDeletes) != 0 { + t.Fatalf( + "ledger work was not settled: reservations=%+v pendingDeletes=%+v", + got.Status.Reservations, + got.Status.PendingDeletes, + ) + } + if !got.Status.Reserved.IsZero() || !got.Status.Allocated.IsZero() { + t.Fatalf( + "ledger quantities were not released: reserved=%s allocated=%s", + got.Status.Reserved.String(), + got.Status.Allocated.String(), + ) + } + }) + + t.Run("keeps reservations when the prior matching claim is still present", func(t *testing.T) { + t.Parallel() + + ledger := &capsulev1beta2.QuantityLedger{ + ObjectMeta: metav1.ObjectMeta{Name: key.Name, Namespace: key.Namespace}, + Status: capsulev1beta2.QuantityLedgerStatus{ + Allocated: resource.MustParse("2"), + Reserved: resource.MustParse("1"), + Reservations: []capsulev1beta2.QuantityLedgerReservation{ + { + ID: "update", + Usage: resource.MustParse("2"), + Delta: &delta, + ObjectRef: ref, + CreatedAt: now, + UpdatedAt: now, + ExpiresAt: &expiresAt, + }, + }, + PendingDeletes: []capsulev1beta2.QuantityLedgerPendingDelete{ + {ID: "delete", ObjectRef: ref, CreatedAt: now}, + }, + }, + } + kubeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithStatusSubresource(&capsulev1beta2.QuantityLedger{}). + WithObjects(ledger). + Build() + claims := []capsulev1beta2.CustomQuotaClaimItem{ + { + GroupVersionKind: metav1.GroupVersionKind{Version: "v1", Kind: "Pod"}, + NamespacedObjectWithUIDReference: capsulemeta.NamespacedObjectWithUIDReference{ + Name: ref.Name, + Namespace: capsulemeta.RFC1123SubdomainName(ref.Namespace), + UID: ref.UID, + }, + Usage: resource.MustParse("1"), + }, + } + + requeueAfter, err := reconcileQuantityLedgerAllocation( + context.Background(), + kubeClient, + kubeClient, + logr.Discard(), + key, + resource.MustParse("1"), + claims, + ) + if err != nil { + t.Fatalf("reconcileQuantityLedgerAllocation() error = %v", err) + } + if requeueAfter == nil { + t.Fatal("requeueAfter = nil, want pending work to be retried") + } + + got := &capsulev1beta2.QuantityLedger{} + if err := kubeClient.Get(context.Background(), key, got); err != nil { + t.Fatalf("get ledger: %v", err) + } + if len(got.Status.Reservations) != 1 || len(got.Status.PendingDeletes) != 1 { + t.Fatalf( + "ledger work was released before the transition persisted: reservations=%+v pendingDeletes=%+v", + got.Status.Reservations, + got.Status.PendingDeletes, + ) + } + }) + + t.Run("drops an expired pending delete when the object still matches", func(t *testing.T) { + t.Parallel() + + expiredCreatedAt := metav1.NewTime(now.Add(-31 * time.Second)) + ledger := &capsulev1beta2.QuantityLedger{ + ObjectMeta: metav1.ObjectMeta{Name: key.Name, Namespace: key.Namespace}, + Status: capsulev1beta2.QuantityLedgerStatus{ + Allocated: resource.MustParse("1"), + PendingDeletes: []capsulev1beta2.QuantityLedgerPendingDelete{ + {ID: "failed-or-false-update", ObjectRef: ref, CreatedAt: expiredCreatedAt}, + }, + }, + } + kubeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithStatusSubresource(&capsulev1beta2.QuantityLedger{}). + WithObjects(ledger). + Build() + claims := []capsulev1beta2.CustomQuotaClaimItem{ + { + GroupVersionKind: metav1.GroupVersionKind{Version: "v1", Kind: "Pod"}, + NamespacedObjectWithUIDReference: capsulemeta.NamespacedObjectWithUIDReference{ + Name: ref.Name, + Namespace: capsulemeta.RFC1123SubdomainName(ref.Namespace), + UID: ref.UID, + }, + Usage: resource.MustParse("1"), + }, + } + + requeueAfter, err := reconcileQuantityLedgerAllocation( + context.Background(), + kubeClient, + kubeClient, + logr.Discard(), + key, + resource.MustParse("1"), + claims, + ) + if err != nil { + t.Fatalf("reconcileQuantityLedgerAllocation() error = %v", err) + } + if requeueAfter != nil { + t.Fatalf("requeueAfter = %s, want nil after expired hint is discarded", requeueAfter.String()) + } + + got := &capsulev1beta2.QuantityLedger{} + if err := kubeClient.Get(context.Background(), key, got); err != nil { + t.Fatalf("get ledger: %v", err) + } + if len(got.Status.PendingDeletes) != 0 { + t.Fatalf("expired pending deletes = %+v, want none", got.Status.PendingDeletes) + } + if got.Status.Allocated.Cmp(resource.MustParse("1")) != 0 { + t.Fatalf("allocated = %s, want observed usage 1", got.Status.Allocated.String()) + } + }) +} + +func TestReconcileQuantityLedgerAllocationUsesDirectReader(t *testing.T) { + t.Parallel() + + scheme := runtime.NewScheme() + if err := capsulev1beta2.AddToScheme(scheme); err != nil { + t.Fatalf("add Capsule scheme: %v", err) + } + + key := types.NamespacedName{Namespace: "tenant-a", Name: "pods"} + ledger := &capsulev1beta2.QuantityLedger{ + ObjectMeta: metav1.ObjectMeta{Name: key.Name, Namespace: key.Namespace}, + } + direct := fake.NewClientBuilder(). + WithScheme(scheme). + WithStatusSubresource(&capsulev1beta2.QuantityLedger{}). + WithObjects(ledger). + Build() + stale := &rejectingGetClient{Client: direct} + + if _, err := reconcileQuantityLedgerAllocation( + context.Background(), + stale, + direct, + logr.Discard(), + key, + resource.MustParse("1"), + nil, + ); err != nil { + t.Fatalf("reconcileQuantityLedgerAllocation() error = %v", err) + } + + got := &capsulev1beta2.QuantityLedger{} + if err := direct.Get(context.Background(), key, got); err != nil { + t.Fatalf("get ledger: %v", err) + } + if got.Status.Allocated.Cmp(resource.MustParse("1")) != 0 { + t.Fatalf("allocated = %s, want 1", got.Status.Allocated.String()) + } +} + +type rejectingGetClient struct { + client.Client +} + +func (c *rejectingGetClient) Get( + context.Context, + client.ObjectKey, + client.Object, + ...client.GetOption, +) error { + return errors.New("cached Get must not be used for ledger conflict retries") +} diff --git a/internal/webhook/customquota/calculation.go b/internal/webhook/customquota/calculation.go index 7e513283..d1675b8f 100644 --- a/internal/webhook/customquota/calculation.go +++ b/internal/webhook/customquota/calculation.go @@ -6,11 +6,14 @@ package customquota import ( "context" "fmt" + "reflect" "slices" "sort" "time" + corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + k8smeta "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -32,7 +35,6 @@ import ( "github.com/projectcapsule/capsule/pkg/runtime/configuration" "github.com/projectcapsule/capsule/pkg/runtime/events" "github.com/projectcapsule/capsule/pkg/runtime/handlers" - index "github.com/projectcapsule/capsule/pkg/runtime/indexers/customquota" "github.com/projectcapsule/capsule/pkg/runtime/quota" "github.com/projectcapsule/capsule/pkg/runtime/selectors" ) @@ -55,15 +57,18 @@ var ledgerMutationBackoff = wait.Backoff{ type objectCalculationHandler struct { targetsCache *cache.CompiledTargetsCache[string] jsonPathCache *cache.JSONPathCache + celCache *cache.CELCache } func ObjectCalculationHandler( targetsCache *cache.CompiledTargetsCache[string], jsonPathCache *cache.JSONPathCache, + celCache *cache.CELCache, ) handlers.Handler { return &objectCalculationHandler{ targetsCache: targetsCache, jsonPathCache: jsonPathCache, + celCache: celCache, } } @@ -74,6 +79,8 @@ func (h *objectCalculationHandler) OnCreate( recorder events.EventRecorder, ) handlers.Func { return func(ctx context.Context, req admission.Request) *admission.Response { + dryRun := req.DryRun != nil && *req.DryRun + log := log.FromContext(ctx).WithValues( "op", "create", "kind", req.Kind.String(), @@ -90,7 +97,7 @@ func (h *objectCalculationHandler) OnCreate( var finalResp *admission.Response err = retry.OnError(customAdmissionBackoff, apierrors.IsConflict, func() error { - matched, err := h.matchAllQuotas(ctx, c, req, u) + matched, err := h.matchAllQuotas(ctx, reader, req, u) if err != nil { finalResp = ad.ErroredResponse(err) @@ -124,7 +131,7 @@ func (h *objectCalculationHandler) OnCreate( for _, item := range evaluated { ledgerKey := quantityLedgerKeyForMatchedQuota(item) - reservation := buildReservation(req, u, item.Usage, item.Key) + reservation := buildReservation(req, u, item.Usage, item.Usage, item.Key) allowed, effectiveUsed, reserved, err := reserveCreateOnLedger( ctx, @@ -132,6 +139,7 @@ func (h *objectCalculationHandler) OnCreate( reader, item, &reservation, + dryRun, ) if err != nil { for _, a := range applied { @@ -178,10 +186,12 @@ func (h *objectCalculationHandler) OnCreate( return nil } - applied = append(applied, appliedReservation{ - LedgerKey: ledgerKey, - ReservationID: reservation.ID, - }) + if !dryRun { + applied = append(applied, appliedReservation{ + LedgerKey: ledgerKey, + ReservationID: reservation.ID, + }) + } } finalResp = nil @@ -204,7 +214,7 @@ func (h *objectCalculationHandler) OnCreate( } } -//nolint:gocognit,cyclop,maintidx +//nolint:gocognit,gocyclo,cyclop,maintidx func (h *objectCalculationHandler) OnUpdate( c client.Client, reader client.Reader, @@ -212,28 +222,90 @@ func (h *objectCalculationHandler) OnUpdate( recorder events.EventRecorder, ) handlers.Func { return func(ctx context.Context, req admission.Request) *admission.Response { + dryRun := req.DryRun != nil && *req.DryRun + statusUpdate := req.SubResource == "status" + logger := log.FromContext(ctx).WithValues( + "op", "update", + "kind", req.Kind.String(), + "namespace", req.Namespace, + "name", req.Name, + "subresource", req.SubResource, + ) + + terminating, namespaceErr := namespaceTerminating(ctx, c, req.Namespace) + if namespaceErr != nil { + logger.Error(namespaceErr, "cannot determine whether namespace is terminating") + } else if terminating { + logger.V(5).Info("allowing update without quota processing because namespace is terminating") + + return nil + } + oldObj, err := getUnstructured(req.OldObject) if err != nil { + if statusUpdate { + logger.Error(err, "allowing status update because the previous object could not be decoded") + + return nil + } + return ad.ErroredResponse(err) } newObj, err := getUnstructured(req.Object) if err != nil { + if statusUpdate { + logger.Error(err, "allowing status update because the new object could not be decoded") + + return nil + } + return ad.ErroredResponse(err) } var finalResp *admission.Response err = retry.OnError(customAdmissionBackoff, apierrors.IsConflict, func() error { - oldMatched, err := h.matchAllQuotas(ctx, c, req, oldObj) + policies, err := loadQuotaPolicySnapshot(ctx, reader, req.Namespace) if err != nil { + if statusUpdate { + logger.Error(err, "allowing status update because quota policies could not be loaded") + + finalResp = nil + + return nil + } + finalResp = ad.ErroredResponse(err) return nil } - newMatched, err := h.matchAllQuotas(ctx, c, req, newObj) + oldMatched, err := h.matchAllQuotasFromSnapshot(ctx, reader, req, oldObj, policies) if err != nil { + if statusUpdate { + logger.Error(err, "allowing status update because previous quota matches could not be evaluated") + + finalResp = nil + + return nil + } + + finalResp = ad.ErroredResponse(err) + + return nil + } + + newMatched, err := h.matchAllQuotasFromSnapshot(ctx, reader, req, newObj, policies) + if err != nil { + if statusUpdate { + logger.Error(err, "allowing status update because new quota matches could not be evaluated") + + finalResp = nil + + return nil + } + finalResp = ad.ErroredResponse(err) return nil @@ -241,6 +313,14 @@ func (h *objectCalculationHandler) OnUpdate( oldEvaluated, err := h.evaluateMatchedQuotas(ctx, oldObj, oldMatched) if err != nil { + if statusUpdate { + logger.Error(err, "allowing status update because previous quota usage could not be calculated") + + finalResp = nil + + return nil + } + finalResp = ad.Denyf( "updating resource %s/%s (%s) cannot be admitted because previous custom quota usage could not be calculated: %v", req.Namespace, @@ -254,6 +334,14 @@ func (h *objectCalculationHandler) OnUpdate( newEvaluated, err := h.evaluateMatchedQuotas(ctx, newObj, newMatched) if err != nil { + if statusUpdate { + logger.Error(err, "allowing status update because new quota usage could not be calculated") + + finalResp = nil + + return nil + } + finalResp = ad.Denyf( "updating resource %s/%s (%s) cannot be admitted because new custom quota usage could not be calculated: %v", req.Namespace, @@ -291,6 +379,7 @@ func (h *objectCalculationHandler) OnUpdate( ReservationID string OldUsage resource.Quantity NewUsage resource.Quantity + PendingDelete *capsulev1beta2.QuantityLedgerPendingDelete } applied := make([]appliedUpdate, 0, len(oldByKey)+len(newByKey)) @@ -322,22 +411,36 @@ func (h *objectCalculationHandler) OnUpdate( ledgerKey := quantityLedgerKeyForMatchedQuota(base) - var pendingDelete *capsulev1beta2.QuantityLedgerObjectRef - if hadOld { - pendingDelete = &capsulev1beta2.QuantityLedgerObjectRef{ - APIGroup: req.Kind.Group, - APIVersion: req.Kind.Version, - Kind: req.Kind.Kind, - Namespace: oldObj.GetNamespace(), - Name: oldObj.GetName(), - UID: oldObj.GetUID(), + var pendingDelete *capsulev1beta2.QuantityLedgerPendingDelete + if hadOld && !hadNew { + pendingDelete = &capsulev1beta2.QuantityLedgerPendingDelete{ + ID: fmt.Sprintf("%s/%s", req.UID, base.Key), + ObjectRef: capsulev1beta2.QuantityLedgerObjectRef{ + APIGroup: req.Kind.Group, + APIVersion: req.Kind.Version, + Kind: req.Kind.Kind, + Namespace: oldObj.GetNamespace(), + Name: oldObj.GetName(), + UID: oldObj.GetUID(), + }, } } var reservation *capsulev1beta2.QuantityLedgerReservation - if hadNew && newUsage.Sign() > 0 { - r := buildReservation(req, newObj, newUsage, base.Key) + if hadNew { + delta := newUsage.DeepCopy() + delta.Sub(oldUsage) + quota.ClampQuantityToZero(&delta) + + // Status is observed state owned by Kubernetes controllers. + // It must notify quota reconciliation, but it must never + // reserve capacity or be rejected for exceeding a quota. + if statusUpdate { + delta = resource.MustParse("0") + } + + r := buildReservation(req, newObj, newUsage, delta, base.Key) reservation = &r } @@ -350,6 +453,8 @@ func (h *objectCalculationHandler) OnUpdate( newUsage, reservation, pendingDelete, + !statusUpdate, + dryRun, ) if err != nil { for _, v := range slices.Backward(applied) { @@ -361,6 +466,7 @@ func (h *objectCalculationHandler) OnUpdate( v.ReservationID, v.OldUsage, v.NewUsage, + v.PendingDelete, ) } @@ -377,9 +483,18 @@ func (h *objectCalculationHandler) OnUpdate( v.ReservationID, v.OldUsage, v.NewUsage, + v.PendingDelete, ) } + if statusUpdate { + logger.Info("allowing status update despite quota limit") + + finalResp = nil + + return nil + } + available := base.Limit.DeepCopy() available.Sub(effectiveUsed) @@ -406,12 +521,15 @@ func (h *objectCalculationHandler) OnUpdate( reservationID = reservation.ID } - applied = append(applied, appliedUpdate{ - LedgerKey: ledgerKey, - ReservationID: reservationID, - OldUsage: oldUsage.DeepCopy(), - NewUsage: newUsage.DeepCopy(), - }) + if !dryRun { + applied = append(applied, appliedUpdate{ + LedgerKey: ledgerKey, + ReservationID: reservationID, + OldUsage: oldUsage.DeepCopy(), + NewUsage: newUsage.DeepCopy(), + PendingDelete: pendingDelete, + }) + } } finalResp = nil @@ -419,6 +537,12 @@ func (h *objectCalculationHandler) OnUpdate( return nil }) if err != nil { + if statusUpdate { + logger.Error(err, "allowing status update because quota reconciliation could not be queued") + + return nil + } + if apierrors.IsConflict(err) { return ad.Denyf( "custom quota admission could not reserve usage due to concurrent quota updates after %d attempts; please retry the request: %v", @@ -441,6 +565,31 @@ func (h *objectCalculationHandler) OnDelete( recorder events.EventRecorder, ) handlers.Func { return func(ctx context.Context, req admission.Request) *admission.Response { + if req.DryRun != nil && *req.DryRun { + return nil + } + + logger := log.FromContext(ctx).WithValues( + "op", "delete", + "kind", req.Kind.String(), + "namespace", req.Namespace, + "name", req.Name, + ) + + // Namespace termination can fan out into many object deletions at once. + // Use the local informer for this best-effort fast path so those deletes + // do not each add an API read before quota processing can be skipped. + // A stale non-terminating result only falls back to the normal, + // conservative ledger path. + terminating, err := namespaceTerminating(ctx, c, req.Namespace) + if err != nil { + logger.Error(err, "cannot determine whether namespace is terminating") + } else if terminating { + logger.V(5).Info("allowing delete without quota processing because namespace is terminating") + + return nil + } + oldObj, err := getUnstructured(req.OldObject) if err != nil { return ad.ErroredResponse(err) @@ -461,13 +610,15 @@ func (h *objectCalculationHandler) OnDelete( } namespacedcq := &capsulev1beta2.CustomQuotaList{} - if err := c.List(ctx, namespacedcq, client.InNamespace(req.Namespace), client.MatchingFields{ - index.ObjectUIDIndexerFieldName: string(uid), - }); err != nil { + if err := reader.List(ctx, namespacedcq, client.InNamespace(req.Namespace)); err != nil { return ad.ErroredResponse(err) } for _, nscq := range namespacedcq.Items { + if !nscq.Status.HasClaimUID(uid) { + continue + } + ledgerKey := types.NamespacedName{ Name: nscq.GetName(), Namespace: nscq.GetNamespace(), @@ -479,13 +630,15 @@ func (h *objectCalculationHandler) OnDelete( } globalcq := &capsulev1beta2.GlobalCustomQuotaList{} - if err := c.List(ctx, globalcq, client.MatchingFields{ - index.ObjectUIDIndexerFieldName: string(uid), - }); err != nil { + if err := reader.List(ctx, globalcq); err != nil { return ad.ErroredResponse(err) } for _, gcq := range globalcq.Items { + if !gcq.Status.HasClaimUID(uid) { + continue + } + ledgerKey := types.NamespacedName{ Name: gcq.GetName(), Namespace: configuration.ControllerNamespace(), @@ -522,7 +675,7 @@ func deleteLedgerReservation( for _, res := range ledger.Status.Reservations { if res.ID == reservationID { - released.Add(res.Usage) + released.Add(reservationDelta(res)) continue } @@ -540,7 +693,7 @@ func deleteLedgerReservation( reserved := resource.MustParse("0") for _, res := range active { - reserved.Add(res.Usage) + reserved.Add(reservationDelta(res)) } ledger.Status.Reservations = active @@ -551,18 +704,67 @@ func deleteLedgerReservation( }) } +type quotaPolicySnapshot struct { + namespaced []capsulev1beta2.CustomQuota + global []capsulev1beta2.GlobalCustomQuota +} + +func loadQuotaPolicySnapshot( + ctx context.Context, + reader client.Reader, + namespace string, +) (quotaPolicySnapshot, error) { + snapshot := quotaPolicySnapshot{} + + // Correctness requires an authoritative policy set. A single snapshot is + // also reused for both sides of UPDATE admission so transient readiness + // changes cannot turn an unchanged object into a false quota transition. + if namespace != "" { + list := &capsulev1beta2.CustomQuotaList{} + if err := reader.List(ctx, list, client.InNamespace(namespace)); err != nil { + return quotaPolicySnapshot{}, err + } + + snapshot.namespaced = list.Items + } + + global := &capsulev1beta2.GlobalCustomQuotaList{} + if err := reader.List(ctx, global); err != nil { + return quotaPolicySnapshot{}, err + } + + snapshot.global = global.Items + + return snapshot, nil +} + func (h *objectCalculationHandler) matchAllQuotas( ctx context.Context, - c client.Client, + reader client.Reader, req admission.Request, u unstructured.Unstructured, ) ([]quota.MatchedQuota, error) { - namespaced, err := h.matchCustomQuotas(ctx, c, req, u) + snapshot, err := loadQuotaPolicySnapshot(ctx, reader, req.Namespace) if err != nil { return nil, err } - global, err := h.matchGlobalCustomQuotas(ctx, c, req, u) + return h.matchAllQuotasFromSnapshot(ctx, reader, req, u, snapshot) +} + +func (h *objectCalculationHandler) matchAllQuotasFromSnapshot( + ctx context.Context, + reader client.Reader, + req admission.Request, + u unstructured.Unstructured, + snapshot quotaPolicySnapshot, +) ([]quota.MatchedQuota, error) { + namespaced, err := h.matchCustomQuotasFromItems(ctx, req, u, snapshot.namespaced) + if err != nil { + return nil, err + } + + global, err := h.matchGlobalCustomQuotasFromItems(ctx, reader, req, u, snapshot.global) if err != nil { return nil, err } @@ -596,36 +798,40 @@ func (h *objectCalculationHandler) matchAllQuotas( return out, nil } -func (h *objectCalculationHandler) matchCustomQuotas( +func (h *objectCalculationHandler) matchCustomQuotasFromItems( ctx context.Context, - c client.Client, req admission.Request, u unstructured.Unstructured, + items []capsulev1beta2.CustomQuota, ) ([]quota.MatchedQuota, error) { - if req.Namespace == "" { - return nil, nil - } - - list := &capsulev1beta2.CustomQuotaList{} - - err := c.List(ctx, list, - client.InNamespace(req.Namespace), - client.MatchingFields{ - index.TargetIndexerFieldName: req.Kind.String(), - }, - ) - if err != nil { - return nil, err - } - - if len(list.Items) == 0 { + if req.Namespace == "" || len(items) == 0 { return nil, nil } objLabels := labels.Set(u.GetLabels()) out := make([]quota.MatchedQuota, 0) - for _, cq := range list.Items { + for _, cq := range items { + if !sourcesTargetKind(cq.Spec.Sources, req.Kind) { + continue + } + + if !customQuotaReadyForAdmission(cq.Generation, cq.Status) { + // Status is observed state and cannot be blocked by quota policy. + // A NotReady quota has no reliable selector/usage model to notify, + // so skip it and allow Kubernetes to persist the status update. + if req.SubResource == "status" { + continue + } + + return nil, fmt.Errorf( + "CustomQuota %s/%s is not ready for generation %d", + cq.Namespace, + cq.Name, + cq.Generation, + ) + } + if !selectors.MatchesSelectors(objLabels, cq.Spec.ScopeSelectors) { continue } @@ -642,7 +848,7 @@ func (h *objectCalculationHandler) matchCustomQuotas( continue } - matches, err := controller.MatchesCompiledSelectorsWithFields(u, target.CompiledSelectors) + matches, err := controller.MatchesCompiledSelectorsWithFields(ctx, u, target.CompiledSelectors) if err != nil { return nil, fmt.Errorf( "evaluate selectors for %s/%s on CustomQuota %s/%s: %w", @@ -664,6 +870,8 @@ func (h *objectCalculationHandler) matchCustomQuotas( Namespace: cq.Namespace, Path: target.Path, CompiledPath: target.CompiledPath, + CEL: target.CEL, + CompiledCEL: target.CompiledCEL, Operation: target.Operation, Limit: cq.Spec.Limit.DeepCopy(), Used: cq.Status.Usage.Used.DeepCopy(), @@ -676,22 +884,14 @@ func (h *objectCalculationHandler) matchCustomQuotas( return out, nil } -func (h *objectCalculationHandler) matchGlobalCustomQuotas( +func (h *objectCalculationHandler) matchGlobalCustomQuotasFromItems( ctx context.Context, - c client.Client, + reader client.Reader, req admission.Request, u unstructured.Unstructured, + items []capsulev1beta2.GlobalCustomQuota, ) ([]quota.MatchedQuota, error) { - list := &capsulev1beta2.GlobalCustomQuotaList{} - - err := c.List(ctx, list, client.MatchingFields{ - index.TargetIndexerFieldName: req.Kind.String(), - }) - if err != nil { - return nil, err - } - - if len(list.Items) == 0 { + if len(items) == 0 { return nil, nil } @@ -699,7 +899,37 @@ func (h *objectCalculationHandler) matchGlobalCustomQuotas( out := make([]quota.MatchedQuota, 0) - for _, gcq := range list.Items { + for _, gcq := range items { + if !sourcesTargetKind(gcq.Spec.Sources, req.Kind) { + continue + } + + if !customQuotaReadyForAdmission(gcq.Generation, gcq.Status.CustomQuotaStatus) { + if req.SubResource == "status" { + continue + } + + applies, err := desiredGlobalQuotaAppliesToNamespace(ctx, reader, &gcq, req.Namespace) + if err != nil { + return nil, fmt.Errorf( + "evaluate namespaces for GlobalCustomQuota %s generation %d: %w", + gcq.Name, + gcq.Generation, + err, + ) + } + + if !applies { + continue + } + + return nil, fmt.Errorf( + "GlobalCustomQuota %s is not ready for generation %d", + gcq.Name, + gcq.Generation, + ) + } + if !gcq.Status.NamespacePresent("*") && !gcq.Status.NamespacePresent(req.Namespace) { continue } @@ -720,7 +950,7 @@ func (h *objectCalculationHandler) matchGlobalCustomQuotas( continue } - matches, err := controller.MatchesCompiledSelectorsWithFields(u, target.CompiledSelectors) + matches, err := controller.MatchesCompiledSelectorsWithFields(ctx, u, target.CompiledSelectors) if err != nil { return nil, fmt.Errorf( "evaluate selectors for %s/%s on GlobalCustomQuota %s: %w", @@ -741,6 +971,8 @@ func (h *objectCalculationHandler) matchGlobalCustomQuotas( Namespace: "", Path: target.Path, CompiledPath: target.CompiledPath, + CEL: target.CEL, + CompiledCEL: target.CompiledCEL, Operation: target.Operation, Limit: gcq.Spec.Limit.DeepCopy(), Used: gcq.Status.Usage.Used.DeepCopy(), @@ -774,6 +1006,27 @@ func getUnstructured(rawExt runtime.RawExtension) (unstructured.Unstructured, er return u, nil } +func namespaceTerminating( + ctx context.Context, + reader client.Reader, + namespace string, +) (bool, error) { + if namespace == "" { + return false, nil + } + + ns := &corev1.Namespace{} + if err := reader.Get(ctx, types.NamespacedName{Name: namespace}, ns); err != nil { + if apierrors.IsNotFound(err) { + return true, nil + } + + return false, err + } + + return ns.DeletionTimestamp != nil, nil +} + func quotaTypeName(global bool) string { if global { return "GlobalCustomQuota" @@ -795,33 +1048,53 @@ func (h *objectCalculationHandler) evaluateMatchedQuotas( ) ([]evaluatedQuota, error) { log := log.FromContext(ctx) - usageByPath := make(map[string]resource.Quantity, len(matched)) + usageByExpression := make(map[string]resource.Quantity, len(matched)) for _, mq := range matched { - // count does not use a path + // count does not use a calculation expression if mq.Operation == quota.OpCount { continue } - if _, ok := usageByPath[mq.Path]; ok { + expressionKey := matchedQuotaExpressionKey(mq) + if _, ok := usageByExpression[expressionKey]; ok { continue } - usage, err := quota.ParseQuantityFromUnstructured(u, mq.CompiledPath) + var ( + usage resource.Quantity + err error + ) + + switch { + case mq.CompiledCEL != nil: + usage, err = mq.CompiledCEL.EvaluateQuantity(ctx, u) + case mq.CompiledPath != nil: + usage, err = quota.ParseQuantityFromUnstructured(u, mq.CompiledPath) + default: + err = fmt.Errorf("compiled usage expression is missing") + } + if err != nil { return nil, fmt.Errorf( - "%s %q source path %q op %q did not resolve to a valid quantity: %w", + "%s %q source path %q cel %q op %q did not resolve to a valid quantity: %w", quotaTypeName(mq.IsGlobal), mq.Name, mq.Path, + mq.CEL, mq.Operation, err, ) } - log.V(5).Info("parsed usage", "path", mq.Path, "parsed", usage.String()) + log.V(5).Info( + "evaluated usage", + "path", mq.Path, + "cel", mq.CEL, + "quantity", usage.String(), + ) - usageByPath[mq.Path] = usage + usageByExpression[expressionKey] = usage } byKey := make(map[string]evaluatedQuota, len(matched)) @@ -846,7 +1119,7 @@ func (h *objectCalculationHandler) evaluateMatchedQuotas( usage = *resource.NewQuantity(1, resource.DecimalSI) case quota.OpSub: - usage = usageByPath[mq.Path].DeepCopy() + usage = usageByExpression[matchedQuotaExpressionKey(mq)].DeepCopy() usage.Neg() ev.Usage.Add(usage) quota.ClampQuantityToZero(&ev.Usage) @@ -855,7 +1128,7 @@ func (h *objectCalculationHandler) evaluateMatchedQuotas( continue case quota.OpAdd: - usage = usageByPath[mq.Path].DeepCopy() + usage = usageByExpression[matchedQuotaExpressionKey(mq)].DeepCopy() default: return nil, fmt.Errorf("unsupported quota operation %q for key %q", mq.Operation, mq.Key) @@ -873,6 +1146,14 @@ func (h *objectCalculationHandler) evaluateMatchedQuotas( return out, nil } +func matchedQuotaExpressionKey(matched quota.MatchedQuota) string { + if matched.CEL != "" { + return "cel:" + matched.CEL + } + + return "path:" + matched.Path +} + func addLedgerPendingDelete( ctx context.Context, c client.Client, @@ -894,6 +1175,14 @@ func addLedgerPendingDelete( } } + if len(ledger.Status.PendingDeletes) >= maxQuantityLedgerPendingDeletes { + return fmt.Errorf( + "quantity ledger %s has reached the maximum of %d pending deletes", + ledgerKey.String(), + maxQuantityLedgerPendingDeletes, + ) + } + ledger.Status.PendingDeletes = append(ledger.Status.PendingDeletes, capsulev1beta2.QuantityLedgerPendingDelete{ ObjectRef: objRef, CreatedAt: now, @@ -907,36 +1196,138 @@ func (h *objectCalculationHandler) getOrCompileCustomQuotaTargets( cq *capsulev1beta2.CustomQuota, ) ([]cache.CompiledTarget, error) { key := controller.MakeCustomQuotaCacheKey(cq.Namespace, cq.Name) + targets := customQuotaTargets(cq.Spec.Sources, cq.Status.Targets) - return h.targetsCache.GetOrBuild(key, func() ([]cache.CompiledTarget, error) { - targets := make([]capsulev1beta2.CustomQuotaStatusTarget, 0, len(cq.Spec.Sources)) - for _, src := range cq.Spec.Sources { - targets = append(targets, capsulev1beta2.CustomQuotaStatusTarget{ - GroupVersionKind: metav1.GroupVersionKind(src.GroupVersionKind()), - CustomQuotaSpecSourceConfig: src.CustomQuotaSpecSourceConfig, - }) - } - - return controller.CompileTargets(h.jsonPathCache, targets) - }) + return h.getOrCompileCurrentTargets(key, targets) } func (h *objectCalculationHandler) getOrCompileGlobalCustomQuotaTargets( gcq *capsulev1beta2.GlobalCustomQuota, ) ([]cache.CompiledTarget, error) { key := controller.MakeGlobalCustomQuotaCacheKey(gcq.Name) + targets := customQuotaTargets(gcq.Spec.Sources, gcq.Status.Targets) - return h.targetsCache.GetOrBuild(key, func() ([]cache.CompiledTarget, error) { - targets := make([]capsulev1beta2.CustomQuotaStatusTarget, 0, len(gcq.Spec.Sources)) - for _, src := range gcq.Spec.Sources { - targets = append(targets, capsulev1beta2.CustomQuotaStatusTarget{ - GroupVersionKind: metav1.GroupVersionKind(src.GroupVersionKind()), - CustomQuotaSpecSourceConfig: src.CustomQuotaSpecSourceConfig, - }) + return h.getOrCompileCurrentTargets(key, targets) +} + +func customQuotaTargets( + sources []capsulev1beta2.CustomQuotaSpecSource, + statusTargets []capsulev1beta2.CustomQuotaStatusTarget, +) []capsulev1beta2.CustomQuotaStatusTarget { + targets := make([]capsulev1beta2.CustomQuotaStatusTarget, 0, len(sources)) + + for i, source := range sources { + scope := k8smeta.RESTScopeName("") + if i < len(statusTargets) { + scope = statusTargets[i].Scope } - return controller.CompileTargets(h.jsonPathCache, targets) - }) + targets = append(targets, capsulev1beta2.CustomQuotaStatusTarget{ + GroupVersionKind: metav1.GroupVersionKind(source.GroupVersionKind()), + CustomQuotaSpecSourceConfig: source.CustomQuotaSpecSourceConfig, + Scope: scope, + }) + } + + return targets +} + +func (h *objectCalculationHandler) getOrCompileCurrentTargets( + key string, + targets []capsulev1beta2.CustomQuotaStatusTarget, +) ([]cache.CompiledTarget, error) { + if compiled, ok := h.targetsCache.Get(key); ok && compiledTargetsCurrent(compiled, targets) { + return compiled, nil + } + + compiled, err := controller.CompileTargets(h.jsonPathCache, h.celCache, targets) + if err != nil { + return nil, err + } + + h.targetsCache.Set(key, compiled) + + return compiled, nil +} + +func compiledTargetsCurrent( + compiled []cache.CompiledTarget, + targets []capsulev1beta2.CustomQuotaStatusTarget, +) bool { + if len(compiled) != len(targets) { + return false + } + + for i := range targets { + if !reflect.DeepEqual(compiled[i].CustomQuotaStatusTarget, targets[i]) { + return false + } + } + + return true +} + +func desiredGlobalQuotaAppliesToNamespace( + ctx context.Context, + reader client.Reader, + quota *capsulev1beta2.GlobalCustomQuota, + namespace string, +) (bool, error) { + if len(quota.Spec.NamespaceSelectors) == 0 { + return true, nil + } + + if namespace == "" { + return false, nil + } + + ns := &corev1.Namespace{} + if err := reader.Get(ctx, types.NamespacedName{Name: namespace}, ns); err != nil { + return false, err + } + + nsLabels := labels.Set(ns.Labels) + + for _, rawSelector := range quota.Spec.NamespaceSelectors { + if rawSelector.LabelSelector == nil { + continue + } + + selector, err := metav1.LabelSelectorAsSelector(rawSelector.LabelSelector) + if err != nil { + return false, err + } + + if selector.Matches(nsLabels) { + return true, nil + } + } + + return false, nil +} + +func sourcesTargetKind( + sources []capsulev1beta2.CustomQuotaSpecSource, + kind metav1.GroupVersionKind, +) bool { + for _, source := range sources { + target := source.GroupVersionKind() + if target.Group == kind.Group && + target.Version == kind.Version && + target.Kind == kind.Kind { + return true + } + } + + return false +} + +func customQuotaReadyForAdmission( + generation int64, + status capsulev1beta2.CustomQuotaStatus, +) bool { + return status.ObservedGeneration == generation && + meta.IsStatusConditionTrue(status.Conditions, meta.ReadyCondition) } func evaluatedByKey(in []evaluatedQuota) map[string]evaluatedQuota { diff --git a/internal/webhook/customquota/cel_validation.go b/internal/webhook/customquota/cel_validation.go new file mode 100644 index 00000000..aa75c2f8 --- /dev/null +++ b/internal/webhook/customquota/cel_validation.go @@ -0,0 +1,42 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package customquota + +import ( + "fmt" + + "k8s.io/apiserver/pkg/cel/environment" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/internal/cache" +) + +func validateCELExpressions( + celCache *cache.CELCache, + sources []capsulev1beta2.CustomQuotaSpecSource, +) error { + for sourceIndex, source := range sources { + if source.CEL != "" { + if _, err := celCache.GetOrCompileQuantity(source.CEL, environment.NewExpressions); err != nil { + return fmt.Errorf("spec.sources[%d].cel: %w", sourceIndex, err) + } + } + + for selectorIndex, selector := range source.Selectors { + for expressionIndex, expression := range selector.CELExpressions { + if _, err := celCache.GetOrCompileBoolean(expression, environment.NewExpressions); err != nil { + return fmt.Errorf( + "spec.sources[%d].selectors[%d].celExpressions[%d]: %w", + sourceIndex, + selectorIndex, + expressionIndex, + err, + ) + } + } + } + } + + return nil +} diff --git a/internal/webhook/customquota/cel_validation_test.go b/internal/webhook/customquota/cel_validation_test.go new file mode 100644 index 00000000..dea42c9c --- /dev/null +++ b/internal/webhook/customquota/cel_validation_test.go @@ -0,0 +1,80 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package customquota + +import ( + "strings" + "testing" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/internal/cache" + "github.com/projectcapsule/capsule/pkg/runtime/selectors" +) + +func TestValidateCELExpressions(t *testing.T) { + t.Parallel() + + celCache, err := cache.NewCELCache() + if err != nil { + t.Fatalf("NewCELCache() error = %v", err) + } + + t.Run("accepts quantity calculation and boolean selectors", func(t *testing.T) { + t.Parallel() + + err := validateCELExpressions(celCache, []capsulev1beta2.CustomQuotaSpecSource{ + { + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + CEL: `quantity(object.spec.resources.requests["cpu"])`, + Selectors: []selectors.SelectorWithFields{ + { + CELExpressions: []string{ + `object.spec.restartPolicy == "Always"`, + }, + }, + }, + }, + }, + }) + if err != nil { + t.Fatalf("validateCELExpressions() error = %v", err) + } + }) + + t.Run("rejects non-quantity calculation output", func(t *testing.T) { + t.Parallel() + + err := validateCELExpressions(celCache, []capsulev1beta2.CustomQuotaSpecSource{ + { + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + CEL: `object.spec.enabled == true`, + }, + }, + }) + if err == nil || !strings.Contains(err.Error(), "kubernetes.Quantity") { + t.Fatalf("validateCELExpressions() error = %v, want quantity output error", err) + } + }) + + t.Run("rejects non-boolean selector output", func(t *testing.T) { + t.Parallel() + + err := validateCELExpressions(celCache, []capsulev1beta2.CustomQuotaSpecSource{ + { + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Selectors: []selectors.SelectorWithFields{ + { + CELExpressions: []string{ + `quantity("1")`, + }, + }, + }, + }, + }, + }) + if err == nil || !strings.Contains(err.Error(), "must evaluate to bool") { + t.Fatalf("validateCELExpressions() error = %v, want boolean output error", err) + } + }) +} diff --git a/internal/webhook/customquota/customquota_validating.go b/internal/webhook/customquota/customquota_validating.go index d8c89c4c..5c7fd3c7 100644 --- a/internal/webhook/customquota/customquota_validating.go +++ b/internal/webhook/customquota/customquota_validating.go @@ -22,15 +22,18 @@ import ( type customQuotaValidationHandler struct { targetsCache *cache.CompiledTargetsCache[string] jsonPathCache *cache.JSONPathCache + celCache *cache.CELCache } func CustomQuotaValidationHandler( targetsCache *cache.CompiledTargetsCache[string], jsonPathCache *cache.JSONPathCache, + celCache *cache.CELCache, ) handlers.Handler { return &customQuotaValidationHandler{ targetsCache: targetsCache, jsonPathCache: jsonPathCache, + celCache: celCache, } } @@ -52,6 +55,10 @@ func (h *customQuotaValidationHandler) OnCreate( return ad.Denyf("invalid spec.limit: %v", err) } + if err := validateCELExpressions(h.celCache, q.Spec.Sources); err != nil { + return ad.Denyf("invalid CEL expression: %v", err) + } + return nil } } @@ -76,6 +83,7 @@ func (h *customQuotaValidationHandler) OnDelete( } h.jsonPathCache.DeleteMany(obj.Spec.CollectJSONPathExpressions()...) + h.celCache.DeleteMany(obj.Spec.CollectCELExpressions()...) return nil } @@ -104,6 +112,10 @@ func (h *customQuotaValidationHandler) OnUpdate( return ad.Denyf("invalid spec.limit: %v", err) } + if err := validateCELExpressions(h.celCache, newQuota.Spec.Sources); err != nil { + return ad.Denyf("invalid CEL expression: %v", err) + } + used := oldQuota.Status.Usage.Used // No recorded usage: allow normal mutation rules below. diff --git a/internal/webhook/customquota/globalcustomquota_validating.go b/internal/webhook/customquota/globalcustomquota_validating.go index b2ca3067..4c9ac853 100644 --- a/internal/webhook/customquota/globalcustomquota_validating.go +++ b/internal/webhook/customquota/globalcustomquota_validating.go @@ -22,15 +22,18 @@ import ( type globalCustomQuotaValidationHandler struct { targetsCache *cache.CompiledTargetsCache[string] jsonPathCache *cache.JSONPathCache + celCache *cache.CELCache } func GlobalCustomQuotaValidationHandler( targetsCache *cache.CompiledTargetsCache[string], jsonPathCache *cache.JSONPathCache, + celCache *cache.CELCache, ) handlers.Handler { return &globalCustomQuotaValidationHandler{ targetsCache: targetsCache, jsonPathCache: jsonPathCache, + celCache: celCache, } } @@ -52,6 +55,10 @@ func (h *globalCustomQuotaValidationHandler) OnCreate( return ad.Denyf("invalid spec.limit: %v", err) } + if err := validateCELExpressions(h.celCache, q.Spec.Sources); err != nil { + return ad.Denyf("invalid CEL expression: %v", err) + } + return nil } } @@ -75,6 +82,7 @@ func (h *globalCustomQuotaValidationHandler) OnDelete( } h.jsonPathCache.DeleteMany(obj.Spec.CollectJSONPathExpressions()...) + h.celCache.DeleteMany(obj.Spec.CollectCELExpressions()...) return nil } @@ -103,6 +111,10 @@ func (h *globalCustomQuotaValidationHandler) OnUpdate( return ad.Denyf("invalid spec.limit: %v", err) } + if err := validateCELExpressions(h.celCache, newQuota.Spec.Sources); err != nil { + return ad.Denyf("invalid CEL expression: %v", err) + } + used := oldQuota.Status.Usage.Used // No recorded usage: allow normal mutation rules below. diff --git a/internal/webhook/customquota/utils.go b/internal/webhook/customquota/utils.go index 839dc585..8262818c 100644 --- a/internal/webhook/customquota/utils.go +++ b/internal/webhook/customquota/utils.go @@ -22,6 +22,11 @@ import ( "github.com/projectcapsule/capsule/pkg/runtime/quota" ) +const ( + maxQuantityLedgerReservations = 1024 + maxQuantityLedgerPendingDeletes = 1024 +) + func quantityLedgerKeyForMatchedQuota(item evaluatedQuota) types.NamespacedName { if item.IsGlobal { return types.NamespacedName{ @@ -42,6 +47,7 @@ func reserveCreateOnLedger( reader client.Reader, item evaluatedQuota, reservation *capsulev1beta2.QuantityLedgerReservation, + dryRun bool, ) (bool, resource.Quantity, resource.Quantity, error) { var ( allowed bool @@ -59,13 +65,6 @@ func reserveCreateOnLedger( now := metav1.Now() - allocated := ledger.Status.Allocated.DeepCopy() - if allocated.IsZero() { - allocated = resource.MustParse("0") - } - - requested := reservation.Usage.DeepCopy() - // Idempotency: if this admission request already has a reservation, // do not increment Allocated a second time. activeReservations := make([]capsulev1beta2.QuantityLedgerReservation, 0, len(ledger.Status.Reservations)+1) @@ -81,6 +80,7 @@ func reserveCreateOnLedger( // Keep Allocated unchanged for retry/idempotent update. existing.Usage = reservation.Usage.DeepCopy() + existing.Delta = copyQuantityPtr(reservation.Delta) existing.ObjectRef = reservation.ObjectRef existing.UpdatedAt = now existing.ExpiresAt = reservation.ExpiresAt @@ -89,26 +89,36 @@ func reserveCreateOnLedger( activeReservations = append(activeReservations, existing) } - nextAllocated := allocated.DeepCopy() if !foundReservation { - nextAllocated.Add(requested) + if len(activeReservations) >= maxQuantityLedgerReservations { + return fmt.Errorf( + "quantity ledger %s has reached the maximum of %d inflight reservations", + ledgerKey.String(), + maxQuantityLedgerReservations, + ) + } + + activeReservations = append(activeReservations, *reservation) } + newReserved := sumReservationDeltas(activeReservations) + nextAllocated := observedLedgerAllocation(ledger) + nextAllocated.Add(newReserved) + if nextAllocated.Cmp(item.Limit) > 0 { allowed = false effectiveUsed = nextAllocated - reserved = allocated + reserved = ledger.Status.Reserved.DeepCopy() return nil } - if !foundReservation { - activeReservations = append(activeReservations, *reservation) - } + if dryRun { + allowed = true + effectiveUsed = nextAllocated + reserved = ledger.Status.Reserved.DeepCopy() - newReserved := resource.MustParse("0") - for _, r := range activeReservations { - newReserved.Add(r.Usage) + return nil } ledger.Status.Reservations = activeReservations @@ -134,10 +144,12 @@ func replaceUsageOnLedger( c client.Client, reader client.Reader, item evaluatedQuota, - oldUsage resource.Quantity, - newUsage resource.Quantity, + _ resource.Quantity, + _ resource.Quantity, reservation *capsulev1beta2.QuantityLedgerReservation, - pendingDelete *capsulev1beta2.QuantityLedgerObjectRef, + pendingDelete *capsulev1beta2.QuantityLedgerPendingDelete, + enforceLimit bool, + dryRun bool, ) (bool, resource.Quantity, resource.Quantity, error) { var ( allowed bool @@ -166,6 +178,7 @@ func replaceUsageOnLedger( if reservation != nil && existing.ID == reservation.ID { foundReservation = true existing.Usage = reservation.Usage.DeepCopy() + existing.Delta = copyQuantityPtr(reservation.Delta) existing.ObjectRef = reservation.ObjectRef existing.UpdatedAt = now existing.ExpiresAt = reservation.ExpiresAt @@ -175,6 +188,14 @@ func replaceUsageOnLedger( } if reservation != nil && !foundReservation { + if len(activeReservations) >= maxQuantityLedgerReservations { + return fmt.Errorf( + "quantity ledger %s has reached the maximum of %d inflight reservations", + ledgerKey.String(), + maxQuantityLedgerReservations, + ) + } + activeReservations = append(activeReservations, *reservation) } @@ -185,7 +206,7 @@ func replaceUsageOnLedger( exists := false for _, pd := range activeDeletes { - if pd.ObjectRef.UID != "" && pd.ObjectRef.UID == pendingDelete.UID { + if sameQuantityLedgerPendingDelete(pd, *pendingDelete) { exists = true break @@ -193,24 +214,29 @@ func replaceUsageOnLedger( } if !exists { - activeDeletes = append(activeDeletes, capsulev1beta2.QuantityLedgerPendingDelete{ - ObjectRef: *pendingDelete, - CreatedAt: now, - }) + if len(activeDeletes) >= maxQuantityLedgerPendingDeletes { + return fmt.Errorf( + "quantity ledger %s has reached the maximum of %d pending deletes", + ledgerKey.String(), + maxQuantityLedgerPendingDeletes, + ) + } + + newPendingDelete := *pendingDelete + newPendingDelete.CreatedAt = now + activeDeletes = append(activeDeletes, newPendingDelete) } } - nextAllocated := ledger.Status.Allocated.DeepCopy() - if nextAllocated.IsZero() { - nextAllocated = resource.MustParse("0") - } + // Never release capacity from admission. The API operation can still + // fail after this webhook returns, so only the positive usage delta is + // reserved here. Reconciliation releases decreases after observing the + // persisted object. + newReserved := sumReservationDeltas(activeReservations) + nextAllocated := observedLedgerAllocation(ledger) + nextAllocated.Add(newReserved) - nextAllocated.Sub(oldUsage) - quota.ClampQuantityToZero(&nextAllocated) - - nextAllocated.Add(newUsage) - - if nextAllocated.Cmp(item.Limit) > 0 { + if enforceLimit && nextAllocated.Cmp(item.Limit) > 0 { allowed = false effectiveUsed = nextAllocated reserved = ledger.Status.Reserved.DeepCopy() @@ -218,9 +244,12 @@ func replaceUsageOnLedger( return nil } - newReserved := resource.MustParse("0") - for _, res := range activeReservations { - newReserved.Add(res.Usage) + if dryRun { + allowed = true + effectiveUsed = nextAllocated + reserved = ledger.Status.Reserved.DeepCopy() + + return nil } ledger.Status.Reservations = activeReservations @@ -248,8 +277,9 @@ func rollbackUsageReplacementOnLedger( reader client.Reader, ledgerKey types.NamespacedName, reservationID string, - oldUsage resource.Quantity, - newUsage resource.Quantity, + _ resource.Quantity, + _ resource.Quantity, + pendingDelete *capsulev1beta2.QuantityLedgerPendingDelete, ) error { return retry.RetryOnConflict(retry.DefaultBackoff, func() error { ledger := &capsulev1beta2.QuantityLedger{} @@ -262,41 +292,81 @@ func rollbackUsageReplacementOnLedger( } activeReservations := make([]capsulev1beta2.QuantityLedgerReservation, 0, len(ledger.Status.Reservations)) + released := resource.MustParse("0") for _, res := range ledger.Status.Reservations { if reservationID != "" && res.ID == reservationID { + released.Add(reservationDelta(res)) + continue } activeReservations = append(activeReservations, res) } + activeDeletes := make([]capsulev1beta2.QuantityLedgerPendingDelete, 0, len(ledger.Status.PendingDeletes)) + removedPendingDelete := false + + for _, pd := range ledger.Status.PendingDeletes { + if pendingDelete != nil && sameQuantityLedgerPendingDelete(pd, *pendingDelete) { + removedPendingDelete = true + + continue + } + + activeDeletes = append(activeDeletes, pd) + } + + if released.Sign() == 0 && !removedPendingDelete { + return nil + } + allocated := ledger.Status.Allocated.DeepCopy() if allocated.IsZero() { allocated = resource.MustParse("0") } - allocated.Sub(newUsage) + allocated.Sub(released) quota.ClampQuantityToZero(&allocated) - allocated.Add(oldUsage) newReserved := resource.MustParse("0") for _, res := range activeReservations { - newReserved.Add(res.Usage) + newReserved.Add(reservationDelta(res)) } ledger.Status.Allocated = allocated ledger.Status.Reservations = activeReservations + ledger.Status.PendingDeletes = activeDeletes ledger.Status.Reserved = newReserved return c.Status().Update(ctx, ledger) }) } +func sameQuantityLedgerPendingDelete( + a capsulev1beta2.QuantityLedgerPendingDelete, + b capsulev1beta2.QuantityLedgerPendingDelete, +) bool { + if a.ID != "" || b.ID != "" { + return a.ID != "" && a.ID == b.ID + } + + if a.ObjectRef.UID != "" && b.ObjectRef.UID != "" { + return a.ObjectRef.UID == b.ObjectRef.UID + } + + return a.ObjectRef.APIGroup == b.ObjectRef.APIGroup && + a.ObjectRef.APIVersion == b.ObjectRef.APIVersion && + a.ObjectRef.Kind == b.ObjectRef.Kind && + a.ObjectRef.Namespace == b.ObjectRef.Namespace && + a.ObjectRef.Name == b.ObjectRef.Name +} + func buildReservation( req admission.Request, u unstructured.Unstructured, usage resource.Quantity, + delta resource.Quantity, quotaKey string, ) capsulev1beta2.QuantityLedgerReservation { now := metav1.Now() @@ -313,12 +383,55 @@ func buildReservation( UID: u.GetUID(), }, Usage: usage.DeepCopy(), + Delta: quantityPtr(delta), CreatedAt: now, UpdatedAt: now, ExpiresAt: &expiresAt, } } +func quantityPtr(in resource.Quantity) *resource.Quantity { + out := in.DeepCopy() + + return &out +} + +func copyQuantityPtr(in *resource.Quantity) *resource.Quantity { + if in == nil { + return nil + } + + return quantityPtr(*in) +} + +func reservationDelta(res capsulev1beta2.QuantityLedgerReservation) resource.Quantity { + if res.Delta == nil { + return res.Usage.DeepCopy() + } + + return res.Delta.DeepCopy() +} + +func observedLedgerAllocation(ledger *capsulev1beta2.QuantityLedger) resource.Quantity { + observed := ledger.Status.Allocated.DeepCopy() + observed.Sub(ledger.Status.Reserved) + quota.ClampQuantityToZero(&observed) + + return observed +} + +func sumReservationDeltas( + reservations []capsulev1beta2.QuantityLedgerReservation, +) resource.Quantity { + total := resource.MustParse("0") + + for _, reservation := range reservations { + total.Add(reservationDelta(reservation)) + } + + return total +} + func allKeys[K comparable, V any](a map[K]V, b map[K]V) []K { out := make([]K, 0, len(a)+len(b)) seen := make(map[K]struct{}, len(a)+len(b)) diff --git a/internal/webhook/customquota/utils_test.go b/internal/webhook/customquota/utils_test.go new file mode 100644 index 00000000..d21b6815 --- /dev/null +++ b/internal/webhook/customquota/utils_test.go @@ -0,0 +1,927 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package customquota + +import ( + "context" + "strings" + "testing" + "time" + + admissionv1 "k8s.io/api/admission/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/internal/cache" + "github.com/projectcapsule/capsule/pkg/api/meta" + caprunt "github.com/projectcapsule/capsule/pkg/api/runtime" + "github.com/projectcapsule/capsule/pkg/runtime/quota" + "github.com/projectcapsule/capsule/pkg/runtime/selectors" +) + +func TestReserveCreateOnLedgerDryRunDoesNotMutate(t *testing.T) { + t.Parallel() + + ctx := context.Background() + key := types.NamespacedName{Namespace: "tenant-a", Name: "pods"} + ledger := ledgerForTest(key, "2") + cl := ledgerClientForTest(t, ledger) + + usage := resource.MustParse("1") + reservation := capsulev1beta2.QuantityLedgerReservation{ + ID: "dry-run", + Usage: usage.DeepCopy(), + Delta: quantityPtr(usage), + } + + allowed, effective, _, err := reserveCreateOnLedger( + ctx, + cl, + cl, + evaluatedQuota{MatchedQuota: quota.MatchedQuota{ + Name: key.Name, + Namespace: key.Namespace, + Limit: resource.MustParse("3"), + }}, + &reservation, + true, + ) + if err != nil { + t.Fatalf("reserveCreateOnLedger() error = %v", err) + } + if !allowed { + t.Fatal("reserveCreateOnLedger() dry-run unexpectedly denied") + } + if effective.Cmp(resource.MustParse("3")) != 0 { + t.Fatalf("effective allocation = %s, want 3", effective.String()) + } + + got := &capsulev1beta2.QuantityLedger{} + if err := cl.Get(ctx, key, got); err != nil { + t.Fatalf("get ledger: %v", err) + } + if got.Status.Allocated.Cmp(resource.MustParse("2")) != 0 { + t.Fatalf("persisted allocation = %s, want 2", got.Status.Allocated.String()) + } + if len(got.Status.Reservations) != 0 { + t.Fatalf("persisted reservations = %d, want 0", len(got.Status.Reservations)) + } +} + +func TestReserveCreateOnLedgerReleasesExpiredReservation(t *testing.T) { + t.Parallel() + + ctx := context.Background() + key := types.NamespacedName{Namespace: "tenant-a", Name: "pods"} + ledger := ledgerForTest(key, "2") + expiredAt := metav1.NewTime(time.Now().Add(-time.Minute)) + expiredUsage := resource.MustParse("1") + ledger.Status.Reserved = expiredUsage.DeepCopy() + ledger.Status.Reservations = []capsulev1beta2.QuantityLedgerReservation{ + { + ID: "expired", + Usage: expiredUsage.DeepCopy(), + Delta: quantityPtr(expiredUsage), + ExpiresAt: &expiredAt, + }, + } + cl := ledgerClientForTest(t, ledger) + + usage := resource.MustParse("1") + reservation := capsulev1beta2.QuantityLedgerReservation{ + ID: "current", + Usage: usage.DeepCopy(), + Delta: quantityPtr(usage), + } + + allowed, effective, _, err := reserveCreateOnLedger( + ctx, + cl, + cl, + evaluatedQuota{MatchedQuota: quota.MatchedQuota{ + Name: key.Name, + Namespace: key.Namespace, + Limit: resource.MustParse("2"), + }}, + &reservation, + false, + ) + if err != nil { + t.Fatalf("reserveCreateOnLedger() error = %v", err) + } + if !allowed { + t.Fatal("reserveCreateOnLedger() denied after an expired reservation") + } + if effective.Cmp(resource.MustParse("2")) != 0 { + t.Fatalf("effective allocation = %s, want 2", effective.String()) + } + + got := &capsulev1beta2.QuantityLedger{} + if err := cl.Get(ctx, key, got); err != nil { + t.Fatalf("get ledger: %v", err) + } + if len(got.Status.Reservations) != 1 || got.Status.Reservations[0].ID != "current" { + t.Fatalf("active reservations = %#v, want only current", got.Status.Reservations) + } +} + +func TestReplaceUsageOnLedgerDoesNotReleaseDecreaseBeforePersistence(t *testing.T) { + t.Parallel() + + ctx := context.Background() + key := types.NamespacedName{Namespace: "tenant-a", Name: "cpu"} + ledger := ledgerForTest(key, "10") + cl := ledgerClientForTest(t, ledger) + + newUsage := resource.MustParse("1") + zero := resource.MustParse("0") + reservation := capsulev1beta2.QuantityLedgerReservation{ + ID: "decrease", + Usage: newUsage.DeepCopy(), + Delta: quantityPtr(zero), + } + + allowed, _, _, err := replaceUsageOnLedger( + ctx, + cl, + cl, + evaluatedQuota{MatchedQuota: quota.MatchedQuota{ + Name: key.Name, + Namespace: key.Namespace, + Limit: resource.MustParse("10"), + }}, + resource.MustParse("10"), + newUsage, + &reservation, + nil, + true, + false, + ) + if err != nil { + t.Fatalf("replaceUsageOnLedger() error = %v", err) + } + if !allowed { + t.Fatal("replaceUsageOnLedger() unexpectedly denied a decrease") + } + + got := &capsulev1beta2.QuantityLedger{} + if err := cl.Get(ctx, key, got); err != nil { + t.Fatalf("get ledger: %v", err) + } + if got.Status.Allocated.Cmp(resource.MustParse("10")) != 0 { + t.Fatalf("allocation was released before persistence: got %s, want 10", got.Status.Allocated.String()) + } + if len(got.Status.Reservations) != 1 { + t.Fatalf("reservations = %d, want 1", len(got.Status.Reservations)) + } + if delta := reservationDelta(got.Status.Reservations[0]); !delta.IsZero() { + t.Fatalf("decrease reservation delta = %s, want 0", delta.String()) + } +} + +func TestReplaceUsageOnLedgerReservesOnlyPositiveDelta(t *testing.T) { + t.Parallel() + + ctx := context.Background() + key := types.NamespacedName{Namespace: "tenant-a", Name: "cpu"} + ledger := ledgerForTest(key, "5") + cl := ledgerClientForTest(t, ledger) + + newUsage := resource.MustParse("8") + delta := resource.MustParse("3") + reservation := capsulev1beta2.QuantityLedgerReservation{ + ID: "increase", + Usage: newUsage.DeepCopy(), + Delta: quantityPtr(delta), + } + + allowed, _, _, err := replaceUsageOnLedger( + ctx, + cl, + cl, + evaluatedQuota{MatchedQuota: quota.MatchedQuota{ + Name: key.Name, + Namespace: key.Namespace, + Limit: resource.MustParse("8"), + }}, + resource.MustParse("5"), + newUsage, + &reservation, + nil, + true, + false, + ) + if err != nil { + t.Fatalf("replaceUsageOnLedger() error = %v", err) + } + if !allowed { + t.Fatal("replaceUsageOnLedger() unexpectedly denied") + } + + got := &capsulev1beta2.QuantityLedger{} + if err := cl.Get(ctx, key, got); err != nil { + t.Fatalf("get ledger: %v", err) + } + if got.Status.Allocated.Cmp(resource.MustParse("8")) != 0 { + t.Fatalf("allocation = %s, want 8", got.Status.Allocated.String()) + } + if got.Status.Reserved.Cmp(delta) != 0 { + t.Fatalf("reserved = %s, want 3", got.Status.Reserved.String()) + } +} + +func TestRollbackUsageReplacementRemovesPendingDelete(t *testing.T) { + t.Parallel() + + ctx := context.Background() + key := types.NamespacedName{Namespace: "tenant-a", Name: "cpu"} + ref := capsulev1beta2.QuantityLedgerObjectRef{ + APIVersion: "v1", + Kind: "Pod", + Namespace: "tenant-a", + Name: "pod-a", + UID: "pod-uid", + } + ledger := ledgerForTest(key, "10") + ledger.Status.PendingDeletes = []capsulev1beta2.QuantityLedgerPendingDelete{ + {ID: "request-a", ObjectRef: ref, CreatedAt: metav1.Now()}, + {ID: "request-b", ObjectRef: ref, CreatedAt: metav1.Now()}, + } + cl := ledgerClientForTest(t, ledger) + pendingDelete := &capsulev1beta2.QuantityLedgerPendingDelete{ + ID: "request-a", + ObjectRef: ref, + } + + if err := rollbackUsageReplacementOnLedger( + ctx, + cl, + cl, + key, + "", + resource.MustParse("10"), + resource.MustParse("1"), + pendingDelete, + ); err != nil { + t.Fatalf("rollbackUsageReplacementOnLedger() error = %v", err) + } + + got := &capsulev1beta2.QuantityLedger{} + if err := cl.Get(ctx, key, got); err != nil { + t.Fatalf("get ledger: %v", err) + } + if len(got.Status.PendingDeletes) != 1 { + t.Fatalf("pending deletes = %d, want 1", len(got.Status.PendingDeletes)) + } + if got.Status.PendingDeletes[0].ID != "request-b" { + t.Fatalf("remaining pending delete = %q, want request-b", got.Status.PendingDeletes[0].ID) + } + if got.Status.Allocated.Cmp(resource.MustParse("10")) != 0 { + t.Fatalf("allocation = %s, want 10", got.Status.Allocated.String()) + } +} + +func TestReservationDeltaBackwardsCompatibility(t *testing.T) { + t.Parallel() + + usage := resource.MustParse("4") + got := reservationDelta(capsulev1beta2.QuantityLedgerReservation{Usage: usage}) + if got.Cmp(usage) != 0 { + t.Fatalf("legacy reservation delta = %s, want %s", got.String(), usage.String()) + } +} + +func TestStatusSubresourceUpdateQueuesQuotaReconciliation(t *testing.T) { + t.Parallel() + + ctx := context.Background() + scheme := runtime.NewScheme() + if err := capsulev1beta2.AddToScheme(scheme); err != nil { + t.Fatalf("add Capsule scheme: %v", err) + } + if err := corev1.AddToScheme(scheme); err != nil { + t.Fatalf("add core scheme: %v", err) + } + + key := types.NamespacedName{Namespace: "tenant-a", Name: "active-pod-cpu"} + namespace := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: key.Namespace}, + } + customQuota := &capsulev1beta2.CustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: key.Name, + Namespace: key.Namespace, + Generation: 1, + }, + Spec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("2"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: caprunt.VersionKind{APIVersion: "v1", Kind: "Pod"}, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpAdd, + Path: ".spec.containers[*].resources.limits.cpu", + Selectors: []selectors.SelectorWithFields{ + { + FieldSelectors: []string{ + ".status.phase!=Succeeded", + ".status.phase!=Failed", + ".status.phase!=Unknown", + }, + }, + }, + }, + }, + }, + }, + Status: capsulev1beta2.CustomQuotaStatus{ + ObservedGeneration: 1, + Usage: capsulev1beta2.CustomQuotaStatusUsage{ + Used: resource.MustParse("2"), + Available: resource.MustParse("0"), + }, + Conditions: meta.ConditionList{ + {Type: meta.ReadyCondition, Status: metav1.ConditionTrue}, + }, + }, + } + ledger := ledgerForTest(key, "2") + + cl := fake.NewClientBuilder(). + WithScheme(scheme). + WithStatusSubresource(&capsulev1beta2.QuantityLedger{}). + WithObjects(namespace, customQuota, ledger). + Build() + + oldPod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "completed", + Namespace: key.Namespace, + UID: "pod-uid", + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "main", + Resources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1"), + }, + }, + }, + }, + }, + Status: corev1.PodStatus{Phase: corev1.PodRunning}, + } + newPod := oldPod.DeepCopy() + newPod.Status.Phase = corev1.PodSucceeded + + handler := &objectCalculationHandler{ + targetsCache: cache.NewCompiledTargetsCache[string](), + jsonPathCache: cache.NewJSONPathCache(), + } + req := admission.Request{AdmissionRequest: admissionv1.AdmissionRequest{ + UID: "status-request", + Operation: admissionv1.Update, + Kind: metav1.GroupVersionKind{Version: "v1", Kind: "Pod"}, + Namespace: key.Namespace, + Name: oldPod.Name, + SubResource: "status", + OldObject: runtime.RawExtension{Object: oldPod}, + Object: runtime.RawExtension{Object: newPod}, + }} + + if resp := handler.OnUpdate(cl, cl, nil, nil)(ctx, req); resp != nil { + t.Fatalf("status update response = %#v, want allowed", resp) + } + + got := &capsulev1beta2.QuantityLedger{} + if err := cl.Get(ctx, key, got); err != nil { + t.Fatalf("get ledger: %v", err) + } + if len(got.Status.PendingDeletes) != 1 { + t.Fatalf("pending deletes = %d, want 1", len(got.Status.PendingDeletes)) + } + if got.Status.PendingDeletes[0].ObjectRef.UID != oldPod.UID { + t.Fatalf( + "pending delete UID = %q, want %q", + got.Status.PendingDeletes[0].ObjectRef.UID, + oldPod.UID, + ) + } +} + +func TestStatusSubresourceUpdateSkipsNotReadyQuota(t *testing.T) { + t.Parallel() + + ctx := context.Background() + scheme := runtime.NewScheme() + if err := capsulev1beta2.AddToScheme(scheme); err != nil { + t.Fatalf("add Capsule scheme: %v", err) + } + if err := corev1.AddToScheme(scheme); err != nil { + t.Fatalf("add core scheme: %v", err) + } + + namespace := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "tenant-a"}} + customQuota := &capsulev1beta2.CustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "not-ready", + Namespace: namespace.Name, + Generation: 1, + }, + Spec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("1"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: caprunt.VersionKind{APIVersion: "v1", Kind: "Pod"}, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpCount, + }, + }, + }, + }, + Status: capsulev1beta2.CustomQuotaStatus{ + ObservedGeneration: 1, + Conditions: meta.ConditionList{ + {Type: meta.ReadyCondition, Status: metav1.ConditionFalse}, + }, + }, + } + cl := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(namespace, customQuota). + Build() + + oldPod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "pod-a", Namespace: namespace.Name, UID: "pod-uid"}, + Status: corev1.PodStatus{Phase: corev1.PodPending}, + } + newPod := oldPod.DeepCopy() + newPod.Status.Phase = corev1.PodRunning + + handler := &objectCalculationHandler{ + targetsCache: cache.NewCompiledTargetsCache[string](), + jsonPathCache: cache.NewJSONPathCache(), + } + req := statusUpdateRequest("not-ready-status", oldPod, newPod) + + if resp := handler.OnUpdate(cl, cl, nil, nil)(ctx, req); resp != nil { + t.Fatalf("status update response = %#v, want fail-open allow", resp) + } +} + +func TestStatusSubresourceUpdateUsesOnePolicySnapshot(t *testing.T) { + t.Parallel() + + ctx := context.Background() + scheme := runtime.NewScheme() + if err := capsulev1beta2.AddToScheme(scheme); err != nil { + t.Fatalf("add Capsule scheme: %v", err) + } + if err := corev1.AddToScheme(scheme); err != nil { + t.Fatalf("add core scheme: %v", err) + } + + key := types.NamespacedName{Namespace: "tenant-a", Name: "tracked-pods"} + namespace := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: key.Namespace}} + customQuota := &capsulev1beta2.CustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: key.Name, + Namespace: key.Namespace, + Generation: 1, + }, + Spec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("10"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: caprunt.VersionKind{APIVersion: "v1", Kind: "Pod"}, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpCount, + Selectors: []selectors.SelectorWithFields{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"track": "yes"}, + }, + }, + }, + }, + }, + }, + }, + Status: capsulev1beta2.CustomQuotaStatus{ + ObservedGeneration: 1, + Usage: capsulev1beta2.CustomQuotaStatusUsage{ + Used: resource.MustParse("1"), + Available: resource.MustParse("9"), + }, + Conditions: meta.ConditionList{ + {Type: meta.ReadyCondition, Status: metav1.ConditionTrue}, + }, + }, + } + ledger := ledgerForTest(key, "1") + cl := fake.NewClientBuilder(). + WithScheme(scheme). + WithStatusSubresource(&capsulev1beta2.QuantityLedger{}). + WithObjects(namespace, customQuota, ledger). + Build() + reader := &readinessFlappingReader{Reader: cl} + + oldPod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-a", + Namespace: key.Namespace, + UID: "pod-uid", + Labels: map[string]string{"track": "yes"}, + }, + Status: corev1.PodStatus{Phase: corev1.PodPending}, + } + newPod := oldPod.DeepCopy() + newPod.Status.Phase = corev1.PodRunning + + handler := &objectCalculationHandler{ + targetsCache: cache.NewCompiledTargetsCache[string](), + jsonPathCache: cache.NewJSONPathCache(), + } + req := statusUpdateRequest("unchanged-label-status", oldPod, newPod) + + if resp := handler.OnUpdate(cl, reader, nil, nil)(ctx, req); resp != nil { + t.Fatalf("status update response = %#v, want allowed", resp) + } + if reader.customQuotaListCalls != 1 { + t.Fatalf( + "CustomQuota policy list calls = %d, want one immutable snapshot for old and new objects", + reader.customQuotaListCalls, + ) + } + + got := &capsulev1beta2.QuantityLedger{} + if err := cl.Get(ctx, key, got); err != nil { + t.Fatalf("get ledger: %v", err) + } + if len(got.Status.Reservations) != 0 || len(got.Status.PendingDeletes) != 0 { + t.Fatalf( + "unchanged matching status update queued ledger work: reservations=%+v pendingDeletes=%+v", + got.Status.Reservations, + got.Status.PendingDeletes, + ) + } +} + +func TestStatusSubresourceIncreaseQueuesZeroDeltaWithoutEnforcement(t *testing.T) { + t.Parallel() + + ctx := context.Background() + scheme := runtime.NewScheme() + if err := capsulev1beta2.AddToScheme(scheme); err != nil { + t.Fatalf("add Capsule scheme: %v", err) + } + if err := corev1.AddToScheme(scheme); err != nil { + t.Fatalf("add core scheme: %v", err) + } + + key := types.NamespacedName{Namespace: "tenant-a", Name: "running-pods"} + namespace := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: key.Namespace}} + customQuota := &capsulev1beta2.CustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: key.Name, + Namespace: key.Namespace, + Generation: 1, + }, + Spec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("2"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: caprunt.VersionKind{APIVersion: "v1", Kind: "Pod"}, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpCount, + Selectors: []selectors.SelectorWithFields{ + {FieldSelectors: []string{".status.phase=Running"}}, + }, + }, + }, + }, + }, + Status: capsulev1beta2.CustomQuotaStatus{ + ObservedGeneration: 1, + Usage: capsulev1beta2.CustomQuotaStatusUsage{ + Used: resource.MustParse("2"), + Available: resource.MustParse("0"), + }, + Conditions: meta.ConditionList{ + {Type: meta.ReadyCondition, Status: metav1.ConditionTrue}, + }, + }, + } + ledger := ledgerForTest(key, "2") + cl := fake.NewClientBuilder(). + WithScheme(scheme). + WithStatusSubresource(&capsulev1beta2.QuantityLedger{}). + WithObjects(namespace, customQuota, ledger). + Build() + + oldPod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "pod-a", Namespace: key.Namespace, UID: "pod-uid"}, + Status: corev1.PodStatus{Phase: corev1.PodPending}, + } + newPod := oldPod.DeepCopy() + newPod.Status.Phase = corev1.PodRunning + + handler := &objectCalculationHandler{ + targetsCache: cache.NewCompiledTargetsCache[string](), + jsonPathCache: cache.NewJSONPathCache(), + } + req := statusUpdateRequest("increase-status", oldPod, newPod) + + if resp := handler.OnUpdate(cl, cl, nil, nil)(ctx, req); resp != nil { + t.Fatalf("status update response = %#v, want allowed beyond quota limit", resp) + } + + got := &capsulev1beta2.QuantityLedger{} + if err := cl.Get(ctx, key, got); err != nil { + t.Fatalf("get ledger: %v", err) + } + if len(got.Status.Reservations) != 1 { + t.Fatalf("reservations = %d, want one reconciliation notification", len(got.Status.Reservations)) + } + if delta := reservationDelta(got.Status.Reservations[0]); !delta.IsZero() { + t.Fatalf("status notification delta = %s, want 0", delta.String()) + } + if got.Status.Allocated.Cmp(resource.MustParse("2")) != 0 { + t.Fatalf("allocated = %s, want unchanged value 2", got.Status.Allocated.String()) + } +} + +func TestTerminatingNamespaceBypassesQuotaProcessing(t *testing.T) { + t.Parallel() + + ctx := context.Background() + scheme := runtime.NewScheme() + if err := corev1.AddToScheme(scheme); err != nil { + t.Fatalf("add core scheme: %v", err) + } + + now := metav1.Now() + namespace := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "terminating", + DeletionTimestamp: &now, + Finalizers: []string{"kubernetes"}, + }, + } + cl := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(namespace). + Build() + handler := &objectCalculationHandler{ + targetsCache: cache.NewCompiledTargetsCache[string](), + jsonPathCache: cache.NewJSONPathCache(), + } + req := admission.Request{AdmissionRequest: admissionv1.AdmissionRequest{ + UID: "terminating-status", + Operation: admissionv1.Update, + Kind: metav1.GroupVersionKind{Version: "v1", Kind: "Pod"}, + Namespace: namespace.Name, + Name: "pod-a", + SubResource: "status", + }} + + if resp := handler.OnUpdate(cl, cl, nil, nil)(ctx, req); resp != nil { + t.Fatalf("terminating namespace status response = %#v, want allowed", resp) + } + + req.Operation = admissionv1.Delete + req.SubResource = "" + if resp := handler.OnDelete(cl, cl, nil, nil)(ctx, req); resp != nil { + t.Fatalf("terminating namespace delete response = %#v, want allowed", resp) + } +} + +func TestCustomQuotaReadyForAdmissionRequiresCurrentGeneration(t *testing.T) { + t.Parallel() + + status := capsulev1beta2.CustomQuotaStatus{ + ObservedGeneration: 3, + Conditions: meta.ConditionList{ + { + Type: meta.ReadyCondition, + Status: metav1.ConditionTrue, + }, + }, + } + + if !customQuotaReadyForAdmission(3, status) { + t.Fatal("current ready generation must be active") + } + if customQuotaReadyForAdmission(4, status) { + t.Fatal("stale ready status must not activate a new generation") + } + + status.Conditions[0].Status = metav1.ConditionFalse + if customQuotaReadyForAdmission(3, status) { + t.Fatal("non-ready quota must not be active") + } +} + +func TestMatchAllQuotasFailsClosedUntilCurrentGenerationIsReady(t *testing.T) { + t.Parallel() + + ctx := context.Background() + scheme := runtime.NewScheme() + if err := capsulev1beta2.AddToScheme(scheme); err != nil { + t.Fatalf("add Capsule scheme: %v", err) + } + + customQuota := &capsulev1beta2.CustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pods", + Namespace: "tenant-a", + Generation: 2, + }, + Spec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("10"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: caprunt.VersionKind{APIVersion: "v1", Kind: "Pod"}, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpCount, + }, + }, + }, + }, + Status: capsulev1beta2.CustomQuotaStatus{ + ObservedGeneration: 1, + Conditions: meta.ConditionList{ + {Type: meta.ReadyCondition, Status: metav1.ConditionTrue}, + }, + }, + } + + cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(customQuota).Build() + handler := &objectCalculationHandler{ + targetsCache: cache.NewCompiledTargetsCache[string](), + jsonPathCache: cache.NewJSONPathCache(), + } + request := admission.Request{AdmissionRequest: admissionv1.AdmissionRequest{ + Namespace: "tenant-a", + Kind: metav1.GroupVersionKind{Version: "v1", Kind: "Pod"}, + }} + object := unstructured.Unstructured{} + object.SetAPIVersion("v1") + object.SetKind("Pod") + object.SetNamespace("tenant-a") + object.SetName("pod-a") + + _, err := handler.matchAllQuotas(ctx, cl, request, object) + if err == nil || !strings.Contains(err.Error(), "not ready for generation 2") { + t.Fatalf("matchAllQuotas() error = %v, want current-generation readiness error", err) + } + + current := &capsulev1beta2.CustomQuota{} + if err := cl.Get(ctx, types.NamespacedName{Namespace: "tenant-a", Name: "pods"}, current); err != nil { + t.Fatalf("get CustomQuota: %v", err) + } + current.Status.ObservedGeneration = current.Generation + if err := cl.Update(ctx, current); err != nil { + t.Fatalf("mark CustomQuota ready: %v", err) + } + + matched, err := handler.matchAllQuotas(ctx, cl, request, object) + if err != nil { + t.Fatalf("matchAllQuotas() error = %v", err) + } + if len(matched) != 1 { + t.Fatalf("matchAllQuotas() matches = %d, want 1", len(matched)) + } +} + +func TestCompiledTargetsCacheRefreshesInPlace(t *testing.T) { + t.Parallel() + + targetsCache := cache.NewCompiledTargetsCache[string]() + handler := &objectCalculationHandler{ + targetsCache: targetsCache, + jsonPathCache: cache.NewJSONPathCache(), + } + customQuota := &capsulev1beta2.CustomQuota{ + ObjectMeta: metav1.ObjectMeta{Name: "objects", Namespace: "tenant-a"}, + Spec: capsulev1beta2.CustomQuotaSpec{ + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: caprunt.VersionKind{APIVersion: "v1", Kind: "Pod"}, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpCount, + }, + }, + }, + }, + } + key := "tenant-a/objects" + targetsCache.Set(key, []cache.CompiledTarget{ + { + CustomQuotaStatusTarget: capsulev1beta2.CustomQuotaStatusTarget{ + GroupVersionKind: metav1.GroupVersionKind{Version: "v1", Kind: "Service"}, + }, + }, + }) + + compiled, err := handler.getOrCompileCustomQuotaTargets(customQuota) + if err != nil { + t.Fatalf("getOrCompileCustomQuotaTargets() error = %v", err) + } + if len(compiled) != 1 || compiled[0].Kind != "Pod" { + t.Fatalf("compiled targets = %#v, want current Pod source", compiled) + } + if entries := targetsCache.Stats(); entries != 1 { + t.Fatalf("compiled target cache entries = %d, want one stable quota key", entries) + } +} + +func ledgerForTest(key types.NamespacedName, allocated string) *capsulev1beta2.QuantityLedger { + return &capsulev1beta2.QuantityLedger{ + ObjectMeta: metav1.ObjectMeta{ + Name: key.Name, + Namespace: key.Namespace, + }, + Status: capsulev1beta2.QuantityLedgerStatus{ + Allocated: resource.MustParse(allocated), + }, + } +} + +func statusUpdateRequest( + uid types.UID, + oldPod *corev1.Pod, + newPod *corev1.Pod, +) admission.Request { + return admission.Request{AdmissionRequest: admissionv1.AdmissionRequest{ + UID: uid, + Operation: admissionv1.Update, + Kind: metav1.GroupVersionKind{Version: "v1", Kind: "Pod"}, + Namespace: oldPod.Namespace, + Name: oldPod.Name, + SubResource: "status", + OldObject: runtime.RawExtension{Object: oldPod}, + Object: runtime.RawExtension{Object: newPod}, + }} +} + +func ledgerClientForTest(t *testing.T, ledger *capsulev1beta2.QuantityLedger) client.Client { + t.Helper() + + scheme := runtime.NewScheme() + if err := capsulev1beta2.AddToScheme(scheme); err != nil { + t.Fatalf("add Capsule scheme: %v", err) + } + + return fake.NewClientBuilder(). + WithScheme(scheme). + WithStatusSubresource(&capsulev1beta2.QuantityLedger{}). + WithObjects(ledger). + Build() +} + +type readinessFlappingReader struct { + client.Reader + + customQuotaListCalls int +} + +func (r *readinessFlappingReader) List( + ctx context.Context, + list client.ObjectList, + opts ...client.ListOption, +) error { + if err := r.Reader.List(ctx, list, opts...); err != nil { + return err + } + + quotas, ok := list.(*capsulev1beta2.CustomQuotaList) + if !ok { + return nil + } + + r.customQuotaListCalls++ + if r.customQuotaListCalls < 2 { + return nil + } + + for i := range quotas.Items { + condition := quotas.Items[i].Status.Conditions.GetConditionByType(meta.ReadyCondition) + if condition != nil { + condition.Status = metav1.ConditionFalse + } + } + + return nil +} diff --git a/internal/webhook/namespace/mutation/handler.go b/internal/webhook/namespace/mutation/handler.go index 453dc848..afabb9d5 100644 --- a/internal/webhook/namespace/mutation/handler.go +++ b/internal/webhook/namespace/mutation/handler.go @@ -11,6 +11,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + webhookutils "github.com/projectcapsule/capsule/internal/webhook/utils" ad "github.com/projectcapsule/capsule/pkg/runtime/admission" "github.com/projectcapsule/capsule/pkg/runtime/configuration" "github.com/projectcapsule/capsule/pkg/runtime/events" @@ -37,6 +38,8 @@ func (h *handler) OnCreate( recorder events.EventRecorder, ) handlers.Func { return func(ctx context.Context, req admission.Request) *admission.Response { + reader = webhookutils.NewTenantCachingReader(reader) + user := handlers.ResolveAdmissionUser(ctx, c, req, h.cfg) if !user.IsAdmin() && !user.IsCapsule() { @@ -92,12 +95,6 @@ func (h *handler) OnUpdate( recorder events.EventRecorder, ) handlers.Func { return func(ctx context.Context, req admission.Request) *admission.Response { - user := handlers.ResolveAdmissionUser(ctx, c, req, h.cfg) - - if !user.IsAdmin() && !user.IsCapsule() { - return nil - } - ns := &corev1.Namespace{} if err := decoder.Decode(req, ns); err != nil { return ad.ErroredResponse(err) @@ -108,6 +105,25 @@ func (h *handler) OnUpdate( return ad.ErroredResponse(err) } + // Namespace finalization and terminating updates must never depend on + // Tenant resolution. The standard mutating rule excludes subresources, + // but retain this guard for custom webhook configurations. + if req.SubResource != "" || + ns.DeletionTimestamp != nil || + oldNs.DeletionTimestamp != nil || + ns.Status.Phase == corev1.NamespaceTerminating || + oldNs.Status.Phase == corev1.NamespaceTerminating { + return nil + } + + reader = webhookutils.NewTenantCachingReader(reader) + + user := handlers.ResolveAdmissionUser(ctx, c, req, h.cfg) + + if !user.IsAdmin() && !user.IsCapsule() { + return nil + } + for _, hndl := range h.handlers { response := hndl.OnUpdate(c, reader, user, ns, oldNs, decoder, recorder)(ctx, req) if response == nil { diff --git a/internal/webhook/namespace/mutation/handler_test.go b/internal/webhook/namespace/mutation/handler_test.go index c122659f..af03817c 100644 --- a/internal/webhook/namespace/mutation/handler_test.go +++ b/internal/webhook/namespace/mutation/handler_test.go @@ -83,6 +83,51 @@ func TestNamespaceHandlerDoesNotInterceptUnlabelledAdministratorCreate(t *testin } } +func TestNamespaceHandlerDoesNotInterceptFinalize(t *testing.T) { + t.Parallel() + + scheme := runtime.NewScheme() + if err := corev1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + + now := metav1.Now() + oldNs := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "terminating", + DeletionTimestamp: &now, + }, + Status: corev1.NamespaceStatus{Phase: corev1.NamespaceTerminating}, + } + newNs := oldNs.DeepCopy() + newNs.Spec.Finalizers = nil + + oldRaw, err := json.Marshal(oldNs) + if err != nil { + t.Fatal(err) + } + newRaw, err := json.Marshal(newNs) + if err != nil { + t.Fatal(err) + } + + response := NamespaceHandler(nil).OnUpdate( + nil, + nil, + admission.NewDecoder(scheme), + nil, + )(context.Background(), admission.Request{AdmissionRequest: admissionv1.AdmissionRequest{ + Operation: admissionv1.Update, + SubResource: "finalize", + Object: runtime.RawExtension{Raw: newRaw}, + OldObject: runtime.RawExtension{Raw: oldRaw}, + }}) + + if response != nil { + t.Fatalf("finalize response = %#v, want no interception", response) + } +} + func TestNamespaceHandlerRejectsTenantOwnerLabelMigrationWithEmptyOwnerReferences(t *testing.T) { t.Parallel() diff --git a/internal/webhook/namespace/validation/handler.go b/internal/webhook/namespace/validation/handler.go index a23155a0..2205a349 100644 --- a/internal/webhook/namespace/validation/handler.go +++ b/internal/webhook/namespace/validation/handler.go @@ -6,18 +6,21 @@ package validation import ( "context" "fmt" + "reflect" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/types" + apierrors "k8s.io/apimachinery/pkg/api/errors" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + webhookutils "github.com/projectcapsule/capsule/internal/webhook/utils" ad "github.com/projectcapsule/capsule/pkg/runtime/admission" "github.com/projectcapsule/capsule/pkg/runtime/configuration" "github.com/projectcapsule/capsule/pkg/runtime/events" "github.com/projectcapsule/capsule/pkg/runtime/handlers" "github.com/projectcapsule/capsule/pkg/tenant" + "github.com/projectcapsule/capsule/pkg/users" ) func NamespaceHandler(configuration configuration.Configuration, hndlers ...handlers.TypedHandlerWithTenantUser[*corev1.Namespace]) handlers.Handler { @@ -39,6 +42,8 @@ func (h *handler) OnCreate( recorder events.EventRecorder, ) handlers.Func { return func(ctx context.Context, req admission.Request) *admission.Response { + reader = webhookutils.NewTenantCachingReader(reader) + user := handlers.ResolveAdmissionUser(ctx, c, req, h.cfg) ns := &corev1.Namespace{} @@ -63,12 +68,7 @@ func (h *handler) OnCreate( return nil } - if terminating := h.rejectOnTermination( - ctx, - c, - ns, - tnt, - ); terminating != nil { + if terminating := h.rejectOnTermination(ns, tnt); terminating != nil { return terminating } @@ -89,8 +89,6 @@ func (h *handler) OnDelete( recorder events.EventRecorder, ) handlers.Func { return func(ctx context.Context, req admission.Request) *admission.Response { - user := handlers.ResolveAdmissionUser(ctx, c, req, h.cfg) - oldNs := &corev1.Namespace{} if err := decoder.DecodeRaw(req.OldObject, oldNs); err != nil { return ad.ErroredResponse(err) @@ -103,7 +101,17 @@ func (h *handler) OnDelete( return nil } + reader = webhookutils.NewTenantCachingReader(reader) + tnt, err := tenant.ResolveNamespaceTenant(ctx, reader, oldNs) + if apierrors.IsNotFound(err) { + // Kubernetes authorization already controls namespace deletion. + // A stale Tenant reference must not make a namespace undeletable. + return nil + } + + user := handlers.ResolveAdmissionUser(ctx, c, req, h.cfg) + if err != nil && !user.IsAdmin() { return ad.ErroredResponse(err) } @@ -122,7 +130,6 @@ func (h *handler) OnDelete( } } -//nolint:cyclop func (h *handler) OnUpdate( c client.Client, reader client.Reader, @@ -130,8 +137,6 @@ func (h *handler) OnUpdate( recorder events.EventRecorder, ) handlers.Func { return func(ctx context.Context, req admission.Request) *admission.Response { - user := handlers.ResolveAdmissionUser(ctx, c, req, h.cfg) - ns := &corev1.Namespace{} if err := decoder.Decode(req, ns); err != nil { return ad.ErroredResponse(err) @@ -142,23 +147,16 @@ func (h *handler) OnUpdate( return ad.ErroredResponse(err) } - oldHasTenantReference := tenant.HasTenantReference(oldNs) - - newHasTenantReference := tenant.HasTenantReference(ns) - - if user.IsAdmin() && !tenant.HasConsistentTenantReference(ns) { - return ad.Deny("tenant label and ownerReference must both be set consistently or both be absent") + if response, stop := validateTerminatingNamespaceUpdate(req, oldNs, ns); stop { + return response } - if !user.IsAdmin() { - switch { - case !oldHasTenantReference && newHasTenantReference: - return ad.Deny("namespace can not be patched into a tenant") - case oldHasTenantReference && !newHasTenantReference: - return ad.Deny("namespace can not remove tenant ownership") - case !oldHasTenantReference && !newHasTenantReference: - return nil - } + reader = webhookutils.NewTenantCachingReader(reader) + + user := handlers.ResolveAdmissionUser(ctx, c, req, h.cfg) + + if response, stop := validateNamespaceTenantReferenceTransition(user, oldNs, ns); stop { + return response } oldTenant, err := tenant.ResolveNamespaceTenant(ctx, reader, oldNs) @@ -206,7 +204,7 @@ func (h *handler) OnUpdate( return nil } - if terminating := h.rejectOnTermination(ctx, c, ns, newTenant); terminating != nil { + if terminating := h.rejectOnTermination(ns, newTenant); terminating != nil { return terminating } @@ -232,9 +230,71 @@ func namespaceTenantChanged(oldTenant, newTenant *capsulev1beta2.Tenant) bool { return oldTenant.GetName() != newTenant.GetName() || oldTenant.GetUID() != newTenant.GetUID() } +func isTerminatingNamespaceUpdate( + req admission.Request, + oldNs, newNs *corev1.Namespace, +) bool { + return req.SubResource == "finalize" || + newNs.DeletionTimestamp != nil || + oldNs.DeletionTimestamp != nil || + newNs.Status.Phase == corev1.NamespaceTerminating || + oldNs.Status.Phase == corev1.NamespaceTerminating +} + +func validateTerminatingNamespaceUpdate( + req admission.Request, + oldNs, newNs *corev1.Namespace, +) (*admission.Response, bool) { + if !isTerminatingNamespaceUpdate(req, oldNs, newNs) { + return nil, false + } + + if namespaceTenantAssignmentChanged(oldNs, newNs) { + return ad.Deny("namespace tenant ownership can not change during termination"), true + } + + return nil, true +} + +func validateNamespaceTenantReferenceTransition( + user users.AdmissionUser, + oldNs, newNs *corev1.Namespace, +) (*admission.Response, bool) { + if user.IsAdmin() { + if !tenant.HasConsistentTenantReference(newNs) { + return ad.Deny("tenant label and ownerReference must both be set consistently or both be absent"), true + } + + return nil, false + } + + oldHasTenantReference := tenant.HasTenantReference(oldNs) + newHasTenantReference := tenant.HasTenantReference(newNs) + + switch { + case !oldHasTenantReference && newHasTenantReference: + return ad.Deny("namespace can not be patched into a tenant"), true + case oldHasTenantReference && !newHasTenantReference: + return ad.Deny("namespace can not remove tenant ownership"), true + case !oldHasTenantReference && !newHasTenantReference: + return nil, true + default: + return nil, false + } +} + +func namespaceTenantAssignmentChanged(oldNs, newNs *corev1.Namespace) bool { + if tenant.TenanLabelValue(oldNs) != tenant.TenanLabelValue(newNs) { + return true + } + + return !reflect.DeepEqual( + tenant.TenantOwnerReferences(oldNs), + tenant.TenantOwnerReferences(newNs), + ) +} + func (h *handler) rejectOnTermination( - ctx context.Context, - c client.Reader, ns *corev1.Namespace, t *capsulev1beta2.Tenant, ) *admission.Response { @@ -242,15 +302,11 @@ func (h *handler) rejectOnTermination( return nil } - tnt := &capsulev1beta2.Tenant{} - - _ = c.Get(ctx, types.NamespacedName{Name: t.GetName()}, tnt) - - if tnt.DeletionTimestamp == nil { + if t.DeletionTimestamp == nil { return nil } - instance := tnt.Status.GetInstance(&capsulev1beta2.TenantStatusNamespaceItem{ + instance := t.Status.GetInstance(&capsulev1beta2.TenantStatusNamespaceItem{ Name: ns.GetName(), UID: ns.GetUID(), }) diff --git a/internal/webhook/namespace/validation/handler_test.go b/internal/webhook/namespace/validation/handler_test.go new file mode 100644 index 00000000..ce30bfb4 --- /dev/null +++ b/internal/webhook/namespace/validation/handler_test.go @@ -0,0 +1,145 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package validation + +import ( + "context" + "encoding/json" + "testing" + + admissionv1 "k8s.io/api/admission/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/pkg/api/meta" +) + +func TestNamespaceHandlerAllowsUnchangedFinalizeWithoutTenant(t *testing.T) { + t.Parallel() + + scheme := namespaceValidationScheme(t) + now := metav1.Now() + oldNs := namespaceWithTenantReference("workloads", "missing", "missing-uid") + oldNs.DeletionTimestamp = &now + oldNs.Status.Phase = corev1.NamespaceTerminating + newNs := oldNs.DeepCopy() + newNs.Spec.Finalizers = nil + + response := NamespaceHandler(nil).OnUpdate( + nil, + nil, + admission.NewDecoder(scheme), + nil, + )(context.Background(), namespaceUpdateRequest(t, oldNs, newNs, "finalize")) + + if response != nil { + t.Fatalf("finalize response = %#v, want no interception", response) + } +} + +func TestNamespaceHandlerRejectsTenantChangeDuringFinalize(t *testing.T) { + t.Parallel() + + scheme := namespaceValidationScheme(t) + now := metav1.Now() + oldNs := namespaceWithTenantReference("workloads", "solar", "solar-uid") + oldNs.DeletionTimestamp = &now + oldNs.Status.Phase = corev1.NamespaceTerminating + newNs := namespaceWithTenantReference("workloads", "lunar", "lunar-uid") + newNs.DeletionTimestamp = &now + newNs.Status.Phase = corev1.NamespaceTerminating + + response := NamespaceHandler(nil).OnUpdate( + nil, + nil, + admission.NewDecoder(scheme), + nil, + )(context.Background(), namespaceUpdateRequest(t, oldNs, newNs, "finalize")) + + if response == nil || response.Allowed { + t.Fatalf("finalize response = %#v, want tenant assignment denial", response) + } +} + +func TestNamespaceHandlerAllowsDeleteWithMissingTenant(t *testing.T) { + t.Parallel() + + scheme := namespaceValidationScheme(t) + reader := fake.NewClientBuilder().WithScheme(scheme).Build() + oldNs := namespaceWithTenantReference("workloads", "missing", "missing-uid") + raw, err := json.Marshal(oldNs) + if err != nil { + t.Fatal(err) + } + + response := NamespaceHandler(nil).OnDelete( + reader, + reader, + admission.NewDecoder(scheme), + nil, + )(context.Background(), admission.Request{AdmissionRequest: admissionv1.AdmissionRequest{ + Operation: admissionv1.Delete, + OldObject: runtime.RawExtension{Raw: raw}, + }}) + + if response != nil { + t.Fatalf("delete response = %#v, want missing Tenant to be ignored", response) + } +} + +func namespaceValidationScheme(t *testing.T) *runtime.Scheme { + t.Helper() + + scheme := runtime.NewScheme() + if err := corev1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + if err := capsulev1beta2.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + + return scheme +} + +func namespaceWithTenantReference(name, tenantName, tenantUID string) *corev1.Namespace { + return &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: map[string]string{meta.TenantLabel: tenantName}, + OwnerReferences: []metav1.OwnerReference{{ + APIVersion: capsulev1beta2.GroupVersion.String(), + Kind: "Tenant", + Name: tenantName, + UID: types.UID(tenantUID), + }}, + }} +} + +func namespaceUpdateRequest( + t *testing.T, + oldNs, newNs *corev1.Namespace, + subresource string, +) admission.Request { + t.Helper() + + oldRaw, err := json.Marshal(oldNs) + if err != nil { + t.Fatal(err) + } + newRaw, err := json.Marshal(newNs) + if err != nil { + t.Fatal(err) + } + + return admission.Request{AdmissionRequest: admissionv1.AdmissionRequest{ + Operation: admissionv1.Update, + SubResource: subresource, + Object: runtime.RawExtension{Raw: newRaw}, + OldObject: runtime.RawExtension{Raw: oldRaw}, + }} +} diff --git a/internal/webhook/pvc/handler.go b/internal/webhook/pvc/handler.go index bfcacaf9..dd353e04 100644 --- a/internal/webhook/pvc/handler.go +++ b/internal/webhook/pvc/handler.go @@ -4,7 +4,10 @@ package pvc import ( + admissionv1 "k8s.io/api/admission/v1" corev1 "k8s.io/api/core/v1" + apiequality "k8s.io/apimachinery/pkg/api/equality" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" "github.com/projectcapsule/capsule/pkg/runtime/handlers" ) @@ -14,6 +17,48 @@ func Handler(handler ...handlers.TypedHandlerWithTenant[*corev1.PersistentVolume Factory: func() *corev1.PersistentVolumeClaim { return &corev1.PersistentVolumeClaim{} }, - Handlers: handler, + Handlers: handler, + Predicate: requiresPVCSpecValidation, } } + +func MutatingHandler(handler ...handlers.TypedHandlerWithTenant[*corev1.PersistentVolumeClaim]) handlers.Handler { + return &handlers.TypedTenantHandler[*corev1.PersistentVolumeClaim]{ + Factory: func() *corev1.PersistentVolumeClaim { + return &corev1.PersistentVolumeClaim{} + }, + Handlers: handler, + Predicate: func( + req admission.Request, + pvc *corev1.PersistentVolumeClaim, + oldPVC *corev1.PersistentVolumeClaim, + ) bool { + if !requiresPVCSpecValidation(req, pvc, oldPVC) { + return false + } + + if pvc.Spec.Selector != nil { + return true + } + + return req.Operation == admissionv1.Create && pvc.Spec.VolumeName != "" + }, + } +} + +func requiresPVCSpecValidation( + req admission.Request, + pvc *corev1.PersistentVolumeClaim, + oldPVC *corev1.PersistentVolumeClaim, +) bool { + // Finalizer cleanup must remain possible after a bound PV has disappeared. + // Continue validating any update that changes the PVC spec. + if req.Operation != admissionv1.Update || + pvc == nil || + oldPVC == nil || + pvc.DeletionTimestamp == nil { + return true + } + + return !apiequality.Semantic.DeepEqual(pvc.Spec, oldPVC.Spec) +} diff --git a/internal/webhook/pvc/handler_test.go b/internal/webhook/pvc/handler_test.go new file mode 100644 index 00000000..a4face83 --- /dev/null +++ b/internal/webhook/pvc/handler_test.go @@ -0,0 +1,195 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package pvc + +import ( + "context" + "encoding/json" + "testing" + + admissionv1 "k8s.io/api/admission/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +func TestMutatingHandlerSkipsDynamicClaimsBeforeTenantLookup(t *testing.T) { + t.Parallel() + + scheme := runtime.NewScheme() + if err := corev1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + + base := fake.NewClientBuilder().WithScheme(scheme).Build() + reader := &pvcCountingReader{Reader: base} + handler := MutatingHandler() + request := pvcAdmissionRequest(t, &corev1.PersistentVolumeClaim{}) + + if response := handler.OnCreate(nil, reader, admission.NewDecoder(scheme), nil)( + context.Background(), + request, + ); response != nil { + t.Fatalf("response = %#v, want nil", response) + } + + if reader.gets != 0 { + t.Fatalf("tenant lookup gets = %d, want 0 for a dynamic claim", reader.gets) + } +} + +func TestValidatingHandlerSkipsTerminatingClaimWithUnchangedSpec(t *testing.T) { + t.Parallel() + + scheme := runtime.NewScheme() + if err := corev1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + + now := metav1.Now() + oldPVC := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + DeletionTimestamp: &now, + Finalizers: []string{"kubernetes.io/pvc-protection"}, + }, + Spec: corev1.PersistentVolumeClaimSpec{VolumeName: "caladan"}, + } + newPVC := oldPVC.DeepCopy() + newPVC.Finalizers = nil + + base := fake.NewClientBuilder().WithScheme(scheme).Build() + reader := &pvcCountingReader{Reader: base} + handler := Handler(PersistentVolumeValidatingVolume()) + request := pvcUpdateAdmissionRequest(t, oldPVC, newPVC) + + if response := handler.OnUpdate(nil, reader, admission.NewDecoder(scheme), nil)( + context.Background(), + request, + ); response != nil { + t.Fatalf("response = %#v, want nil", response) + } + + if reader.gets != 0 { + t.Fatalf("admission gets = %d, want 0 for terminating claim cleanup", reader.gets) + } +} + +func TestRequiresPVCSpecValidation(t *testing.T) { + t.Parallel() + + now := metav1.Now() + terminating := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{DeletionTimestamp: &now}, + Spec: corev1.PersistentVolumeClaimSpec{VolumeName: "caladan"}, + } + changed := terminating.DeepCopy() + changed.Spec.VolumeName = "salusa" + active := terminating.DeepCopy() + active.DeletionTimestamp = nil + + tests := []struct { + name string + operation admissionv1.Operation + oldPVC *corev1.PersistentVolumeClaim + pvc *corev1.PersistentVolumeClaim + want bool + }{ + { + name: "create", + operation: admissionv1.Create, + pvc: active, + want: true, + }, + { + name: "active update", + operation: admissionv1.Update, + oldPVC: active.DeepCopy(), + pvc: active, + want: true, + }, + { + name: "terminating finalizer update", + operation: admissionv1.Update, + oldPVC: terminating.DeepCopy(), + pvc: terminating, + want: false, + }, + { + name: "terminating spec update", + operation: admissionv1.Update, + oldPVC: terminating.DeepCopy(), + pvc: changed, + want: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := admission.Request{AdmissionRequest: admissionv1.AdmissionRequest{ + Operation: tt.operation, + }} + + if got := requiresPVCSpecValidation(req, tt.pvc, tt.oldPVC); got != tt.want { + t.Fatalf("requiresPVCSpecValidation() = %t, want %t", got, tt.want) + } + }) + } +} + +type pvcCountingReader struct { + client.Reader + + gets int +} + +func (r *pvcCountingReader) Get( + ctx context.Context, + key client.ObjectKey, + obj client.Object, + opts ...client.GetOption, +) error { + r.gets++ + + return r.Reader.Get(ctx, key, obj, opts...) +} + +func pvcAdmissionRequest(t *testing.T, pvc *corev1.PersistentVolumeClaim) admission.Request { + t.Helper() + + raw, err := json.Marshal(pvc) + if err != nil { + t.Fatal(err) + } + + return admission.Request{AdmissionRequest: admissionv1.AdmissionRequest{ + Operation: admissionv1.Create, + Namespace: "solar", + Object: runtime.RawExtension{ + Raw: raw, + }, + }} +} + +func pvcUpdateAdmissionRequest( + t *testing.T, + oldPVC *corev1.PersistentVolumeClaim, + pvc *corev1.PersistentVolumeClaim, +) admission.Request { + t.Helper() + + request := pvcAdmissionRequest(t, pvc) + request.Operation = admissionv1.Update + + raw, err := json.Marshal(oldPVC) + if err != nil { + t.Fatal(err) + } + + request.OldObject = runtime.RawExtension{Raw: raw} + + return request +} diff --git a/internal/webhook/router.go b/internal/webhook/router.go index a6c564b9..eaed793d 100644 --- a/internal/webhook/router.go +++ b/internal/webhook/router.go @@ -16,6 +16,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + webhookutils "github.com/projectcapsule/capsule/internal/webhook/utils" "github.com/projectcapsule/capsule/pkg/runtime/events" "github.com/projectcapsule/capsule/pkg/runtime/handlers" ) @@ -83,22 +84,27 @@ func (r *handlerRouter) Handle(ctx context.Context, req admission.Request) admis attribute.String("admission.webhook.path", r.path), ) + reader := r.reader + if len(r.handlers) > 1 { + reader = webhookutils.NewRequestCachingReader(reader) + } + switch req.Operation { case admissionv1.Create: for _, h := range r.handlers { - if response := h.OnCreate(r.client, r.reader, r.decoder, r.recorder)(ctx, req); response != nil { + if response := h.OnCreate(r.client, reader, r.decoder, r.recorder)(ctx, req); response != nil { return r.recordResponse(span, *response) } } case admissionv1.Update: for _, h := range r.handlers { - if response := h.OnUpdate(r.client, r.reader, r.decoder, r.recorder)(ctx, req); response != nil { + if response := h.OnUpdate(r.client, reader, r.decoder, r.recorder)(ctx, req); response != nil { return r.recordResponse(span, *response) } } case admissionv1.Delete: for _, h := range r.handlers { - if response := h.OnDelete(r.client, r.reader, r.decoder, r.recorder)(ctx, req); response != nil { + if response := h.OnDelete(r.client, reader, r.decoder, r.recorder)(ctx, req); response != nil { return r.recordResponse(span, *response) } } diff --git a/internal/webhook/rules/generic/mutation/metadata.go b/internal/webhook/rules/generic/mutation/metadata.go index 2ab5ec6a..c094152e 100644 --- a/internal/webhook/rules/generic/mutation/metadata.go +++ b/internal/webhook/rules/generic/mutation/metadata.go @@ -7,7 +7,6 @@ import ( "context" "encoding/json" "fmt" - "maps" "net/http" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -49,7 +48,9 @@ func (*metadataRules) mutate(obj *unstructured.Unstructured, bodies []*apirules. return &response } - MutateMetadata(obj, gvk, bodies) + if !MutateMetadata(obj, gvk, bodies) { + return nil + } marshaled, err := json.Marshal(obj) if err != nil { @@ -64,14 +65,23 @@ func (*metadataRules) mutate(obj *unstructured.Unstructured, bodies []*apirules. } } -func MutateMetadata(obj metav1.Object, gvk schema.GroupVersionKind, bodies []*apirules.NamespaceRuleBodyNamespace) { +func MutateMetadata( + obj metav1.Object, + gvk schema.GroupVersionKind, + bodies []*apirules.NamespaceRuleBodyNamespace, +) bool { if obj == nil { - return + return false } labels, annotations := obj.GetLabels(), obj.GetAnnotations() - defaultLabels, managedLabels := map[string]string{}, map[string]string{} - defaultAnnotations, managedAnnotations := map[string]string{}, map[string]string{} + + var ( + defaultLabels map[string]string + managedLabels map[string]string + defaultAnnotations map[string]string + managedAnnotations map[string]string + ) for _, body := range bodies { if body == nil || body.Enforce == nil { @@ -83,16 +93,29 @@ func MutateMetadata(obj metav1.Object, gvk schema.GroupVersionKind, bodies []*ap continue } + if defaultLabels == nil { + defaultLabels = map[string]string{} + managedLabels = map[string]string{} + defaultAnnotations = map[string]string{} + managedAnnotations = map[string]string{} + } + collectMutation(rule.Labels, defaultLabels, managedLabels) collectMutation(rule.Annotations, defaultAnnotations, managedAnnotations) } } - labels = applyMutation(labels, defaultLabels, managedLabels) - annotations = applyMutation(annotations, defaultAnnotations, managedAnnotations) + labels, labelsChanged := applyMutation(labels, defaultLabels, managedLabels) + annotations, annotationsChanged := applyMutation(annotations, defaultAnnotations, managedAnnotations) + + if !labelsChanged && !annotationsChanged { + return false + } obj.SetLabels(labels) obj.SetAnnotations(annotations) + + return true } func collectMutation(policies map[string]apirules.MetadataValueRule, defaults, managed map[string]string) { @@ -107,22 +130,32 @@ func collectMutation(policies map[string]apirules.MetadataValueRule, defaults, m } } -func applyMutation(current, defaults, managed map[string]string) map[string]string { +func applyMutation(current, defaults, managed map[string]string) (map[string]string, bool) { if len(defaults) == 0 && len(managed) == 0 { - return current + return current, false } if current == nil { current = map[string]string{} } + changed := false + for key, value := range defaults { if _, ok := current[key]; !ok { current[key] = value + changed = true } } - maps.Copy(current, managed) + for key, value := range managed { + if current[key] == value { + continue + } - return current + current[key] = value + changed = true + } + + return current, changed } diff --git a/internal/webhook/rules/generic/mutation/metadata_test.go b/internal/webhook/rules/generic/mutation/metadata_test.go index b424fe54..74cdc1d3 100644 --- a/internal/webhook/rules/generic/mutation/metadata_test.go +++ b/internal/webhook/rules/generic/mutation/metadata_test.go @@ -35,7 +35,9 @@ func TestMutateMetadataDefaultsAndManaged(t *testing.T) { }}, }}} - MutateMetadata(obj, schema.GroupVersionKind{Version: "v1", Kind: "ConfigMap"}, bodies) + if changed := MutateMetadata(obj, schema.GroupVersionKind{Version: "v1", Kind: "ConfigMap"}, bodies); !changed { + t.Fatal("MutateMetadata() changed = false, want true") + } if got := obj.GetLabels()["default-missing"]; got != "fallback" { t.Fatalf("default = %q", got) } @@ -53,3 +55,18 @@ func TestMutateMetadataDefaultsAndManaged(t *testing.T) { t.Fatal("subjects were removed") } } + +func TestMutateMetadataReportsNoop(t *testing.T) { + t.Parallel() + + obj := &unstructured.Unstructured{} + obj.SetLabels(map[string]string{"existing": "value"}) + + if changed := MutateMetadata( + obj, + schema.GroupVersionKind{Version: "v1", Kind: "ConfigMap"}, + nil, + ); changed { + t.Fatal("MutateMetadata() changed = true without matching rules") + } +} diff --git a/internal/webhook/rules/generic/validation/register.go b/internal/webhook/rules/generic/validation/register.go index 66eb8478..52d68919 100644 --- a/internal/webhook/rules/generic/validation/register.go +++ b/internal/webhook/rules/generic/validation/register.go @@ -4,11 +4,18 @@ package validation import ( + "context" + "slices" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" "github.com/projectcapsule/capsule/internal/cache" "github.com/projectcapsule/capsule/pkg/runtime/configuration" + "github.com/projectcapsule/capsule/pkg/runtime/events" "github.com/projectcapsule/capsule/pkg/runtime/handlers" ) @@ -17,24 +24,52 @@ const Path = "/rules/generic/validating" type genericValidating struct { regexCache *cache.RegexCache configuration configuration.Configuration + resourceRules []handlers.Handler } -func Register(regexCache *cache.RegexCache, cfg configuration.Configuration) handlers.Webhook { +func Register( + regexCache *cache.RegexCache, + cfg configuration.Configuration, + resourceRules ...handlers.Handler, +) handlers.Webhook { return &genericValidating{ regexCache: regexCache, configuration: cfg, + resourceRules: resourceRules, } } func (w *genericValidating) GetHandlers() []handlers.Handler { - return []handlers.Handler{ + out := make([]handlers.Handler, 0, len(w.resourceRules)+2) + out = append(out, matchingRequest( + matchesGenericMetadataRequest, genericHandler(w.configuration, GenericRules(w.regexCache), ), + )) + out = append(out, w.resourceRules...) + out = append(out, matchingRequest( + func(req admission.Request) bool { + _, supported := ingressTypeForGVK(requestGVK(req)) + + return supported && req.SubResource == "" + }, ingressHandler(w.configuration, IngressRules(w.regexCache), ), + )) + + return out +} + +func matchesGenericMetadataRequest(req admission.Request) bool { + if req.SubResource == "" { + return true } + + gvk := requestGVK(req) + + return gvk.Group == "" && gvk.Kind == "Namespace" } func (genericValidating) GetPath() string { @@ -62,3 +97,87 @@ func genericHandler(cfg configuration.Configuration, Configuration: cfg, } } + +// ForKind scopes a validator hosted by the generic rules endpoint to one +// concrete Kubernetes kind. The main resource is always included; named +// subresources may be included explicitly. The scope check runs before +// tenant/ruleset resolution and object decoding. +func ForKind( + gk schema.GroupKind, + handler handlers.Handler, + subresources ...string, +) handlers.Handler { + return matchingRequest(func(req admission.Request) bool { + gvk := requestGVK(req) + if gvk.Group != gk.Group || gvk.Kind != gk.Kind { + return false + } + + return req.SubResource == "" || slices.Contains(subresources, req.SubResource) + }, handler) +} + +type requestPredicate func(admission.Request) bool + +type matchingHandler struct { + predicate requestPredicate + handler handlers.Handler +} + +func matchingRequest(predicate requestPredicate, handler handlers.Handler) handlers.Handler { + return &matchingHandler{ + predicate: predicate, + handler: handler, + } +} + +func (h *matchingHandler) OnCreate( + c client.Client, + reader client.Reader, + decoder admission.Decoder, + recorder events.EventRecorder, +) handlers.Func { + next := h.handler.OnCreate(c, reader, decoder, recorder) + + return h.handle(next) +} + +func (h *matchingHandler) OnUpdate( + c client.Client, + reader client.Reader, + decoder admission.Decoder, + recorder events.EventRecorder, +) handlers.Func { + next := h.handler.OnUpdate(c, reader, decoder, recorder) + + return h.handle(next) +} + +func (h *matchingHandler) OnDelete( + c client.Client, + reader client.Reader, + decoder admission.Decoder, + recorder events.EventRecorder, +) handlers.Func { + next := h.handler.OnDelete(c, reader, decoder, recorder) + + return h.handle(next) +} + +func (h *matchingHandler) handle(next handlers.Func) handlers.Func { + return func(ctx context.Context, req admission.Request) *admission.Response { + if !h.predicate(req) { + return nil + } + + return next(ctx, req) + } +} + +func requestGVK(req admission.Request) schema.GroupVersionKind { + return schema.GroupVersionKind{ + Group: req.Kind.Group, + Version: req.Kind.Version, + Kind: req.Kind.Kind, + } +} diff --git a/internal/webhook/rules/generic/validation/register_test.go b/internal/webhook/rules/generic/validation/register_test.go new file mode 100644 index 00000000..48fdb681 --- /dev/null +++ b/internal/webhook/rules/generic/validation/register_test.go @@ -0,0 +1,168 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package validation + +import ( + "context" + "testing" + + admissionv1 "k8s.io/api/admission/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + "github.com/projectcapsule/capsule/pkg/runtime/events" + "github.com/projectcapsule/capsule/pkg/runtime/handlers" +) + +func TestForKindFiltersBeforeCallingHandler(t *testing.T) { + t.Parallel() + + spy := &requestSpyHandler{} + handler := ForKind(schema.GroupKind{Kind: "Pod"}, spy, "ephemeralcontainers") + handle := handler.OnCreate(nil, nil, nil, nil) + + if response := handle(context.Background(), requestWithKind("", "Service")); response != nil { + t.Fatalf("service response = %#v, want nil", response) + } + if spy.calls != 0 { + t.Fatalf("handler calls = %d, want 0 for a different kind", spy.calls) + } + + if response := handle(context.Background(), requestWithKind("", "Pod")); response != nil { + t.Fatalf("pod response = %#v, want nil", response) + } + if spy.calls != 1 { + t.Fatalf("handler calls = %d, want 1 for Pod", spy.calls) + } + + statusRequest := requestWithKind("", "Pod") + statusRequest.SubResource = "status" + if response := handle(context.Background(), statusRequest); response != nil { + t.Fatalf("pod status response = %#v, want nil", response) + } + if spy.calls != 1 { + t.Fatalf("handler calls = %d, want Pod status to be filtered", spy.calls) + } + + ephemeralRequest := requestWithKind("", "Pod") + ephemeralRequest.SubResource = "ephemeralcontainers" + if response := handle(context.Background(), ephemeralRequest); response != nil { + t.Fatalf("pod ephemeral containers response = %#v, want nil", response) + } + if spy.calls != 2 { + t.Fatalf("handler calls = %d, want Pod ephemeral containers to be included", spy.calls) + } +} + +func TestGenericValidatingIncludesResourceHandlers(t *testing.T) { + t.Parallel() + + webhook := Register(nil, nil, &requestSpyHandler{}, &requestSpyHandler{}) + if got := len(webhook.GetHandlers()); got != 4 { + t.Fatalf("handlers = %d, want generic metadata, two resource handlers, and ingress", got) + } +} + +func TestMatchesGenericMetadataRequest(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + request admission.Request + wantMatched bool + }{ + { + name: "main resource", + request: requestWithKind("apps", "Deployment"), + wantMatched: true, + }, + { + name: "namespace status", + request: func() admission.Request { + req := requestWithKind("", "Namespace") + req.SubResource = "status" + + return req + }(), + wantMatched: true, + }, + { + name: "deployment scale", + request: func() admission.Request { + req := requestWithKind("apps", "Deployment") + req.SubResource = "scale" + + return req + }(), + wantMatched: false, + }, + { + name: "pod status", + request: func() admission.Request { + req := requestWithKind("", "Pod") + req.SubResource = "status" + + return req + }(), + wantMatched: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + if got := matchesGenericMetadataRequest(tt.request); got != tt.wantMatched { + t.Fatalf("matchesGenericMetadataRequest() = %t, want %t", got, tt.wantMatched) + } + }) + } +} + +type requestSpyHandler struct { + calls int +} + +func (h *requestSpyHandler) OnCreate( + client.Client, + client.Reader, + admission.Decoder, + events.EventRecorder, +) handlers.Func { + return func(context.Context, admission.Request) *admission.Response { + h.calls++ + + return nil + } +} + +func (h *requestSpyHandler) OnUpdate( + client.Client, + client.Reader, + admission.Decoder, + events.EventRecorder, +) handlers.Func { + return h.OnCreate(nil, nil, nil, nil) +} + +func (h *requestSpyHandler) OnDelete( + client.Client, + client.Reader, + admission.Decoder, + events.EventRecorder, +) handlers.Func { + return h.OnCreate(nil, nil, nil, nil) +} + +func requestWithKind(group, kind string) admission.Request { + return admission.Request{AdmissionRequest: admissionv1.AdmissionRequest{ + Kind: metav1.GroupVersionKind{ + Group: group, + Version: "v1", + Kind: kind, + }, + }} +} diff --git a/internal/webhook/service/validating.go b/internal/webhook/service/validating.go index a037445c..ccf472ef 100644 --- a/internal/webhook/service/validating.go +++ b/internal/webhook/service/validating.go @@ -171,15 +171,22 @@ func (h *validating) handle( return nil } + allowedNetworks := make([]*net.IPNet, 0, len(tnt.Spec.ServiceOptions.ExternalServiceIPs.Allowed)) + + for _, allowed := range tnt.Spec.ServiceOptions.ExternalServiceIPs.Allowed { + if !strings.Contains(string(allowed), "/") { + allowed += "/32" + } + + _, allowedIP, _ := net.ParseCIDR(string(allowed)) + if allowedIP != nil { + allowedNetworks = append(allowedNetworks, allowedIP) + } + } + ipInCIDR := func(ip net.IP) bool { - for _, allowed := range tnt.Spec.ServiceOptions.ExternalServiceIPs.Allowed { - if !strings.Contains(string(allowed), "/") { - allowed += "/32" - } - - _, allowedIP, _ := net.ParseCIDR(string(allowed)) - - if allowedIP.Contains(ip) { + for i := range allowedNetworks { + if allowedNetworks[i].Contains(ip) { return true } } diff --git a/internal/webhook/serviceaccounts/handler.go b/internal/webhook/serviceaccounts/handler.go index 186fe9cc..c716a783 100644 --- a/internal/webhook/serviceaccounts/handler.go +++ b/internal/webhook/serviceaccounts/handler.go @@ -4,10 +4,16 @@ package serviceaccounts import ( - corev1 "k8s.io/api/core/v1" + "context" + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + "github.com/projectcapsule/capsule/pkg/api/meta" "github.com/projectcapsule/capsule/pkg/runtime/configuration" "github.com/projectcapsule/capsule/pkg/runtime/handlers" + "github.com/projectcapsule/capsule/pkg/users" ) func Handler(cfg configuration.Configuration, handler ...handlers.TypedHandlerWithTenantUser[*corev1.ServiceAccount]) handlers.Handler { @@ -17,5 +23,47 @@ func Handler(cfg configuration.Configuration, handler ...handlers.TypedHandlerWi }, Handlers: handler, Configuration: cfg, + Predicate: func( + _ admission.Request, + sa *corev1.ServiceAccount, + _ *corev1.ServiceAccount, + ) bool { + labels := sa.GetLabels() + if len(labels) == 0 { + return false + } + + _, promotion := labels[meta.ServiceAccountPromotionLabel] + _, ownerPromotion := labels[meta.OwnerPromotionLabel] + + return promotion || ownerPromotion + }, + UserResolver: resolvePromotionUser, } } + +func resolvePromotionUser( + _ context.Context, + _ client.Client, + req admission.Request, + cfg configuration.Configuration, +) users.AdmissionUser { + user := users.NewAdmissionUser(users.AdmissionUserUnknown, req.UserInfo) + + if user.IsControllerServiceAccount() { + user.Type = users.AdmissionUserAdmin + + return user + } + + config := cfg.GetConfigObject() + if users.HasIgnoredGroup(req.UserInfo.Groups, config.Spec.IgnoreUserWithGroups) { + return user + } + + if config.Spec.Administrators.IsPresent(req.UserInfo.Username, req.UserInfo.Groups) { + user.Type = users.AdmissionUserAdmin + } + + return user +} diff --git a/internal/webhook/serviceaccounts/handler_test.go b/internal/webhook/serviceaccounts/handler_test.go new file mode 100644 index 00000000..18a4a18c --- /dev/null +++ b/internal/webhook/serviceaccounts/handler_test.go @@ -0,0 +1,76 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package serviceaccounts + +import ( + "context" + "encoding/json" + "testing" + + admissionv1 "k8s.io/api/admission/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +func TestHandlerSkipsUnlabeledServiceAccountsBeforeTenantLookup(t *testing.T) { + t.Parallel() + + scheme := runtime.NewScheme() + if err := corev1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + + base := fake.NewClientBuilder().WithScheme(scheme).Build() + reader := &serviceAccountCountingReader{Reader: base} + handler := Handler(nil) + request := serviceAccountAdmissionRequest(t, &corev1.ServiceAccount{}) + + if response := handler.OnCreate(nil, reader, admission.NewDecoder(scheme), nil)( + context.Background(), + request, + ); response != nil { + t.Fatalf("response = %#v, want nil", response) + } + + if reader.gets != 0 { + t.Fatalf("tenant lookup gets = %d, want 0 without promotion labels", reader.gets) + } +} + +type serviceAccountCountingReader struct { + client.Reader + + gets int +} + +func (r *serviceAccountCountingReader) Get( + ctx context.Context, + key client.ObjectKey, + obj client.Object, + opts ...client.GetOption, +) error { + r.gets++ + + return r.Reader.Get(ctx, key, obj, opts...) +} + +func serviceAccountAdmissionRequest(t *testing.T, sa *corev1.ServiceAccount) admission.Request { + t.Helper() + + raw, err := json.Marshal(sa) + if err != nil { + t.Fatal(err) + } + + return admission.Request{AdmissionRequest: admissionv1.AdmissionRequest{ + Operation: admissionv1.Create, + Namespace: "solar", + Object: runtime.RawExtension{ + Raw: raw, + }, + }} +} diff --git a/internal/webhook/utils/request_reader.go b/internal/webhook/utils/request_reader.go new file mode 100644 index 00000000..582353e0 --- /dev/null +++ b/internal/webhook/utils/request_reader.go @@ -0,0 +1,127 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package utils + +import ( + "context" + "fmt" + "reflect" + "sync" + + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type requestReadKey struct { + key client.ObjectKey + objectType reflect.Type +} + +type requestReadResult struct { + object client.Object + err error + ready chan struct{} +} + +type requestCachingReader struct { + client.Reader + + mu sync.Mutex + results map[requestReadKey]requestReadResult +} + +// NewRequestCachingReader deduplicates identical direct reads made by handlers +// participating in one admission request. Results are deep-copied before they +// are returned so one handler cannot mutate the snapshot observed by another. +func NewRequestCachingReader(reader client.Reader) client.Reader { + return &requestCachingReader{ + Reader: reader, + results: make(map[requestReadKey]requestReadResult), + } +} + +func (r *requestCachingReader) Get( + ctx context.Context, + key client.ObjectKey, + obj client.Object, + opts ...client.GetOption, +) error { + if len(opts) > 0 { + return r.Reader.Get(ctx, key, obj, opts...) + } + + cacheKey := requestReadKey{ + key: key, + objectType: reflect.TypeOf(obj), + } + + r.mu.Lock() + if result, found := r.results[cacheKey]; found { + r.mu.Unlock() + <-result.ready + + r.mu.Lock() + result = r.results[cacheKey] + r.mu.Unlock() + + if result.object != nil { + if err := copyClientObject(result.object, obj); err != nil { + return err + } + } + + return result.err + } + + result := requestReadResult{ready: make(chan struct{})} + r.results[cacheKey] = result + r.mu.Unlock() + + err := r.Reader.Get(ctx, key, obj) + if err == nil { + copied, ok := obj.DeepCopyObject().(client.Object) + if !ok { + result.err = fmt.Errorf("deep copy of %T does not implement client.Object", obj) + + r.mu.Lock() + r.results[cacheKey] = result + close(result.ready) + r.mu.Unlock() + + return result.err + } + + result.object = copied + } + + result.err = err + + r.mu.Lock() + r.results[cacheKey] = result + close(result.ready) + r.mu.Unlock() + + return err +} + +func copyClientObject(src, dst client.Object) error { + srcCopy, ok := src.DeepCopyObject().(client.Object) + if !ok { + return fmt.Errorf("deep copy of %T does not implement client.Object", src) + } + + srcValue := reflect.ValueOf(srcCopy) + dstValue := reflect.ValueOf(dst) + + if srcValue.Kind() != reflect.Pointer || + dstValue.Kind() != reflect.Pointer || + srcValue.Type() != dstValue.Type() || + srcValue.IsNil() || + dstValue.IsNil() { + return fmt.Errorf("cannot copy cached %T into %T", src, dst) + } + + dstValue.Elem().Set(srcValue.Elem()) + + return nil +} diff --git a/internal/webhook/utils/request_reader_test.go b/internal/webhook/utils/request_reader_test.go new file mode 100644 index 00000000..dab92393 --- /dev/null +++ b/internal/webhook/utils/request_reader_test.go @@ -0,0 +1,129 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package utils + +import ( + "context" + "sync" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func TestRequestCachingReaderDeduplicatesAndIsolatesGets(t *testing.T) { + t.Parallel() + + scheme := runtime.NewScheme() + if err := corev1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + + base := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(&corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "solar", + Labels: map[string]string{"original": "true"}, + }, + }). + Build() + counting := &countingReader{Reader: base} + reader := NewRequestCachingReader(counting) + + first := &corev1.Namespace{} + if err := reader.Get(context.Background(), client.ObjectKey{Name: "solar"}, first); err != nil { + t.Fatal(err) + } + + first.Labels["mutated"] = "true" + + second := &corev1.Namespace{} + if err := reader.Get(context.Background(), client.ObjectKey{Name: "solar"}, second); err != nil { + t.Fatal(err) + } + + if counting.gets != 1 { + t.Fatalf("underlying gets = %d, want 1", counting.gets) + } + + if second.Labels["original"] != "true" || second.Labels["mutated"] != "" { + t.Fatalf("cached object was not isolated: %#v", second.Labels) + } +} + +func TestRequestCachingReaderSeparatesObjectTypes(t *testing.T) { + t.Parallel() + + scheme := runtime.NewScheme() + if err := corev1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + + base := fake.NewClientBuilder().WithScheme(scheme).Build() + counting := &countingReader{Reader: base} + reader := NewRequestCachingReader(counting) + key := client.ObjectKey{Name: "missing"} + + _ = reader.Get(context.Background(), key, &corev1.Namespace{}) + _ = reader.Get(context.Background(), key, &corev1.ConfigMap{}) + + if counting.gets != 2 { + t.Fatalf("underlying gets = %d, want 2 for different object types", counting.gets) + } +} + +func TestRequestCachingReaderDeduplicatesConcurrentGets(t *testing.T) { + t.Parallel() + + scheme := runtime.NewScheme() + if err := corev1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + + base := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(&corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: "solar"}, + }). + Build() + counting := &countingReader{Reader: base} + reader := NewRequestCachingReader(counting) + + start := make(chan struct{}) + errors := make(chan error, 2) + + var wg sync.WaitGroup + for range 2 { + wg.Add(1) + + go func() { + defer wg.Done() + <-start + + errors <- reader.Get( + context.Background(), + client.ObjectKey{Name: "solar"}, + &corev1.Namespace{}, + ) + }() + } + + close(start) + wg.Wait() + close(errors) + + for err := range errors { + if err != nil { + t.Fatal(err) + } + } + + if counting.gets != 1 { + t.Fatalf("underlying gets = %d, want 1", counting.gets) + } +} diff --git a/internal/webhook/utils/tenant_reader.go b/internal/webhook/utils/tenant_reader.go new file mode 100644 index 00000000..385be23b --- /dev/null +++ b/internal/webhook/utils/tenant_reader.go @@ -0,0 +1,64 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package utils + +import ( + "context" + + "sigs.k8s.io/controller-runtime/pkg/client" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" +) + +type tenantReadResult struct { + tenant *capsulev1beta2.Tenant + err error +} + +type tenantCachingReader struct { + client.Reader + + results map[client.ObjectKey]tenantReadResult +} + +// NewTenantCachingReader deduplicates direct Tenant reads within one admission +// request while preserving the API reader's fresh snapshot for all other +// objects and list operations. +func NewTenantCachingReader(reader client.Reader) client.Reader { + return &tenantCachingReader{ + Reader: reader, + results: make(map[client.ObjectKey]tenantReadResult), + } +} + +func (r *tenantCachingReader) Get( + ctx context.Context, + key client.ObjectKey, + obj client.Object, + opts ...client.GetOption, +) error { + tenant, ok := obj.(*capsulev1beta2.Tenant) + if !ok || len(opts) > 0 { + return r.Reader.Get(ctx, key, obj, opts...) + } + + if result, found := r.results[key]; found { + if result.tenant != nil { + result.tenant.DeepCopyInto(tenant) + } + + return result.err + } + + err := r.Reader.Get(ctx, key, tenant) + result := tenantReadResult{err: err} + + if err == nil { + result.tenant = tenant.DeepCopy() + } + + r.results[key] = result + + return err +} diff --git a/internal/webhook/utils/tenant_reader_test.go b/internal/webhook/utils/tenant_reader_test.go new file mode 100644 index 00000000..6303f7b8 --- /dev/null +++ b/internal/webhook/utils/tenant_reader_test.go @@ -0,0 +1,71 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package utils + +import ( + "context" + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" +) + +type countingReader struct { + client.Reader + + gets int +} + +func (r *countingReader) Get( + ctx context.Context, + key client.ObjectKey, + obj client.Object, + opts ...client.GetOption, +) error { + r.gets++ + + return r.Reader.Get(ctx, key, obj, opts...) +} + +func TestTenantCachingReaderDeduplicatesTenantGets(t *testing.T) { + t.Parallel() + + scheme := runtime.NewScheme() + if err := capsulev1beta2.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + + base := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(&capsulev1beta2.Tenant{ + ObjectMeta: metav1.ObjectMeta{Name: "solar"}, + }). + Build() + counting := &countingReader{Reader: base} + reader := NewTenantCachingReader(counting) + + first := &capsulev1beta2.Tenant{} + if err := reader.Get(context.Background(), client.ObjectKey{Name: "solar"}, first); err != nil { + t.Fatal(err) + } + + first.Labels = map[string]string{"mutated": "true"} + + second := &capsulev1beta2.Tenant{} + if err := reader.Get(context.Background(), client.ObjectKey{Name: "solar"}, second); err != nil { + t.Fatal(err) + } + + if counting.gets != 1 { + t.Fatalf("underlying Tenant gets = %d, want 1", counting.gets) + } + + if second.Labels["mutated"] != "" { + t.Fatal("cached Tenant result was not deep-copied") + } +} diff --git a/pkg/ruleengine/audience.go b/pkg/ruleengine/audience.go index 17c2265e..2ee16d75 100644 --- a/pkg/ruleengine/audience.go +++ b/pkg/ruleengine/audience.go @@ -22,15 +22,22 @@ func FilterNamespaceRulesByAudience( req admission.Request, bodies []*rules.NamespaceRuleBodyNamespace, ) ([]*rules.NamespaceRuleBodyNamespace, error) { - out := make([]*rules.NamespaceRuleBodyNamespace, 0, len(bodies)) + var out []*rules.NamespaceRuleBodyNamespace - for _, body := range bodies { + for i, body := range bodies { if body == nil || len(body.Audience) == 0 { - out = append(out, body) + if out != nil { + out = append(out, body) + } continue } + if out == nil { + out = make([]*rules.NamespaceRuleBodyNamespace, 0, len(bodies)) + out = append(out, bodies[:i]...) + } + matched, err := matchesAudience(cfg, tnt, req, body.Audience) if err != nil { return nil, err @@ -41,6 +48,10 @@ func FilterNamespaceRulesByAudience( } } + if out == nil { + return bodies, nil + } + return out, nil } diff --git a/pkg/runtime/cel/expression.go b/pkg/runtime/cel/expression.go new file mode 100644 index 00000000..f4b6cc3c --- /dev/null +++ b/pkg/runtime/cel/expression.go @@ -0,0 +1,241 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package cel + +import ( + "context" + "fmt" + "strings" + + celgo "github.com/google/cel-go/cel" + "github.com/google/cel-go/common/types" + "github.com/google/cel-go/common/types/traits" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/util/version" + celconfig "k8s.io/apiserver/pkg/apis/cel" + apiservercel "k8s.io/apiserver/pkg/cel" + "k8s.io/apiserver/pkg/cel/environment" +) + +const ( + ObjectVariable = "object" + MaxExpressionLength = 4096 +) + +type ResultType string + +const ( + ResultTypeBoolean ResultType = "boolean" + ResultTypeQuantity ResultType = "quantity" +) + +type Compiler struct { + envSet *environment.EnvSet +} + +type CompiledExpression struct { + expression string + program celgo.Program + resultType ResultType +} + +func NewCompiler() (*Compiler, error) { + base := environment.MustBaseEnvSet(environment.DefaultCompatibilityVersion()) + + envSet, err := base.Extend( + environment.VersionedOptions{ + IntroducedVersion: version.MajorMinor(1, 0), + EnvOptions: []celgo.EnvOption{ + celgo.Variable(ObjectVariable, celgo.DynType), + }, + }, + environment.StrictCostOpt, + ) + if err != nil { + return nil, fmt.Errorf("build Kubernetes CEL environment: %w", err) + } + + return &Compiler{envSet: envSet}, nil +} + +func (c *Compiler) CompileBoolean(expression string, mode environment.Type) (*CompiledExpression, error) { + return c.compile(expression, mode, ResultTypeBoolean) +} + +func (c *Compiler) CompileQuantity(expression string, mode environment.Type) (*CompiledExpression, error) { + return c.compile(expression, mode, ResultTypeQuantity) +} + +func (c *Compiler) compile( + expression string, + mode environment.Type, + resultType ResultType, +) (*CompiledExpression, error) { + if c == nil || c.envSet == nil { + return nil, fmt.Errorf("CEL compiler is nil") + } + + expression = strings.TrimSpace(expression) + if expression == "" { + return nil, fmt.Errorf("CEL expression must not be empty") + } + + if len(expression) > MaxExpressionLength { + return nil, fmt.Errorf("CEL expression exceeds max length of %d", MaxExpressionLength) + } + + env, err := c.envSet.Env(mode) + if err != nil { + return nil, fmt.Errorf("load Kubernetes CEL %s environment: %w", mode, err) + } + + ast, issues := env.Compile(expression) + if issues != nil && issues.Err() != nil { + return nil, fmt.Errorf("compile CEL expression %q: %w", expression, issues.Err()) + } + + if err := validateOutputType(ast.OutputType(), resultType); err != nil { + return nil, fmt.Errorf("compile CEL expression %q: %w", expression, err) + } + + program, err := env.Program( + ast, + celgo.InterruptCheckFrequency(celconfig.CheckFrequency), + ) + if err != nil { + return nil, fmt.Errorf("create CEL program for %q: %w", expression, err) + } + + return &CompiledExpression{ + expression: expression, + program: program, + resultType: resultType, + }, nil +} + +func validateOutputType(output *celgo.Type, resultType ResultType) error { + switch resultType { + case ResultTypeBoolean: + if !output.IsExactType(celgo.BoolType) { + return fmt.Errorf("expression must evaluate to bool, got %s", output) + } + + case ResultTypeQuantity: + quantityListType := celgo.ListType(apiservercel.QuantityType) + if !output.IsExactType(apiservercel.QuantityType) && !output.IsExactType(quantityListType) { + return fmt.Errorf( + "expression must evaluate to kubernetes.Quantity or list, got %s", + output, + ) + } + + default: + return fmt.Errorf("unsupported CEL result type %q", resultType) + } + + return nil +} + +func (c *CompiledExpression) Expression() string { + if c == nil { + return "" + } + + return c.expression +} + +func (c *CompiledExpression) EvaluateBoolean( + ctx context.Context, + object unstructured.Unstructured, +) (bool, error) { + if c == nil || c.program == nil { + return false, fmt.Errorf("compiled CEL expression is nil") + } + + if c.resultType != ResultTypeBoolean { + return false, fmt.Errorf("compiled CEL expression %q does not return bool", c.expression) + } + + value, _, err := c.program.ContextEval(ctx, map[string]any{ + ObjectVariable: object.Object, + }) + if err != nil { + return false, fmt.Errorf("evaluate CEL expression %q: %w", c.expression, err) + } + + result, ok := value.(types.Bool) + if !ok { + return false, fmt.Errorf("CEL expression %q returned %T, expected bool", c.expression, value) + } + + return bool(result), nil +} + +func (c *CompiledExpression) EvaluateQuantity( + ctx context.Context, + object unstructured.Unstructured, +) (resource.Quantity, error) { + if c == nil || c.program == nil { + return resource.Quantity{}, fmt.Errorf("compiled CEL expression is nil") + } + + if c.resultType != ResultTypeQuantity { + return resource.Quantity{}, fmt.Errorf( + "compiled CEL expression %q does not return a quantity", + c.expression, + ) + } + + value, _, err := c.program.ContextEval(ctx, map[string]any{ + ObjectVariable: object.Object, + }) + if err != nil { + return resource.Quantity{}, fmt.Errorf("evaluate CEL expression %q: %w", c.expression, err) + } + + if quantity, ok := value.(apiservercel.Quantity); ok { + if quantity.Quantity == nil { + return resource.Quantity{}, fmt.Errorf("CEL expression %q returned a nil quantity", c.expression) + } + + return quantity.DeepCopy(), nil + } + + list, ok := value.(traits.Lister) + if !ok { + return resource.Quantity{}, fmt.Errorf( + "CEL expression %q returned %T, expected kubernetes.Quantity or list", + c.expression, + value, + ) + } + + total := resource.Quantity{} + count := 0 + iterator := list.Iterator() + + for iterator.HasNext() == types.True { + item := iterator.Next() + + quantity, ok := item.(apiservercel.Quantity) + if !ok || quantity.Quantity == nil { + return resource.Quantity{}, fmt.Errorf( + "CEL expression %q returned a list containing %T, expected kubernetes.Quantity", + c.expression, + item, + ) + } + + total.Add(quantity.DeepCopy()) + + count++ + } + + if count == 0 { + return resource.Quantity{}, fmt.Errorf("CEL expression %q returned an empty quantity list", c.expression) + } + + return total, nil +} diff --git a/pkg/runtime/cel/expression_test.go b/pkg/runtime/cel/expression_test.go new file mode 100644 index 00000000..3cd41e32 --- /dev/null +++ b/pkg/runtime/cel/expression_test.go @@ -0,0 +1,182 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package cel + +import ( + "context" + "strings" + "testing" + + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apiserver/pkg/cel/environment" +) + +func TestCompiledExpressionEvaluateBoolean(t *testing.T) { + t.Parallel() + + compiler, err := NewCompiler() + if err != nil { + t.Fatalf("NewCompiler() error = %v", err) + } + + compiled, err := compiler.CompileBoolean( + `object.spec.containers.exists(c, c.image == "nginx:1.27.0")`, + environment.StoredExpressions, + ) + if err != nil { + t.Fatalf("CompileBoolean() error = %v", err) + } + + object := unstructured.Unstructured{Object: map[string]any{ + "spec": map[string]any{ + "containers": []any{ + map[string]any{"name": "main", "image": "nginx:1.27.0"}, + }, + }, + }} + + matched, err := compiled.EvaluateBoolean(context.Background(), object) + if err != nil { + t.Fatalf("EvaluateBoolean() error = %v", err) + } + if !matched { + t.Fatal("EvaluateBoolean() = false, want true") + } +} + +func TestCompileBooleanRejectsNonBooleanResult(t *testing.T) { + t.Parallel() + + compiler, err := NewCompiler() + if err != nil { + t.Fatalf("NewCompiler() error = %v", err) + } + + _, err = compiler.CompileBoolean(`object.metadata.name`, environment.StoredExpressions) + if err == nil || !strings.Contains(err.Error(), "must evaluate to bool") { + t.Fatalf("CompileBoolean() error = %v, want boolean result type error", err) + } +} + +func TestCompiledExpressionEvaluateSingleQuantity(t *testing.T) { + t.Parallel() + + compiler, err := NewCompiler() + if err != nil { + t.Fatalf("NewCompiler() error = %v", err) + } + + compiled, err := compiler.CompileQuantity( + `quantity(object.spec.resources.requests["storage"])`, + environment.StoredExpressions, + ) + if err != nil { + t.Fatalf("CompileQuantity() error = %v", err) + } + + object := unstructured.Unstructured{Object: map[string]any{ + "spec": map[string]any{ + "resources": map[string]any{ + "requests": map[string]any{"storage": "2Gi"}, + }, + }, + }} + + got, err := compiled.EvaluateQuantity(context.Background(), object) + if err != nil { + t.Fatalf("EvaluateQuantity() error = %v", err) + } + if got.Cmp(resource.MustParse("2Gi")) != 0 { + t.Fatalf("EvaluateQuantity() = %s, want 2Gi", got.String()) + } +} + +func TestCompiledExpressionSumsQuantityList(t *testing.T) { + t.Parallel() + + compiler, err := NewCompiler() + if err != nil { + t.Fatalf("NewCompiler() error = %v", err) + } + + compiled, err := compiler.CompileQuantity( + `object.spec.containers`+ + `.filter(c, has(c.resources) && has(c.resources.requests) && "cpu" in c.resources.requests)`+ + `.map(c, quantity(c.resources.requests["cpu"]))`, + environment.StoredExpressions, + ) + if err != nil { + t.Fatalf("CompileQuantity() error = %v", err) + } + + object := unstructured.Unstructured{Object: map[string]any{ + "spec": map[string]any{ + "containers": []any{ + map[string]any{ + "resources": map[string]any{ + "requests": map[string]any{"cpu": "250m"}, + }, + }, + map[string]any{ + "resources": map[string]any{ + "requests": map[string]any{"cpu": "500m"}, + }, + }, + map[string]any{"name": "no-request"}, + }, + }, + }} + + got, err := compiled.EvaluateQuantity(context.Background(), object) + if err != nil { + t.Fatalf("EvaluateQuantity() error = %v", err) + } + if got.Cmp(resource.MustParse("750m")) != 0 { + t.Fatalf("EvaluateQuantity() = %s, want 750m", got.String()) + } +} + +func TestCompiledExpressionRejectsEmptyQuantityList(t *testing.T) { + t.Parallel() + + compiler, err := NewCompiler() + if err != nil { + t.Fatalf("NewCompiler() error = %v", err) + } + + compiled, err := compiler.CompileQuantity( + `object.spec.values.map(v, quantity(v))`, + environment.StoredExpressions, + ) + if err != nil { + t.Fatalf("CompileQuantity() error = %v", err) + } + + object := unstructured.Unstructured{Object: map[string]any{ + "spec": map[string]any{"values": []any{}}, + }} + + _, err = compiled.EvaluateQuantity(context.Background(), object) + if err == nil || !strings.Contains(err.Error(), "empty quantity list") { + t.Fatalf("EvaluateQuantity() error = %v, want empty list error", err) + } +} + +func TestCompileQuantityRejectsStringResult(t *testing.T) { + t.Parallel() + + compiler, err := NewCompiler() + if err != nil { + t.Fatalf("NewCompiler() error = %v", err) + } + + _, err = compiler.CompileQuantity( + `string(object.spec.value)`, + environment.StoredExpressions, + ) + if err == nil || !strings.Contains(err.Error(), "kubernetes.Quantity") { + t.Fatalf("CompileQuantity() error = %v, want quantity result type error", err) + } +} diff --git a/pkg/runtime/configuration/client.go b/pkg/runtime/configuration/client.go index e040fc8c..6d08f025 100644 --- a/pkg/runtime/configuration/client.go +++ b/pkg/runtime/configuration/client.go @@ -32,6 +32,8 @@ type capsuleConfiguration struct { client client.Client } +const informerConfigurationReadTimeout = 25 * time.Millisecond + func DefaultCapsuleConfiguration() capsulev1beta2.CapsuleConfigurationSpec { d, _ := time.ParseDuration("1h") @@ -62,21 +64,31 @@ func NewCapsuleConfiguration(ctx context.Context, c client.Client, reader client cfg := &capsulev1beta2.CapsuleConfiguration{} key := types.NamespacedName{Name: name} - if err := reader.Get(ctx, key, cfg); err == nil { - return cfg - } else if !apierrors.IsNotFound(err) { - panic(errors.Wrap(err, "cannot retrieve Capsule configuration with name "+name)) - } + // Configuration accessors are used repeatedly within admission + // requests. Prefer the manager's informer-backed client so each + // accessor does not become a live API/etcd read. The direct reader + // remains the authoritative fallback while the informer is + // starting or has not observed a newly created configuration yet. + cacheCtx, cancel := context.WithTimeout(ctx, informerConfigurationReadTimeout) + cacheErr := c.Get(cacheCtx, key, cfg) - err := c.Get(ctx, key, cfg) - if err == nil { + cancel() + + if cacheErr == nil { return cfg } - if !apierrors.IsNotFound(err) { - panic(errors.Wrap(err, "cannot retrieve Capsule configuration with name "+name)) + directErr := reader.Get(ctx, key, cfg) + if directErr == nil { + return cfg } + if !apierrors.IsNotFound(directErr) { + panic(errors.Wrap(directErr, "cannot retrieve Capsule configuration with name "+name)) + } + + // The direct reader is authoritative and reported NotFound. + // Ignore the failed cached read and create the default configuration. cfg = &capsulev1beta2.CapsuleConfiguration{ ObjectMeta: metav1.ObjectMeta{ Name: name, diff --git a/pkg/runtime/configuration/configuration_test.go b/pkg/runtime/configuration/configuration_test.go index d6d9115f..fddc320c 100644 --- a/pkg/runtime/configuration/configuration_test.go +++ b/pkg/runtime/configuration/configuration_test.go @@ -5,6 +5,7 @@ package configuration_test import ( "context" + "errors" "reflect" "testing" "time" @@ -22,6 +23,36 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/fake" ) +type failingConfigurationReader struct { + client.Reader + + gets int +} + +type timeoutConfigurationClient struct { + client.Client +} + +func (c *timeoutConfigurationClient) Get( + context.Context, + client.ObjectKey, + client.Object, + ...client.GetOption, +) error { + return context.DeadlineExceeded +} + +func (r *failingConfigurationReader) Get( + context.Context, + client.ObjectKey, + client.Object, + ...client.GetOption, +) error { + r.gets++ + + return errors.New("direct reader should not be used") +} + func TestDefaultCapsuleConfiguration(t *testing.T) { t.Parallel() @@ -80,6 +111,58 @@ func TestNewCapsuleConfigurationCreatesDefaultWhenMissing(t *testing.T) { } } +func TestNewCapsuleConfigurationCreatesDefaultAfterInformerTimeout(t *testing.T) { + t.Parallel() + + ctx := context.Background() + storage := configurationFakeClient(t) + cachedClient := &timeoutConfigurationClient{Client: storage} + directReader := configurationFakeClient(t) + + cfg := configuration.NewCapsuleConfiguration( + ctx, + cachedClient, + directReader, + &rest.Config{Host: "https://kubernetes.default"}, + "capsule", + ) + + got := cfg.GetConfigObject() + if got.Name != "capsule" { + t.Fatalf("GetConfigObject().Name = %q, want capsule", got.Name) + } + if !reflect.DeepEqual(got.Spec, configuration.DefaultCapsuleConfiguration()) { + t.Fatalf("default configuration = %#v, want %#v", got.Spec, configuration.DefaultCapsuleConfiguration()) + } + + stored := &capsulev1beta2.CapsuleConfiguration{} + if err := storage.Get(ctx, client.ObjectKey{Name: "capsule"}, stored); err != nil { + t.Fatalf("created configuration was not stored: %v", err) + } +} + +func TestCapsuleConfigurationUsesInformerClientBeforeDirectReader(t *testing.T) { + t.Parallel() + + ctx := context.Background() + stored := &capsulev1beta2.CapsuleConfiguration{ + ObjectMeta: metav1.ObjectMeta{Name: "capsule"}, + Spec: capsulev1beta2.CapsuleConfigurationSpec{ + ForceTenantPrefix: true, + }, + } + cl := configurationFakeClient(t, stored) + reader := &failingConfigurationReader{Reader: cl} + cfg := configuration.NewCapsuleConfiguration(ctx, cl, reader, &rest.Config{}, "capsule") + + if !cfg.ForceTenantPrefix() { + t.Fatal("ForceTenantPrefix() = false, want cached configuration value") + } + if reader.gets != 0 { + t.Fatalf("direct reader Get calls = %d, want 0", reader.gets) + } +} + func TestCapsuleConfigurationGetters(t *testing.T) { t.Parallel() diff --git a/pkg/runtime/events/recorder.go b/pkg/runtime/events/recorder.go index 19108332..aac4c380 100644 --- a/pkg/runtime/events/recorder.go +++ b/pkg/runtime/events/recorder.go @@ -22,6 +22,9 @@ import ( const ( ReportingController = "controller.projectcapsule.dev" ReportingInstance = "capsule-admission" + + eventQueueSize = 1024 + eventCreateTimeout = 10 * time.Second ) type EventRecorder interface { @@ -39,6 +42,7 @@ type eventRecorder struct { client client.Client configuration configuration.Configuration log logr.Logger + queue chan *eventsv1.Event } func NewEventRecorder( @@ -47,12 +51,20 @@ func NewEventRecorder( recorder k8sevents.EventRecorder, configuration configuration.Configuration, ) EventRecorder { - return &eventRecorder{ + r := &eventRecorder{ EventRecorder: recorder, client: c, log: log.WithName("event-recorder"), configuration: configuration, } + + if c != nil { + r.queue = make(chan *eventsv1.Event, eventQueueSize) + + go r.run() + } + + return r } func (r *eventRecorder) Emit(ctx context.Context, e LabeledEvent) { @@ -130,18 +142,41 @@ func (r *eventRecorder) Emit(ctx context.Context, e LabeledEvent) { event.Related = &relatedRef } - if err := r.client.Create(ctx, event); err != nil { + select { + case r.queue <- event: + default: r.log.Error( - err, - "cannot emit labeled event", + nil, + "cannot enqueue labeled event: queue is full", "reason", e.Reason(), "action", e.Action(), "type", e.EventType(), "regarding", regardingRef.Name, "namespace", namespace, ) + } +} - return +func (r *eventRecorder) run() { + for event := range r.queue { + ctx, cancel := context.WithTimeout(context.Background(), eventCreateTimeout) + err := r.client.Create(ctx, event) + + cancel() + + if err == nil { + continue + } + + r.log.Error( + err, + "cannot emit labeled event", + "reason", event.Reason, + "action", event.Action, + "type", event.Type, + "regarding", event.Regarding.Name, + "namespace", event.Namespace, + ) } } diff --git a/pkg/runtime/events/recorder_test.go b/pkg/runtime/events/recorder_test.go index 9890d1e6..32ef7266 100644 --- a/pkg/runtime/events/recorder_test.go +++ b/pkg/runtime/events/recorder_test.go @@ -6,6 +6,7 @@ package events_test import ( "context" "testing" + "time" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" "github.com/projectcapsule/capsule/pkg/api/meta" @@ -97,11 +98,24 @@ func TestLabeledEventEmitCreatesEvent(t *testing.T) { ).WithLabels(map[string]string{"capsule.clastix.io/test": "true"}).Emit(ctx) var eventList eventsv1.EventList - if err := cl.List(ctx, &eventList, client.InNamespace("audit")); err != nil { - t.Fatalf("listing emitted events: %v", err) - } - if len(eventList.Items) != 1 { - t.Fatalf("emitted events = %d, want 1", len(eventList.Items)) + deadline := time.Now().Add(time.Second) + + for { + eventList.Items = nil + + if err := cl.List(ctx, &eventList, client.InNamespace("audit")); err != nil { + t.Fatalf("listing emitted events: %v", err) + } + + if len(eventList.Items) == 1 { + break + } + + if time.Now().After(deadline) { + t.Fatalf("emitted events = %d, want 1", len(eventList.Items)) + } + + time.Sleep(time.Millisecond) } got := eventList.Items[0] diff --git a/pkg/runtime/handlers/typed_tenant_object.go b/pkg/runtime/handlers/typed_tenant_object.go index 055ba4e2..f867bcea 100644 --- a/pkg/runtime/handlers/typed_tenant_object.go +++ b/pkg/runtime/handlers/typed_tenant_object.go @@ -1,7 +1,6 @@ // Copyright 2020-2026 Project Capsule Authors // SPDX-License-Identifier: Apache-2.0 -//nolint:dupl package handlers import ( @@ -22,12 +21,32 @@ type TypedHandlerWithTenant[T client.Object] interface { } type TypedTenantHandler[T client.Object] struct { - Factory NewObjectFunc[T] - Handlers []TypedHandlerWithTenant[T] + Factory NewObjectFunc[T] + Handlers []TypedHandlerWithTenant[T] + Predicate func(req admission.Request, obj T, oldObj T) bool } func (h *TypedTenantHandler[T]) OnCreate(c client.Client, reader client.Reader, decoder admission.Decoder, recorder events.EventRecorder) Func { return func(ctx context.Context, req admission.Request) *admission.Response { + var ( + obj T + decoded bool + ) + + if h.Predicate != nil { + obj = h.Factory() + if err := decoder.Decode(req, obj); err != nil { + return ErroredResponse(err) + } + + decoded = true + + var oldObj T + if !h.Predicate(req, obj, oldObj) { + return nil + } + } + tnt, err := h.resolveTenant(ctx, reader, req) if err != nil { return ErroredResponse(err) @@ -37,9 +56,11 @@ func (h *TypedTenantHandler[T]) OnCreate(c client.Client, reader client.Reader, return nil } - obj := h.Factory() - if err := decoder.Decode(req, obj); err != nil { - return ErroredResponse(err) + if !decoded { + obj = h.Factory() + if err := decoder.Decode(req, obj); err != nil { + return ErroredResponse(err) + } } for _, hndl := range h.Handlers { @@ -54,6 +75,30 @@ func (h *TypedTenantHandler[T]) OnCreate(c client.Client, reader client.Reader, func (h *TypedTenantHandler[T]) OnUpdate(c client.Client, reader client.Reader, decoder admission.Decoder, recorder events.EventRecorder) Func { return func(ctx context.Context, req admission.Request) *admission.Response { + var ( + newObj T + oldObj T + decoded bool + ) + + if h.Predicate != nil { + newObj = h.Factory() + if err := decoder.Decode(req, newObj); err != nil { + return ErroredResponse(err) + } + + oldObj = h.Factory() + if err := decoder.DecodeRaw(req.OldObject, oldObj); err != nil { + return ErroredResponse(err) + } + + decoded = true + + if !h.Predicate(req, newObj, oldObj) { + return nil + } + } + tnt, err := h.resolveTenant(ctx, reader, req) if err != nil { return ErroredResponse(err) @@ -63,14 +108,16 @@ func (h *TypedTenantHandler[T]) OnUpdate(c client.Client, reader client.Reader, return nil } - newObj := h.Factory() - if err := decoder.Decode(req, newObj); err != nil { - return ErroredResponse(err) - } + if !decoded { + newObj = h.Factory() + if err := decoder.Decode(req, newObj); err != nil { + return ErroredResponse(err) + } - oldObj := h.Factory() - if err := decoder.DecodeRaw(req.OldObject, oldObj); err != nil { - return ErroredResponse(err) + oldObj = h.Factory() + if err := decoder.DecodeRaw(req.OldObject, oldObj); err != nil { + return ErroredResponse(err) + } } for _, hndl := range h.Handlers { diff --git a/pkg/runtime/handlers/typed_tenant_ruleset.go b/pkg/runtime/handlers/typed_tenant_ruleset.go index ca59cd09..9f670ed8 100644 --- a/pkg/runtime/handlers/typed_tenant_ruleset.go +++ b/pkg/runtime/handlers/typed_tenant_ruleset.go @@ -60,6 +60,12 @@ type TypedTenantWithRulesetHandler[T client.Object] struct { Configuration configuration.Configuration } +type rulesetReadResult struct { + rules []*rules.NamespaceRuleBodyNamespace + found bool + err error +} + func (h *TypedTenantWithRulesetHandler[T]) OnCreate( c client.Client, reader client.Reader, @@ -67,6 +73,12 @@ func (h *TypedTenantWithRulesetHandler[T]) OnCreate( recorder events.EventRecorder, ) Func { return func(ctx context.Context, req admission.Request) *admission.Response { + if req.Namespace == "" { + return nil + } + + rulesetResult := h.readRulesetAsync(ctx, reader, req.Namespace) + tnt, err := h.resolveTenant(ctx, reader, req) if err != nil { return ErroredResponse(err) @@ -81,7 +93,7 @@ func (h *TypedTenantWithRulesetHandler[T]) OnCreate( return ErroredResponse(err) } - ruleBlocks, err := h.resolveRuleset(ctx, c, reader, req, req.Namespace, tnt) + ruleBlocks, err := h.resolveRuleset(ctx, c, req.Namespace, tnt, <-rulesetResult) if err != nil { return ErroredResponse(err) } @@ -108,7 +120,13 @@ func (h *TypedTenantWithRulesetHandler[T]) OnUpdate( recorder events.EventRecorder, ) Func { return func(ctx context.Context, req admission.Request) *admission.Response { - tnt, err := h.resolveTenant(ctx, c, req) + if req.Namespace == "" { + return nil + } + + rulesetResult := h.readRulesetAsync(ctx, reader, req.Namespace) + + tnt, err := h.resolveTenant(ctx, reader, req) if err != nil { return ErroredResponse(err) } @@ -127,7 +145,7 @@ func (h *TypedTenantWithRulesetHandler[T]) OnUpdate( return ErroredResponse(err) } - ruleBlocks, err := h.resolveRuleset(ctx, c, reader, req, req.Namespace, tnt) + ruleBlocks, err := h.resolveRuleset(ctx, c, req.Namespace, tnt, <-rulesetResult) if err != nil { return ErroredResponse(err) } @@ -170,26 +188,53 @@ func (h *TypedTenantWithRulesetHandler[T]) resolveTenant( return tenant.GetTenantByNamespace(ctx, c, req.Namespace) } +func (h *TypedTenantWithRulesetHandler[T]) readRulesetAsync( + ctx context.Context, + reader client.Reader, + namespace string, +) <-chan rulesetReadResult { + result := make(chan rulesetReadResult, 1) + + go func() { + rs := &capsulev1beta2.RuleStatus{} + key := types.NamespacedName{ + Namespace: namespace, + Name: meta.NameForManagedRuleStatus(), + } + + err := reader.Get(ctx, key, rs) + + switch { + case err == nil: + result <- rulesetReadResult{ + rules: rs.Status.Rules, + found: true, + } + case apierrors.IsNotFound(err): + result <- rulesetReadResult{} + default: + result <- rulesetReadResult{err: err} + } + }() + + return result +} + // Resolve the corresponding managed ruleset for this namespace. // If not yet present, try to calculate it. func (h *TypedTenantWithRulesetHandler[T]) resolveRuleset( ctx context.Context, c client.Client, - reader client.Reader, - req admission.Request, namespace string, tnt *capsulev1beta2.Tenant, + result rulesetReadResult, ) ([]*rules.NamespaceRuleBodyNamespace, error) { - rs := &capsulev1beta2.RuleStatus{} - key := types.NamespacedName{ - Namespace: namespace, - Name: meta.NameForManagedRuleStatus(), + if result.err != nil { + return nil, result.err } - if err := reader.Get(ctx, key, rs); err == nil { - return rs.Status.Rules, nil - } else if !apierrors.IsNotFound(err) { - return nil, err + if result.found { + return result.rules, nil } ns := &corev1.Namespace{} diff --git a/pkg/runtime/handlers/typed_tenant_user_object.go b/pkg/runtime/handlers/typed_tenant_user_object.go index cc3f12e4..521dc70e 100644 --- a/pkg/runtime/handlers/typed_tenant_user_object.go +++ b/pkg/runtime/handlers/typed_tenant_user_object.go @@ -1,7 +1,6 @@ // Copyright 2020-2026 Project Capsule Authors // SPDX-License-Identifier: Apache-2.0 -//nolint:dupl package handlers import ( @@ -27,11 +26,28 @@ type TypedTenantWithUserHandler[T client.Object] struct { Factory NewObjectFunc[T] Handlers []TypedHandlerWithTenantUser[T] Configuration configuration.Configuration + Predicate func(req admission.Request, obj T, oldObj T) bool + UserResolver func( + ctx context.Context, + c client.Client, + req admission.Request, + cfg configuration.Configuration, + ) users.AdmissionUser } func (h *TypedTenantWithUserHandler[T]) OnCreate(c client.Client, reader client.Reader, decoder admission.Decoder, recorder events.EventRecorder) Func { return func(ctx context.Context, req admission.Request) *admission.Response { - user := ResolveAdmissionUser(ctx, c, req, h.Configuration) + obj := h.Factory() + if err := decoder.Decode(req, obj); err != nil { + return ErroredResponse(err) + } + + if h.Predicate != nil { + var oldObj T + if !h.Predicate(req, obj, oldObj) { + return nil + } + } tnt, err := h.resolveTenant(ctx, reader, req) if err != nil { @@ -42,10 +58,7 @@ func (h *TypedTenantWithUserHandler[T]) OnCreate(c client.Client, reader client. return nil } - obj := h.Factory() - if err := decoder.Decode(req, obj); err != nil { - return ErroredResponse(err) - } + user := h.resolveUser(ctx, c, req) for _, hndl := range h.Handlers { if response := hndl.OnCreate(c, reader, user, obj, decoder, recorder, tnt)(ctx, req); response != nil { @@ -59,7 +72,19 @@ func (h *TypedTenantWithUserHandler[T]) OnCreate(c client.Client, reader client. func (h *TypedTenantWithUserHandler[T]) OnUpdate(c client.Client, reader client.Reader, decoder admission.Decoder, recorder events.EventRecorder) Func { return func(ctx context.Context, req admission.Request) *admission.Response { - user := ResolveAdmissionUser(ctx, c, req, h.Configuration) + newObj := h.Factory() + if err := decoder.Decode(req, newObj); err != nil { + return ErroredResponse(err) + } + + oldObj := h.Factory() + if err := decoder.DecodeRaw(req.OldObject, oldObj); err != nil { + return ErroredResponse(err) + } + + if h.Predicate != nil && !h.Predicate(req, newObj, oldObj) { + return nil + } tnt, err := h.resolveTenant(ctx, reader, req) if err != nil { @@ -70,15 +95,7 @@ func (h *TypedTenantWithUserHandler[T]) OnUpdate(c client.Client, reader client. return nil } - newObj := h.Factory() - if err := decoder.Decode(req, newObj); err != nil { - return ErroredResponse(err) - } - - oldObj := h.Factory() - if err := decoder.DecodeRaw(req.OldObject, oldObj); err != nil { - return ErroredResponse(err) - } + user := h.resolveUser(ctx, c, req) for _, hndl := range h.Handlers { if response := hndl.OnUpdate(c, reader, user, oldObj, newObj, decoder, recorder, tnt)(ctx, req); response != nil { @@ -92,8 +109,6 @@ func (h *TypedTenantWithUserHandler[T]) OnUpdate(c client.Client, reader client. func (h *TypedTenantWithUserHandler[T]) OnDelete(c client.Client, reader client.Reader, decoder admission.Decoder, recorder events.EventRecorder) Func { return func(ctx context.Context, req admission.Request) *admission.Response { - user := ResolveAdmissionUser(ctx, c, req, h.Configuration) - tnt, err := h.resolveTenant(ctx, reader, req) if err != nil { return ErroredResponse(err) @@ -108,6 +123,8 @@ func (h *TypedTenantWithUserHandler[T]) OnDelete(c client.Client, reader client. return ErroredResponse(err) } + user := h.resolveUser(ctx, c, req) + for _, hndl := range h.Handlers { if response := hndl.OnDelete(c, reader, user, obj, decoder, recorder, tnt)(ctx, req); response != nil { return response @@ -118,6 +135,18 @@ func (h *TypedTenantWithUserHandler[T]) OnDelete(c client.Client, reader client. } } +func (h *TypedTenantWithUserHandler[T]) resolveUser( + ctx context.Context, + c client.Client, + req admission.Request, +) users.AdmissionUser { + if h.UserResolver != nil { + return h.UserResolver(ctx, c, req, h.Configuration) + } + + return ResolveAdmissionUser(ctx, c, req, h.Configuration) +} + func (h *TypedTenantWithUserHandler[T]) resolveTenant(ctx context.Context, c client.Reader, req admission.Request) (*capsulev1beta2.Tenant, error) { if req.Namespace == "" { return nil, nil diff --git a/pkg/runtime/predicates/controller_work.go b/pkg/runtime/predicates/controller_work.go index ab7ae52f..9042510e 100644 --- a/pkg/runtime/predicates/controller_work.go +++ b/pkg/runtime/predicates/controller_work.go @@ -92,7 +92,18 @@ func (NamespaceTenantStateChangedPredicate) Update(e event.UpdateEvent) bool { type QuantityLedgerWorkChangedPredicate struct{ predicate.Funcs } -func (QuantityLedgerWorkChangedPredicate) Create(event.CreateEvent) bool { return true } +func (QuantityLedgerWorkChangedPredicate) Create(e event.CreateEvent) bool { + ledger, ok := e.Object.(*capsulev1beta2.QuantityLedger) + if !ok { + return false + } + + // A ledger can be created and receive admission reservations before the + // informer observes its first Add event. In that case the Add is the only + // work notification; filtering it would strand the reservations forever. + return quantityLedgerHasWork(ledger) +} + func (QuantityLedgerWorkChangedPredicate) Delete(event.DeleteEvent) bool { return true } func (QuantityLedgerWorkChangedPredicate) Generic(event.GenericEvent) bool { return false } func (QuantityLedgerWorkChangedPredicate) Update(e event.UpdateEvent) bool { @@ -104,9 +115,67 @@ func (QuantityLedgerWorkChangedPredicate) Update(e event.UpdateEvent) bool { return false } - return oldLedger.Generation != newLedger.Generation || - !reflect.DeepEqual(oldLedger.Status.Reservations, newLedger.Status.Reservations) || - !reflect.DeepEqual(oldLedger.Status.PendingDeletes, newLedger.Status.PendingDeletes) + if oldLedger.Generation != newLedger.Generation { + return true + } + + // Informers may coalesce an intermediate settled state. An update can + // therefore replace reservations with pending deletes (or append more + // reservations) while both observed snapshots contain work. Admit every + // newly introduced or changed work item so it cannot be stranded. Pure + // settlement/removal updates remain filtered. + return quantityLedgerIntroducesWork(oldLedger, newLedger) +} + +func quantityLedgerHasWork(ledger *capsulev1beta2.QuantityLedger) bool { + return len(ledger.Status.Reservations) > 0 || + len(ledger.Status.PendingDeletes) > 0 +} + +func quantityLedgerIntroducesWork(oldLedger, newLedger *capsulev1beta2.QuantityLedger) bool { + matchedReservations := make([]bool, len(oldLedger.Status.Reservations)) + + for _, newReservation := range newLedger.Status.Reservations { + matched := false + + for i, oldReservation := range oldLedger.Status.Reservations { + if matchedReservations[i] || !reflect.DeepEqual(oldReservation, newReservation) { + continue + } + + matchedReservations[i] = true + matched = true + + break + } + + if !matched { + return true + } + } + + matchedPendingDeletes := make([]bool, len(oldLedger.Status.PendingDeletes)) + + for _, newPendingDelete := range newLedger.Status.PendingDeletes { + matched := false + + for i, oldPendingDelete := range oldLedger.Status.PendingDeletes { + if matchedPendingDeletes[i] || !reflect.DeepEqual(oldPendingDelete, newPendingDelete) { + continue + } + + matchedPendingDeletes[i] = true + matched = true + + break + } + + if !matched { + return true + } + } + + return false } type ProvisionerSubjectsChangedPredicate struct{ predicate.Funcs } diff --git a/pkg/runtime/predicates/performance_predicates_test.go b/pkg/runtime/predicates/performance_predicates_test.go index 29d20c9b..262e5d80 100644 --- a/pkg/runtime/predicates/performance_predicates_test.go +++ b/pkg/runtime/predicates/performance_predicates_test.go @@ -204,6 +204,19 @@ func TestQuantityLedgerWorkChangedPredicate(t *testing.T) { t.Parallel() p := predicates.QuantityLedgerWorkChangedPredicate{} + if p.Create(event.CreateEvent{Object: &capsulev1beta2.QuantityLedger{}}) { + t.Fatal("settled ledger creation must be filtered; the owning quota create already reconciles it") + } + + createdWithWork := &capsulev1beta2.QuantityLedger{ + Status: capsulev1beta2.QuantityLedgerStatus{ + Reservations: []capsulev1beta2.QuantityLedgerReservation{{ID: "request"}}, + }, + } + if !p.Create(event.CreateEvent{Object: createdWithWork}) { + t.Fatal("ledger first observed with work must trigger reconciliation") + } + oldLedger := &capsulev1beta2.QuantityLedger{} derived := oldLedger.DeepCopy() derived.Status.Allocated = resource.MustParse("1") @@ -216,6 +229,41 @@ func TestQuantityLedgerWorkChangedPredicate(t *testing.T) { if !p.Update(event.UpdateEvent{ObjectOld: oldLedger, ObjectNew: work}) { t.Fatal("reservation update must be admitted") } + + moreWork := work.DeepCopy() + moreWork.Status.Reservations = append( + moreWork.Status.Reservations, + capsulev1beta2.QuantityLedgerReservation{ID: "request-2"}, + ) + if !p.Update(event.UpdateEvent{ObjectOld: work, ObjectNew: moreWork}) { + t.Fatal("additional work must be admitted in case the informer coalesced an intermediate settled state") + } + + replacedWork := oldLedger.DeepCopy() + replacedWork.Status.PendingDeletes = []capsulev1beta2.QuantityLedgerPendingDelete{{ + ID: "delete-request", + }} + if !p.Update(event.UpdateEvent{ObjectOld: work, ObjectNew: replacedWork}) { + t.Fatal("replacement work must be admitted when an intermediate settled state is not observed") + } + + settled := moreWork.DeepCopy() + settled.Status.Reservations = nil + if p.Update(event.UpdateEvent{ObjectOld: moreWork, ObjectNew: settled}) { + t.Fatal("controller settlement updates must be filtered") + } + + partiallySettled := moreWork.DeepCopy() + partiallySettled.Status.Reservations = partiallySettled.Status.Reservations[:1] + if p.Update(event.UpdateEvent{ObjectOld: moreWork, ObjectNew: partiallySettled}) { + t.Fatal("partial controller settlement updates must be filtered") + } + + unchangedWork := work.DeepCopy() + unchangedWork.Status.Allocated = resource.MustParse("1") + if p.Update(event.UpdateEvent{ObjectOld: work, ObjectNew: unchangedWork}) { + t.Fatal("derived updates with unchanged work must be filtered") + } } func TestResourcePoolNamespacesChangedPredicate(t *testing.T) { diff --git a/pkg/runtime/quota/custom_quota.go b/pkg/runtime/quota/custom_quota.go index 7671c339..fb3d154b 100644 --- a/pkg/runtime/quota/custom_quota.go +++ b/pkg/runtime/quota/custom_quota.go @@ -6,6 +6,7 @@ package quota import ( "k8s.io/apimachinery/pkg/api/resource" + celruntime "github.com/projectcapsule/capsule/pkg/runtime/cel" "github.com/projectcapsule/capsule/pkg/runtime/jsonpath" ) @@ -16,6 +17,8 @@ type MatchedQuota struct { Namespace string Path string CompiledPath *jsonpath.CompiledJSONPath + CEL string + CompiledCEL *celruntime.CompiledExpression Operation Operation Limit resource.Quantity Used resource.Quantity diff --git a/pkg/runtime/selectors/fields.go b/pkg/runtime/selectors/fields.go index 50778c0e..635c3e1b 100644 --- a/pkg/runtime/selectors/fields.go +++ b/pkg/runtime/selectors/fields.go @@ -7,6 +7,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" + celruntime "github.com/projectcapsule/capsule/pkg/runtime/cel" "github.com/projectcapsule/capsule/pkg/runtime/jsonpath" ) @@ -27,11 +28,22 @@ type SelectorWithFields struct { // All must evaluate to true for this selector to match. // +optional FieldSelectors []string `json:"fieldSelectors,omitempty"` + + // Additional CEL expressions evaluated against the selected object. + // The object is available as "object". + // All must evaluate to true for this selector to match. + // CEL expressions and fieldSelectors may be used together. + // +kubebuilder:validation:MaxItems=64 + // +kubebuilder:validation:items:MinLength=1 + // +kubebuilder:validation:items:MaxLength=4096 + // +optional + CELExpressions []string `json:"celExpressions,omitempty"` } type CompiledSelectorWithFields struct { LabelSelector labels.Selector FieldMatchers []CompiledFieldSelector + CELMatchers []*celruntime.CompiledExpression } type CompiledFieldSelector struct { diff --git a/pkg/runtime/selectors/namespaced_selectors.go b/pkg/runtime/selectors/namespaced_selectors.go index 69f97101..a789d3d2 100644 --- a/pkg/runtime/selectors/namespaced_selectors.go +++ b/pkg/runtime/selectors/namespaced_selectors.go @@ -62,16 +62,42 @@ func GetNamespacesMatchingSelectors( return nil, nil } - byName := make(map[string]corev1.Namespace) + compiled := make([]labels.Selector, 0, len(namespaceSelector)) for _, selector := range namespaceSelector { - matches, err := selector.GetMatchingNamespaces(ctx, c) - if err != nil { - return nil, err + if selector.LabelSelector == nil { + continue } - for _, ns := range matches { - byName[ns.Name] = ns + match, err := metav1.LabelSelectorAsSelector(selector.LabelSelector) + if err != nil { + return nil, fmt.Errorf("invalid namespace selector: %w", err) + } + + compiled = append(compiled, match) + } + + if len(compiled) == 0 { + return nil, nil + } + + namespaceList := &corev1.NamespaceList{} + if err := c.List(ctx, namespaceList); err != nil { + return nil, fmt.Errorf("failed to list namespaces: %w", err) + } + + byName := make(map[string]corev1.Namespace) + + for i := range namespaceList.Items { + ns := namespaceList.Items[i] + nsLabels := labels.Set(ns.Labels) + + for _, match := range compiled { + if match.Matches(nsLabels) { + byName[ns.Name] = ns + + break + } } } diff --git a/pkg/runtime/selectors/selectors_test.go b/pkg/runtime/selectors/selectors_test.go index 588f7c6b..7052c821 100644 --- a/pkg/runtime/selectors/selectors_test.go +++ b/pkg/runtime/selectors/selectors_test.go @@ -183,8 +183,9 @@ func TestGetNamespacesMatchingSelectors(t *testing.T) { namespace("alpha", map[string]string{"team": "a", "region": "us"}), namespace("beta", map[string]string{"team": "b", "region": "eu"}), ) + counting := &listCountingReader{Reader: cl} - got, err := selectors.GetNamespacesMatchingSelectors(ctx, cl, []selectors.NamespaceSelector{ + got, err := selectors.GetNamespacesMatchingSelectors(ctx, counting, []selectors.NamespaceSelector{ {LabelSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"team": "a"}}}, {LabelSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"region": "eu"}}}, }) @@ -195,6 +196,9 @@ func TestGetNamespacesMatchingSelectors(t *testing.T) { if names := namespaceNames(got); !reflect.DeepEqual(names, []string{"alpha", "beta", "zeta"}) { t.Fatalf("GetNamespacesMatchingSelectors() names = %#v, want sorted unique names", names) } + if counting.listCalls != 1 { + t.Fatalf("GetNamespacesMatchingSelectors() list calls = %d, want 1", counting.listCalls) + } gotNames, err := selectors.GetNamespacesMatchingSelectorsStrings(ctx, cl, []selectors.NamespaceSelector{ {LabelSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"team": "a"}}}, @@ -215,6 +219,22 @@ func TestGetNamespacesMatchingSelectors(t *testing.T) { } } +type listCountingReader struct { + client.Reader + + listCalls int +} + +func (r *listCountingReader) List( + ctx context.Context, + list client.ObjectList, + opts ...client.ListOption, +) error { + r.listCalls++ + + return r.Reader.List(ctx, list, opts...) +} + func TestSelectorWithNamespaceSelectorMatchObjects(t *testing.T) { t.Parallel() diff --git a/pkg/runtime/selectors/zz_generated.deepcopy.go b/pkg/runtime/selectors/zz_generated.deepcopy.go index 6138f3c7..1667bc4f 100644 --- a/pkg/runtime/selectors/zz_generated.deepcopy.go +++ b/pkg/runtime/selectors/zz_generated.deepcopy.go @@ -44,6 +44,11 @@ func (in *SelectorWithFields) DeepCopyInto(out *SelectorWithFields) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.CELExpressions != nil { + in, out := &in.CELExpressions, &out.CELExpressions + *out = make([]string, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SelectorWithFields.