diff --git a/charts/capsule/README.md b/charts/capsule/README.md index 921fdb5e..bcfe7769 100644 --- a/charts/capsule/README.md +++ b/charts/capsule/README.md @@ -87,8 +87,9 @@ The following Values have changed key or Value: | ports | list | `[]` | Set additional ports for the deployment | | priorityClassName | string | `""` | Set the priority class name of the Capsule pod | | proxy.enabled | bool | `false` | Enable Installation of Capsule Proxy | -| rbac.resourcepoolclaims | object | `{"create":false,"labels":{"rbac.authorization.k8s.io/aggregate-to-admin":"true"}}` | Allow the creation of ResourcePoolClaims | -| rbac.resources | object | `{"create":false,"labels":{"rbac.authorization.k8s.io/aggregate-to-admin":"true"}}` | Allow the creation of TenantResources | +| rbac.customquotas | object | `{"create":false,"labels":{}}` | Allow the creation of CustomQuotas | +| rbac.resourcepoolclaims | object | `{"create":false,"labels":{}}` | Allow the creation of ResourcePoolClaims | +| rbac.resources | object | `{"create":false,"labels":{}}` | Allow the creation of TenantResources | | replicaCount | int | `1` | Set the replica count for capsule pod | | securityContext | object | `{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]},"enabled":true,"readOnlyRootFilesystem":true}` | Set the securityContext for the Capsule container | | serviceAccount.annotations | object | `{}` | Annotations to add to the service account. | @@ -283,7 +284,7 @@ The following Values have changed key or Value: | webhooks.hooks.gateways.objectSelector | object | `{}` | [ObjectSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-objectselector) | | webhooks.hooks.gateways.opts | object | `{}` | Capsule Hook Options | | webhooks.hooks.gateways.reinvocationPolicy | string | `"Never"` | [ReinvocationPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#reinvocation-policy) | -| webhooks.hooks.generic | object | `{"enabled":true,"failurePolicy":"Fail","matchConditions":[{"expression":"request.resource.resource != \"events\"","name":"ignore-events"}],"matchPolicy":"Equivalent","namespaceSelector":{"matchExpressions":[{"key":"capsule.clastix.io/tenant","operator":"Exists"}]},"objectSelector":{},"opts":{},"reinvocationPolicy":"Never","rules":[{"apiGroups":["*"],"apiVersions":["*"],"operations":["CREATE","UPDATE"],"resources":["*/*"],"scope":"Namespaced"}]}` | Generic Rules API, including the rule-engine validation for Pods and Services | +| webhooks.hooks.generic | object | `{"enabled":true,"failurePolicy":"Fail","matchConditions":[{"expression":"request.resource.resource != \"events\"","name":"ignore-events"}],"matchPolicy":"Equivalent","namespaceSelector":{"matchExpressions":[{"key":"capsule.clastix.io/tenant","operator":"Exists"}]},"objectSelector":{},"opts":{},"reinvocationPolicy":"IfNeeded","rules":[{"apiGroups":["*"],"apiVersions":["*"],"operations":["CREATE","UPDATE"],"resources":["*/*"],"scope":"Namespaced"}]}` | Generic Rules API, including the rule-engine validation for Pods and Services | | webhooks.hooks.generic.enabled | bool | `true` | Enable the Hook | | webhooks.hooks.generic.failurePolicy | string | `"Fail"` | [FailurePolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#failure-policy) | | webhooks.hooks.generic.matchConditions | list | `[{"expression":"request.resource.resource != \"events\"","name":"ignore-events"}]` | [MatchConditions](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) | @@ -291,7 +292,7 @@ The following Values have changed key or Value: | webhooks.hooks.generic.namespaceSelector | object | `{"matchExpressions":[{"key":"capsule.clastix.io/tenant","operator":"Exists"}]}` | [NamespaceSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-namespaceselector) | | webhooks.hooks.generic.objectSelector | object | `{}` | [ObjectSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-objectselector) | | webhooks.hooks.generic.opts | object | `{}` | Capsule Hook Options | -| webhooks.hooks.generic.reinvocationPolicy | string | `"Never"` | [ReinvocationPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#reinvocation-policy) | +| webhooks.hooks.generic.reinvocationPolicy | string | `"IfNeeded"` | [ReinvocationPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#reinvocation-policy) | | webhooks.hooks.generic.rules | list | `[{"apiGroups":["*"],"apiVersions":["*"],"operations":["CREATE","UPDATE"],"resources":["*/*"],"scope":"Namespaced"}]` | [Rules](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-rules) | | webhooks.hooks.globalcustomquotas.enabled | bool | `true` | Enable the Hook | | webhooks.hooks.globalcustomquotas.failurePolicy | string | `"Fail"` | [FailurePolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#failure-policy) | @@ -324,7 +325,7 @@ The following Values have changed key or Value: | webhooks.hooks.managed.namespaceSelector | object | `{}` | [NamespaceSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-namespaceselector) | | webhooks.hooks.managed.objectSelector | object | `{"matchExpressions":[{"key":"projectcapsule.dev/managed-by","operator":"In","values":["controller"]}]}` | [ObjectSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-objectselector) | | webhooks.hooks.managed.opts | object | `{}` | Capsule Hook Options | -| webhooks.hooks.managed.rules | list | `[{"apiGroups":["*"],"apiVersions":["*"],"operations":["CREATE","UPDATE","DELETE"],"resources":["*"],"scope":"Namespaced"}]` | [Rules](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-rules) | +| webhooks.hooks.managed.rules | list | `[{"apiGroups":["*"],"apiVersions":["*"],"operations":["CREATE","UPDATE","DELETE"],"resources":["*"],"scope":"Namespaced"},{"apiGroups":["capsule.clastix.io"],"apiVersions":["v1beta2"],"operations":["CREATE","UPDATE","DELETE"],"resources":["globalresourcequotas"],"scope":"Cluster"}]` | [Rules](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-rules) | | webhooks.hooks.metadata | object | `{"enabled":true,"failurePolicy":"Ignore","matchConditions":[{"expression":"!has(request.subResource) || request.subResource == \"\"","name":"ignore-subresources"},{"expression":"request.resource.resource != \"events\"","name":"ignore-events"}],"matchPolicy":"Equivalent","namespaceSelector":{"matchExpressions":[{"key":"capsule.clastix.io/tenant","operator":"Exists"}]},"objectSelector":{},"opts":{},"reinvocationPolicy":"Never","rules":[{"apiGroups":["*"],"apiVersions":["*"],"operations":["CREATE","UPDATE"],"resources":["*"],"scope":"Namespaced"}]}` | Additional Metadata webhook | | webhooks.hooks.metadata.enabled | bool | `true` | Enable the Hook | | webhooks.hooks.metadata.failurePolicy | string | `"Ignore"` | [FailurePolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#failure-policy) | diff --git a/charts/capsule/crds/capsule.clastix.io_rulestatuses.yaml b/charts/capsule/crds/capsule.clastix.io_rulestatuses.yaml index c05b8a0a..ac5c20e8 100644 --- a/charts/capsule/crds/capsule.clastix.io_rulestatuses.yaml +++ b/charts/capsule/crds/capsule.clastix.io_rulestatuses.yaml @@ -431,6 +431,95 @@ spec: - message: at least one of exact or exp must be set rule: has(self.exact) || has(self.exp) type: array + resources: + description: |- + Resources defines mutation and enforcement policies for Pod and container + resource requests and limits. The workload targets select where the + policies apply. With no targets, resource policies apply to all compatible + locations: Pod-level resources, regular containers, and init containers. + Resource names unsupported at Pod level still apply to compatible container + locations. + Mutation is applied when a Pod is created. Remove and MatchRequest manage + explicit values, Default fills an absent value, and Ratio fills an absent + limit from its request. An explicit Ratio violation is then handled by the + enclosing allow, deny, or audit action. + properties: + limits: + additionalProperties: + description: WorkloadResourceLimitPolicy defines how + a resource limit is mutated and enforced. + properties: + policy: + description: |- + Policy selects how the limit is handled: Preserve leaves it unchanged, + Default fills an absent limit, Remove deletes it, MatchRequest manages it + to equal the request, and Ratio defaults an absent limit and enforces the + maximum multiplier against explicitly supplied limits. + enum: + - Preserve + - Default + - Remove + - MatchRequest + - Ratio + type: string + value: + anyOf: + - type: integer + - type: string + description: |- + Value is the quantity applied by Default or the maximum limit-to-request + multiplier applied by Ratio. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - policy + type: object + x-kubernetes-validations: + - message: value must be set only for the Default + and Ratio policies + rule: 'self.policy == ''Default'' || self.policy + == ''Ratio'' ? has(self.value) : !has(self.value)' + description: Limits defines policies for resource limits. + minProperties: 1 + type: object + requests: + additionalProperties: + description: WorkloadResourceRequestPolicy defines + how a resource request is mutated. + properties: + policy: + description: |- + Policy selects how the request is handled: Preserve leaves it unchanged, + Default fills an absent request, and Remove deletes it. + enum: + - Preserve + - Default + - Remove + type: string + value: + anyOf: + - type: integer + - type: string + description: Value is the quantity applied by + the Default policy. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - policy + type: object + x-kubernetes-validations: + - message: value must be set only for the Default + policy + rule: 'self.policy == ''Default'' ? has(self.value) + : !has(self.value)' + description: Requests defines policies for resource + requests. + minProperties: 1 + type: object + type: object + x-kubernetes-validations: + - message: at least one of requests or limits must be set + rule: has(self.requests) || has(self.limits) schedulers: description: |- Schedulers defines schedulerName matchers for Pod admission. @@ -468,6 +557,7 @@ spec: If empty, each webhook applies its own backwards-compatible default. items: enum: + - pod - pod/initcontainers - pod/ephemeralcontainers - pod/containers @@ -1019,6 +1109,97 @@ spec: - message: at least one of exact or exp must be set rule: has(self.exact) || has(self.exp) type: array + resources: + description: |- + Resources defines mutation and enforcement policies for Pod and container + resource requests and limits. The workload targets select where the + policies apply. With no targets, resource policies apply to all compatible + locations: Pod-level resources, regular containers, and init containers. + Resource names unsupported at Pod level still apply to compatible container + locations. + Mutation is applied when a Pod is created. Remove and MatchRequest manage + explicit values, Default fills an absent value, and Ratio fills an absent + limit from its request. An explicit Ratio violation is then handled by the + enclosing allow, deny, or audit action. + properties: + limits: + additionalProperties: + description: WorkloadResourceLimitPolicy defines + how a resource limit is mutated and enforced. + properties: + policy: + description: |- + Policy selects how the limit is handled: Preserve leaves it unchanged, + Default fills an absent limit, Remove deletes it, MatchRequest manages it + to equal the request, and Ratio defaults an absent limit and enforces the + maximum multiplier against explicitly supplied limits. + enum: + - Preserve + - Default + - Remove + - MatchRequest + - Ratio + type: string + value: + anyOf: + - type: integer + - type: string + description: |- + Value is the quantity applied by Default or the maximum limit-to-request + multiplier applied by Ratio. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - policy + type: object + x-kubernetes-validations: + - message: value must be set only for the Default + and Ratio policies + rule: 'self.policy == ''Default'' || self.policy + == ''Ratio'' ? has(self.value) : !has(self.value)' + description: Limits defines policies for resource + limits. + minProperties: 1 + type: object + requests: + additionalProperties: + description: WorkloadResourceRequestPolicy defines + how a resource request is mutated. + properties: + policy: + description: |- + Policy selects how the request is handled: Preserve leaves it unchanged, + Default fills an absent request, and Remove deletes it. + enum: + - Preserve + - Default + - Remove + type: string + value: + anyOf: + - type: integer + - type: string + description: Value is the quantity applied by + the Default policy. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - policy + type: object + x-kubernetes-validations: + - message: value must be set only for the Default + policy + rule: 'self.policy == ''Default'' ? has(self.value) + : !has(self.value)' + description: Requests defines policies for resource + requests. + minProperties: 1 + type: object + type: object + x-kubernetes-validations: + - message: at least one of requests or limits must be + set + rule: has(self.requests) || has(self.limits) schedulers: description: |- Schedulers defines schedulerName matchers for Pod admission. @@ -1056,6 +1237,7 @@ spec: If empty, each webhook applies its own backwards-compatible default. items: enum: + - pod - pod/initcontainers - pod/ephemeralcontainers - pod/containers @@ -1543,6 +1725,97 @@ spec: - message: at least one of exact or exp must be set rule: has(self.exact) || has(self.exp) type: array + resources: + description: |- + Resources defines mutation and enforcement policies for Pod and container + resource requests and limits. The workload targets select where the + policies apply. With no targets, resource policies apply to all compatible + locations: Pod-level resources, regular containers, and init containers. + Resource names unsupported at Pod level still apply to compatible container + locations. + Mutation is applied when a Pod is created. Remove and MatchRequest manage + explicit values, Default fills an absent value, and Ratio fills an absent + limit from its request. An explicit Ratio violation is then handled by the + enclosing allow, deny, or audit action. + properties: + limits: + additionalProperties: + description: WorkloadResourceLimitPolicy defines + how a resource limit is mutated and enforced. + properties: + policy: + description: |- + Policy selects how the limit is handled: Preserve leaves it unchanged, + Default fills an absent limit, Remove deletes it, MatchRequest manages it + to equal the request, and Ratio defaults an absent limit and enforces the + maximum multiplier against explicitly supplied limits. + enum: + - Preserve + - Default + - Remove + - MatchRequest + - Ratio + type: string + value: + anyOf: + - type: integer + - type: string + description: |- + Value is the quantity applied by Default or the maximum limit-to-request + multiplier applied by Ratio. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - policy + type: object + x-kubernetes-validations: + - message: value must be set only for the Default + and Ratio policies + rule: 'self.policy == ''Default'' || self.policy + == ''Ratio'' ? has(self.value) : !has(self.value)' + description: Limits defines policies for resource + limits. + minProperties: 1 + type: object + requests: + additionalProperties: + description: WorkloadResourceRequestPolicy defines + how a resource request is mutated. + properties: + policy: + description: |- + Policy selects how the request is handled: Preserve leaves it unchanged, + Default fills an absent request, and Remove deletes it. + enum: + - Preserve + - Default + - Remove + type: string + value: + anyOf: + - type: integer + - type: string + description: Value is the quantity applied + by the Default policy. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - policy + type: object + x-kubernetes-validations: + - message: value must be set only for the Default + policy + rule: 'self.policy == ''Default'' ? has(self.value) + : !has(self.value)' + description: Requests defines policies for resource + requests. + minProperties: 1 + type: object + type: object + x-kubernetes-validations: + - message: at least one of requests or limits must be + set + rule: has(self.requests) || has(self.limits) schedulers: description: |- Schedulers defines schedulerName matchers for Pod admission. @@ -1580,6 +1853,7 @@ spec: If empty, each webhook applies its own backwards-compatible default. items: enum: + - pod - pod/initcontainers - pod/ephemeralcontainers - pod/containers diff --git a/charts/capsule/crds/capsule.clastix.io_tenants.yaml b/charts/capsule/crds/capsule.clastix.io_tenants.yaml index d327375d..467138c2 100644 --- a/charts/capsule/crds/capsule.clastix.io_tenants.yaml +++ b/charts/capsule/crds/capsule.clastix.io_tenants.yaml @@ -2889,6 +2889,97 @@ spec: - message: at least one of exact or exp must be set rule: has(self.exact) || has(self.exp) type: array + resources: + description: |- + Resources defines mutation and enforcement policies for Pod and container + resource requests and limits. The workload targets select where the + policies apply. With no targets, resource policies apply to all compatible + locations: Pod-level resources, regular containers, and init containers. + Resource names unsupported at Pod level still apply to compatible container + locations. + Mutation is applied when a Pod is created. Remove and MatchRequest manage + explicit values, Default fills an absent value, and Ratio fills an absent + limit from its request. An explicit Ratio violation is then handled by the + enclosing allow, deny, or audit action. + properties: + limits: + additionalProperties: + description: WorkloadResourceLimitPolicy defines + how a resource limit is mutated and enforced. + properties: + policy: + description: |- + Policy selects how the limit is handled: Preserve leaves it unchanged, + Default fills an absent limit, Remove deletes it, MatchRequest manages it + to equal the request, and Ratio defaults an absent limit and enforces the + maximum multiplier against explicitly supplied limits. + enum: + - Preserve + - Default + - Remove + - MatchRequest + - Ratio + type: string + value: + anyOf: + - type: integer + - type: string + description: |- + Value is the quantity applied by Default or the maximum limit-to-request + multiplier applied by Ratio. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - policy + type: object + x-kubernetes-validations: + - message: value must be set only for the Default + and Ratio policies + rule: 'self.policy == ''Default'' || self.policy + == ''Ratio'' ? has(self.value) : !has(self.value)' + description: Limits defines policies for resource + limits. + minProperties: 1 + type: object + requests: + additionalProperties: + description: WorkloadResourceRequestPolicy defines + how a resource request is mutated. + properties: + policy: + description: |- + Policy selects how the request is handled: Preserve leaves it unchanged, + Default fills an absent request, and Remove deletes it. + enum: + - Preserve + - Default + - Remove + type: string + value: + anyOf: + - type: integer + - type: string + description: Value is the quantity applied + by the Default policy. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - policy + type: object + x-kubernetes-validations: + - message: value must be set only for the Default + policy + rule: 'self.policy == ''Default'' ? has(self.value) + : !has(self.value)' + description: Requests defines policies for resource + requests. + minProperties: 1 + type: object + type: object + x-kubernetes-validations: + - message: at least one of requests or limits must be + set + rule: has(self.requests) || has(self.limits) schedulers: description: |- Schedulers defines schedulerName matchers for Pod admission. @@ -2926,6 +3017,7 @@ spec: If empty, each webhook applies its own backwards-compatible default. items: enum: + - pod - pod/initcontainers - pod/ephemeralcontainers - pod/containers diff --git a/charts/capsule/templates/rbac-tenants.yaml b/charts/capsule/templates/rbac-tenants.yaml index c04fb6cf..92c3e039 100644 --- a/charts/capsule/templates/rbac-tenants.yaml +++ b/charts/capsule/templates/rbac-tenants.yaml @@ -24,3 +24,16 @@ rules: resources: ["tenantresources"] verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] {{- end }} +{{- if $.Values.rbac.customquotas.create }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: capsule:{{ include "capsule.fullname" $ }}:customquotas + labels: + {{- toYaml $.Values.rbac.customquotas.labels | nindent 4 }} +rules: +- apiGroups: ["capsule.clastix.io"] + resources: ["customquotas"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] +{{- end }} diff --git a/charts/capsule/templates/validate.yaml b/charts/capsule/templates/validate.yaml new file mode 100644 index 00000000..57db069b --- /dev/null +++ b/charts/capsule/templates/validate.yaml @@ -0,0 +1,3 @@ +{{- if and .Values.tls.enableController .Values.certManager.generateCertificates -}} +{{- fail "tls.enableController and certManager.generateCertificates cannot both be enabled: choose exactly one controller to own the webhook TLS Secret and caBundles" -}} +{{- end -}} diff --git a/charts/capsule/values.schema.json b/charts/capsule/values.schema.json index 07662184..19c27a71 100644 --- a/charts/capsule/values.schema.json +++ b/charts/capsule/values.schema.json @@ -1007,6 +1007,18 @@ "rbac": { "type": "object", "properties": { + "customquotas": { + "description": "Allow the creation of CustomQuotas", + "type": "object", + "properties": { + "create": { + "type": "boolean" + }, + "labels": { + "type": "object" + } + } + }, "resourcepoolclaims": { "description": "Allow the creation of ResourcePoolClaims", "type": "object", @@ -1015,12 +1027,7 @@ "type": "boolean" }, "labels": { - "type": "object", - "properties": { - "rbac.authorization.k8s.io/aggregate-to-admin": { - "type": "string" - } - } + "type": "object" } } }, @@ -1032,12 +1039,7 @@ "type": "boolean" }, "labels": { - "type": "object", - "properties": { - "rbac.authorization.k8s.io/aggregate-to-admin": { - "type": "string" - } - } + "type": "object" } } } diff --git a/charts/capsule/values.yaml b/charts/capsule/values.yaml index 3794fd54..3aed8196 100644 --- a/charts/capsule/values.yaml +++ b/charts/capsule/values.yaml @@ -107,13 +107,18 @@ rbac: # -- Allow the creation of TenantResources resources: create: false - labels: - rbac.authorization.k8s.io/aggregate-to-admin: "true" + labels: {} + # rbac.authorization.k8s.io/aggregate-to-admin: "true" # -- Allow the creation of ResourcePoolClaims resourcepoolclaims: create: false - labels: - rbac.authorization.k8s.io/aggregate-to-admin: "true" + labels: {} + # rbac.authorization.k8s.io/aggregate-to-admin: "true" + # -- Allow the creation of CustomQuotas + customquotas: + create: false + labels: {} + # rbac.authorization.k8s.io/aggregate-to-admin: "true" # Manager Options manager: @@ -1149,7 +1154,7 @@ webhooks: expression: 'request.resource.resource != "events"' # -- [ReinvocationPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#reinvocation-policy) - reinvocationPolicy: Never + reinvocationPolicy: IfNeeded # -- [Rules](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-rules) rules: - apiGroups: @@ -1532,6 +1537,20 @@ webhooks: resources: - '*' scope: "Namespaced" + # Rule-generated GlobalResourceQuotas carry the managed-by label and + # must receive the same drift protection as namespaced managed objects. + # The status subresource is intentionally not included. + - apiGroups: + - capsule.clastix.io + apiVersions: + - v1beta2 + operations: + - CREATE + - UPDATE + - DELETE + resources: + - globalresourcequotas + scope: "Cluster" replications: # -- Enable the Hook diff --git a/cmd/controller/main.go b/cmd/controller/main.go index f3ec42dc..16f75783 100644 --- a/cmd/controller/main.go +++ b/cmd/controller/main.go @@ -613,6 +613,21 @@ func main() { _ = manager.AddReadyzCheck("ping", healthz.Ping) _ = manager.AddHealthzCheck("ping", healthz.Ping) + if directCfg.EnableTLSConfiguration() && webhookCertWatcher != nil { + if err := manager.AddReadyzCheck( + "webhook-certificate", + tlscontroller.WebhookCertificateReadinessCheck( + directClient, + directCfg, + ns, + webhookCertWatcher.GetCertificate, + ), + ); err != nil { + setupLog.Error(err, "unable to add webhook certificate readiness check") + os.Exit(1) + } + } + dc, err := discovery.NewDiscoveryClientForConfig(manager.GetConfig()) if err != nil { setupLog.Error(err, "unable to create discovery client") diff --git a/e2e/config_node_user_metadata_test.go b/e2e/config_node_user_metadata_test.go index c7009609..c83819bc 100644 --- a/e2e/config_node_user_metadata_test.go +++ b/e2e/config_node_user_metadata_test.go @@ -211,13 +211,16 @@ var _ = Describe("modifying node labels and annotations", Ordered, Label("config } }) - Expect(ModifyNode(func(node *corev1.Node) error { - node.Labels["foo"] = "bar" - node.Labels["gatsby-foo"] = "bar" - node.Annotations["foo"] = "bar" - node.Annotations["gatsby-foo"] = "bar" - return k8sClient.Update(context.Background(), node) - })).Should(Succeed()) + EventuallyCreation(func() error { + return ModifyNode(func(node *corev1.Node) error { + node.Labels["foo"] = "bar" + node.Labels["gatsby-foo"] = "bar" + node.Annotations["foo"] = "bar" + node.Annotations["gatsby-foo"] = "bar" + + return k8sClient.Update(context.Background(), node) + }) + }).Should(Succeed()) By("adding forbidden labels using exact match", func() { EventuallyCreation(func() error { @@ -228,7 +231,7 @@ var _ = Describe("modifying node labels and annotations", Ordered, Label("config _, err := cs.CoreV1().Nodes().Update(context.Background(), node, metav1.UpdateOptions{}) return err }) - }).ShouldNot(Succeed()) + }).Should(MatchError(ContainSubstring("some labels are marked as forbidden"))) }) By("adding forbidden labels using regex match", func() { EventuallyCreation(func() error { @@ -239,7 +242,7 @@ var _ = Describe("modifying node labels and annotations", Ordered, Label("config _, err := cs.CoreV1().Nodes().Update(context.Background(), node, metav1.UpdateOptions{}) return err }) - }).ShouldNot(Succeed()) + }).Should(MatchError(ContainSubstring("some labels are marked as forbidden"))) }) By("modifying forbidden labels", func() { EventuallyCreation(func() error { @@ -250,7 +253,7 @@ var _ = Describe("modifying node labels and annotations", Ordered, Label("config _, err := cs.CoreV1().Nodes().Update(context.Background(), node, metav1.UpdateOptions{}) return err }) - }).ShouldNot(Succeed()) + }).Should(MatchError(ContainSubstring("some labels are marked as forbidden"))) }) By("adding forbidden annotations using exact match", func() { EventuallyCreation(func() error { @@ -261,7 +264,7 @@ var _ = Describe("modifying node labels and annotations", Ordered, Label("config _, err := cs.CoreV1().Nodes().Update(context.Background(), node, metav1.UpdateOptions{}) return err }) - }).ShouldNot(Succeed()) + }).Should(MatchError(ContainSubstring("some annotations are marked as forbidden"))) }) By("adding forbidden annotations using regex match", func() { EventuallyCreation(func() error { @@ -272,7 +275,7 @@ var _ = Describe("modifying node labels and annotations", Ordered, Label("config _, err := cs.CoreV1().Nodes().Update(context.Background(), node, metav1.UpdateOptions{}) return err }) - }).ShouldNot(Succeed()) + }).Should(MatchError(ContainSubstring("some annotations are marked as forbidden"))) }) By("modifying forbidden annotations", func() { EventuallyCreation(func() error { @@ -283,7 +286,7 @@ var _ = Describe("modifying node labels and annotations", Ordered, Label("config _, err := cs.CoreV1().Nodes().Update(context.Background(), node, metav1.UpdateOptions{}) return err }) - }).ShouldNot(Succeed()) + }).Should(MatchError(ContainSubstring("some annotations are marked as forbidden"))) }) }) diff --git a/e2e/global_resource_quota_test.go b/e2e/global_resource_quota_test.go index 629e9a05..ce58da99 100644 --- a/e2e/global_resource_quota_test.go +++ b/e2e/global_resource_quota_test.go @@ -511,6 +511,53 @@ var _ = Describe("GlobalResourceQuota", Ordered, Label("globalresourcequota", "r "requests.ephemeral-storage (requested=600Mi, current=600Mi, projected=1200Mi, hard=1Gi, exceededBy=176Mi)", )) }) + + It("rejects direct hard-limit reductions and removals below allocated usage", func() { + quotaKey := client.ObjectKey{Name: ephemeralQuotaName} + Eventually(func(g Gomega) { + current := &capsulev1beta2.GlobalResourceQuota{} + g.Expect(k8sClient.Get(ctx, quotaKey, current)).To(Succeed()) + used := current.Status.Total.Used[corev1.ResourceRequestsEphemeralStorage] + g.Expect(used.Cmp(resource.MustParse("600Mi"))).To(Equal(0)) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + + By("rejecting a decrease below usage", func() { + current := &capsulev1beta2.GlobalResourceQuota{} + Expect(k8sClient.Get(ctx, quotaKey, current)).To(Succeed()) + current.Spec.Quota.Hard[corev1.ResourceRequestsEphemeralStorage] = resource.MustParse("500Mi") + + err := k8sClient.Update(ctx, current) + Expect(err).To(MatchError(ContainSubstring( + `spec.quota.hard["requests.ephemeral-storage"] cannot be reduced to 500Mi while 600Mi is allocated`, + ))) + }) + + By("rejecting removal of a resource with usage", func() { + current := &capsulev1beta2.GlobalResourceQuota{} + Expect(k8sClient.Get(ctx, quotaKey, current)).To(Succeed()) + delete(current.Spec.Quota.Hard, corev1.ResourceRequestsEphemeralStorage) + + err := k8sClient.Update(ctx, current) + Expect(err).To(MatchError(ContainSubstring( + `spec.quota.hard["requests.ephemeral-storage"] cannot be removed while 600Mi is allocated`, + ))) + }) + + By("allowing a decrease exactly to allocated usage", func() { + current := &capsulev1beta2.GlobalResourceQuota{} + Expect(k8sClient.Get(ctx, quotaKey, current)).To(Succeed()) + current.Spec.Quota.Hard[corev1.ResourceRequestsEphemeralStorage] = resource.MustParse("600Mi") + + Expect(k8sClient.Update(ctx, current)).To(Succeed()) + Eventually(func(g Gomega) { + reconciled := &capsulev1beta2.GlobalResourceQuota{} + g.Expect(k8sClient.Get(ctx, quotaKey, reconciled)).To(Succeed()) + g.Expect(reconciled.Status.ObservedGeneration).To(Equal(reconciled.Generation)) + hard := reconciled.Status.Total.Hard[corev1.ResourceRequestsEphemeralStorage] + g.Expect(hard.Cmp(resource.MustParse("600Mi"))).To(Equal(0)) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + }) + }) }) func expectResourceListEqual(g Gomega, actual, expected corev1.ResourceList) { diff --git a/e2e/rules_enforce_resources_test.go b/e2e/rules_enforce_resources_test.go new file mode 100644 index 00000000..fc9e98ba --- /dev/null +++ b/e2e/rules_enforce_resources_test.go @@ -0,0 +1,828 @@ +// Copyright 2020-2026 Project Capsule Authors. +// SPDX-License-Identifier: Apache-2.0 + +package e2e + +import ( + "context" + "fmt" + "strings" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "sigs.k8s.io/controller-runtime/pkg/client" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" + "github.com/projectcapsule/capsule/pkg/api/rules" +) + +var _ = Describe("enforcing workload resource namespace rules", Ordered, Label("tenant", "rules", "enforce", "workloads", "resources"), func() { + const ownerName = "e2e-rules-resources" + + var tnt *capsulev1beta2.Tenant + targetCases := []struct { + name string + targets []rules.WorkloadValidationTarget + }{ + {name: "pod", targets: []rules.WorkloadValidationTarget{rules.ValidatePod}}, + {name: "containers", targets: []rules.WorkloadValidationTarget{rules.ValidateContainers}}, + {name: "initcontainers", targets: []rules.WorkloadValidationTarget{rules.ValidateInitContainers}}, + {name: "pod-containers", targets: []rules.WorkloadValidationTarget{rules.ValidatePod, rules.ValidateContainers}}, + {name: "pod-initcontainers", targets: []rules.WorkloadValidationTarget{rules.ValidatePod, rules.ValidateInitContainers}}, + {name: "containers-initcontainers", targets: []rules.WorkloadValidationTarget{rules.ValidateContainers, rules.ValidateInitContainers}}, + { + name: "pod-containers-initcontainers", + targets: []rules.WorkloadValidationTarget{ + rules.ValidatePod, + rules.ValidateContainers, + rules.ValidateInitContainers, + }, + }, + } + + newTenant := func() *capsulev1beta2.Tenant { + tenant := &capsulev1beta2.Tenant{ + ObjectMeta: metav1.ObjectMeta{ + Name: "e2e-rule-resources", + Labels: map[string]string{"env": "e2e"}, + }, + Spec: capsulev1beta2.TenantSpec{ + Owners: rbac.OwnerListSpec{{CoreOwnerSpec: rbac.CoreOwnerSpec{UserSpec: rbac.UserSpec{ + Name: ownerName, + Kind: rbac.UserOwner, + }}}}, + Rules: []*rules.NamespaceRuleBodyTenant{ + { + NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"resource-policy": "managed"}}, + NamespaceRuleBodyNamespace: &rules.NamespaceRuleBodyNamespace{Enforce: &rules.NamespaceRuleEnforceBody{ + Action: rules.ActionTypeDeny, + Workloads: rules.NamespaceRuleEnforceWorkloadsBody{ + Resources: &rules.WorkloadResourceRules{ + Requests: map[corev1.ResourceName]rules.WorkloadResourceRequestPolicy{ + corev1.ResourceCPU: { + Policy: rules.WorkloadResourceRequestPolicyDefault, + Value: e2eResourceQuantity("100m"), + }, + }, + Limits: map[corev1.ResourceName]rules.WorkloadResourceLimitPolicy{ + corev1.ResourceCPU: {Policy: rules.WorkloadResourceLimitPolicyRemove}, + corev1.ResourceMemory: { + Policy: rules.WorkloadResourceLimitPolicyMatchRequest, + }, + }, + }, + }, + }}, + }, + { + NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"resource-policy": "ephemeral-managed"}}, + NamespaceRuleBodyNamespace: &rules.NamespaceRuleBodyNamespace{Enforce: &rules.NamespaceRuleEnforceBody{ + Action: rules.ActionTypeDeny, + Workloads: rules.NamespaceRuleEnforceWorkloadsBody{ + Targets: []rules.WorkloadValidationTarget{ + rules.ValidateContainers, + rules.ValidateInitContainers, + }, + Resources: &rules.WorkloadResourceRules{ + Requests: map[corev1.ResourceName]rules.WorkloadResourceRequestPolicy{ + corev1.ResourceEphemeralStorage: { + Policy: rules.WorkloadResourceRequestPolicyDefault, + Value: e2eResourceQuantity("1Gi"), + }, + }, + Limits: map[corev1.ResourceName]rules.WorkloadResourceLimitPolicy{ + corev1.ResourceEphemeralStorage: { + Policy: rules.WorkloadResourceLimitPolicyDefault, + Value: e2eResourceQuantity("2Gi"), + }, + }, + }, + }, + }}, + }, + { + NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"resource-policy": "ratio"}}, + NamespaceRuleBodyNamespace: &rules.NamespaceRuleBodyNamespace{Enforce: &rules.NamespaceRuleEnforceBody{ + Action: rules.ActionTypeDeny, + Workloads: rules.NamespaceRuleEnforceWorkloadsBody{ + Targets: []rules.WorkloadValidationTarget{rules.ValidateContainers}, + Resources: &rules.WorkloadResourceRules{ + Limits: map[corev1.ResourceName]rules.WorkloadResourceLimitPolicy{ + corev1.ResourceMemory: { + Policy: rules.WorkloadResourceLimitPolicyRatio, + Value: e2eResourceQuantity("1.5"), + }, + }, + }, + }, + }}, + }, + { + NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"resource-policy": "init-only"}}, + NamespaceRuleBodyNamespace: &rules.NamespaceRuleBodyNamespace{Enforce: &rules.NamespaceRuleEnforceBody{ + Action: rules.ActionTypeDeny, + Workloads: rules.NamespaceRuleEnforceWorkloadsBody{ + Targets: []rules.WorkloadValidationTarget{rules.ValidateInitContainers}, + Resources: &rules.WorkloadResourceRules{ + Requests: map[corev1.ResourceName]rules.WorkloadResourceRequestPolicy{ + corev1.ResourceCPU: {Policy: rules.WorkloadResourceRequestPolicyRemove}, + corev1.ResourceMemory: {Policy: rules.WorkloadResourceRequestPolicyPreserve}, + }, + Limits: map[corev1.ResourceName]rules.WorkloadResourceLimitPolicy{ + corev1.ResourceCPU: {Policy: rules.WorkloadResourceLimitPolicyRemove}, + corev1.ResourceMemory: { + Policy: rules.WorkloadResourceLimitPolicyDefault, + Value: e2eResourceQuantity("256Mi"), + }, + corev1.ResourceEphemeralStorage: {Policy: rules.WorkloadResourceLimitPolicyPreserve}, + }, + }, + }, + }}, + }, + { + NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"resource-policy": "pod-ratio"}}, + NamespaceRuleBodyNamespace: &rules.NamespaceRuleBodyNamespace{Enforce: &rules.NamespaceRuleEnforceBody{ + Action: rules.ActionTypeDeny, + Workloads: rules.NamespaceRuleEnforceWorkloadsBody{ + Targets: []rules.WorkloadValidationTarget{rules.ValidatePod}, + Resources: &rules.WorkloadResourceRules{ + Limits: map[corev1.ResourceName]rules.WorkloadResourceLimitPolicy{ + corev1.ResourceCPU: { + Policy: rules.WorkloadResourceLimitPolicyRatio, + Value: e2eResourceQuantity("1.5"), + }, + }, + }, + }, + }}, + }, + { + NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"resource-policy": "ratio-allow"}}, + NamespaceRuleBodyNamespace: resourceRatioRule(rules.ActionTypeAllow, rules.ValidateContainers, "1.5"), + }, + { + NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"resource-policy": "ratio-audit"}}, + NamespaceRuleBodyNamespace: resourceRatioRule(rules.ActionTypeAudit, rules.ValidateContainers, "1.5"), + }, + { + NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"resource-policy": "ephemeral-ratio"}}, + NamespaceRuleBodyNamespace: &rules.NamespaceRuleBodyNamespace{Enforce: &rules.NamespaceRuleEnforceBody{ + Action: rules.ActionTypeDeny, + Workloads: rules.NamespaceRuleEnforceWorkloadsBody{ + Resources: &rules.WorkloadResourceRules{ + Limits: map[corev1.ResourceName]rules.WorkloadResourceLimitPolicy{ + corev1.ResourceEphemeralStorage: { + Policy: rules.WorkloadResourceLimitPolicyRatio, + Value: e2eResourceQuantity("1.5"), + }, + }, + }, + }, + }}, + }, + { + NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"resource-policy": "ratio-override"}}, + NamespaceRuleBodyNamespace: resourceRatioRule(rules.ActionTypeDeny, rules.ValidateContainers, "1.5"), + }, + { + NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"resource-policy": "ratio-override"}}, + NamespaceRuleBodyNamespace: resourceRatioRule(rules.ActionTypeAllow, rules.ValidateContainers, "2"), + }, + { + NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"resource-policy": "ratio-preserve"}}, + NamespaceRuleBodyNamespace: resourceRatioRule(rules.ActionTypeDeny, rules.ValidateContainers, "1.5"), + }, + { + NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"resource-policy": "ratio-preserve"}}, + NamespaceRuleBodyNamespace: &rules.NamespaceRuleBodyNamespace{Enforce: &rules.NamespaceRuleEnforceBody{ + Action: rules.ActionTypeAllow, + Workloads: rules.NamespaceRuleEnforceWorkloadsBody{ + Targets: []rules.WorkloadValidationTarget{rules.ValidateContainers}, + Resources: &rules.WorkloadResourceRules{ + Limits: map[corev1.ResourceName]rules.WorkloadResourceLimitPolicy{ + corev1.ResourceMemory: {Policy: rules.WorkloadResourceLimitPolicyPreserve}, + }, + }, + }, + }}, + }, + }, + }, + } + + for _, targetCase := range targetCases { + tenant.Spec.Rules = append(tenant.Spec.Rules, &rules.NamespaceRuleBodyTenant{ + NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{ + "resource-policy": "target-matrix-" + targetCase.name, + }}, + NamespaceRuleBodyNamespace: resourceRatioTargetsRule( + rules.ActionTypeDeny, + targetCase.targets, + "1.5", + ), + }) + } + + return tenant + } + + newPod := func(name string, memoryLimit string) *corev1.Pod { + resources := corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("1Gi")}, + } + if memoryLimit != "" { + resources.Limits = corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1"), + corev1.ResourceMemory: resource.MustParse(memoryLimit), + } + } + + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + Spec: corev1.PodSpec{ + SecurityContext: nobodyPodSecurityContext(), + Containers: []corev1.Container{{ + Name: "app", + Image: "registry.k8s.io/pause:3.9", + ImagePullPolicy: corev1.PullIfNotPresent, + SecurityContext: restrictedContainerSecurityContext(), + Resources: resources, + }}, + }, + } + } + + createNamespace := func(policy string) *corev1.Namespace { + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.Name, + "resource-policy": policy, + }) + NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) + + return ns + } + + createPodAndExpectDenied := func(cs kubernetes.Interface, namespace string, pod *corev1.Pod, substrings ...string) { + Eventually(func() error { + candidate := pod.DeepCopy() + candidate.Name = fmt.Sprintf("%s-%d", pod.Name, time.Now().UnixNano()%1e6) + + _, err := cs.CoreV1().Pods(namespace).Create(context.Background(), candidate, metav1.CreateOptions{}) + if err == nil { + _ = cs.CoreV1().Pods(namespace).Delete(context.Background(), candidate.Name, metav1.DeleteOptions{}) + + return fmt.Errorf("expected Pod creation to be denied") + } + if apierrors.IsAlreadyExists(err) { + return err + } + + for _, substring := range substrings { + if !strings.Contains(err.Error(), substring) { + return fmt.Errorf("expected error to contain %q, got %v", substring, err) + } + } + + return nil + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + } + + createPodAndExpectAllowed := func(cs kubernetes.Interface, namespace string, pod *corev1.Pod) *corev1.Pod { + var created *corev1.Pod + EventuallyCreation(func() error { + var err error + created, err = cs.CoreV1().Pods(namespace).Create(context.Background(), pod, metav1.CreateOptions{}) + + return err + }).Should(Succeed()) + + return created + } + + expectResourceAuditEvent := func(namespace string, podName string, substrings ...string) { + Eventually(func() error { + events, err := clusterAdminClient().EventsV1().Events(namespace).List(context.Background(), metav1.ListOptions{}) + if err != nil { + return err + } + + for _, event := range events.Items { + if event.Regarding.Name != podName || event.Reason != "NamespaceRuleAudit" { + continue + } + + matched := true + for _, substring := range substrings { + if !strings.Contains(event.Note, substring) { + matched = false + + break + } + } + if matched { + return nil + } + } + + return fmt.Errorf("expected resource audit event for Pod %q containing %q", podName, substrings) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + } + + JustBeforeEach(func() { + tnt = newTenant() + EventuallyCreation(func() error { + tnt.ResourceVersion = "" + + return k8sClient.Create(context.Background(), tnt) + }).Should(Succeed()) + TenantReady(tnt, metav1.ConditionTrue, defaultTimeoutInterval) + }) + + JustAfterEach(func() { + EventuallyDeletion(tnt) + }) + + It("projects ordered resource policies into the namespace RuleStatus", func() { + ns := createNamespace("ratio-override") + + Eventually(func(g Gomega) { + status := &capsulev1beta2.RuleStatus{} + g.Expect(k8sClient.Get(context.Background(), client.ObjectKey{ + Name: meta.NameForManagedRuleStatus(), + Namespace: ns.Name, + }, status)).To(Succeed()) + g.Expect(status.Status.Rules).To(HaveLen(2)) + + for index, expected := range []struct { + action rules.ActionType + ratio string + }{ + {action: rules.ActionTypeDeny, ratio: "1.5"}, + {action: rules.ActionTypeAllow, ratio: "2"}, + } { + projected := status.Status.Rules[index] + g.Expect(projected).NotTo(BeNil()) + g.Expect(projected.Enforce).NotTo(BeNil()) + g.Expect(projected.Enforce.Action).To(Equal(expected.action)) + g.Expect(projected.Enforce.Workloads.Targets).To(Equal( + []rules.WorkloadValidationTarget{rules.ValidateContainers}, + )) + + policy := projected.Enforce.Workloads.Resources.Limits[corev1.ResourceMemory] + g.Expect(policy.Policy).To(Equal(rules.WorkloadResourceLimitPolicyRatio)) + g.Expect(policy.Value).NotTo(BeNil()) + g.Expect(policy.Value.Cmp(resource.MustParse(expected.ratio))).To(Equal(0)) + } + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + }) + + It("defaults, removes, and matches resources at all default targets", func() { + ns := createNamespace("managed") + cs := ownerClient(tnt.Spec.Owners[0].UserSpec) + pod := newPod("managed-resources", "2Gi") + pod.Spec.Containers[0].Resources.Requests[corev1.ResourceCPU] = resource.MustParse("250m") + pod.Spec.Containers = append(pod.Spec.Containers, corev1.Container{ + Name: "sidecar", + Image: "registry.k8s.io/pause:3.9", + ImagePullPolicy: corev1.PullIfNotPresent, + SecurityContext: restrictedContainerSecurityContext(), + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("128Mi")}, + Limits: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("512Mi")}, + }, + }) + pod.Spec.InitContainers = []corev1.Container{{ + Name: "init-with-limit", + Image: "registry.k8s.io/pause:3.9", + ImagePullPolicy: corev1.PullIfNotPresent, + SecurityContext: restrictedContainerSecurityContext(), + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("50m"), + corev1.ResourceMemory: resource.MustParse("512Mi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("2"), + corev1.ResourceMemory: resource.MustParse("1Gi"), + }, + }, + }, { + Name: "init-defaulted", + Image: "registry.k8s.io/pause:3.9", + ImagePullPolicy: corev1.PullIfNotPresent, + SecurityContext: restrictedContainerSecurityContext(), + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("256Mi")}, + }, + }} + pod.Spec.Resources = &corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("2"), + corev1.ResourceMemory: resource.MustParse("2Gi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1"), + corev1.ResourceMemory: resource.MustParse("512Mi"), + }, + } + + created := createPodAndExpectAllowed(cs, ns.Name, pod) + + containerResources := created.Spec.Containers[0].Resources + Expect(containerResources.Requests.Cpu().Cmp(resource.MustParse("250m"))).To(Equal(0)) + Expect(containerResources.Limits).NotTo(HaveKey(corev1.ResourceCPU)) + Expect(containerResources.Limits.Memory().Cmp(resource.MustParse("1Gi"))).To(Equal(0)) + + sidecarResources := created.Spec.Containers[1].Resources + Expect(sidecarResources.Requests.Cpu().Cmp(resource.MustParse("100m"))).To(Equal(0)) + Expect(sidecarResources.Limits).NotTo(HaveKey(corev1.ResourceCPU)) + Expect(sidecarResources.Limits.Memory().Cmp(resource.MustParse("128Mi"))).To(Equal(0)) + + initResources := created.Spec.InitContainers[0].Resources + Expect(initResources.Requests.Cpu().Cmp(resource.MustParse("50m"))).To(Equal(0)) + Expect(initResources.Limits).NotTo(HaveKey(corev1.ResourceCPU)) + Expect(initResources.Limits.Memory().Cmp(resource.MustParse("512Mi"))).To(Equal(0)) + + defaultedInitResources := created.Spec.InitContainers[1].Resources + Expect(defaultedInitResources.Requests.Cpu().Cmp(resource.MustParse("100m"))).To(Equal(0)) + Expect(defaultedInitResources.Limits).NotTo(HaveKey(corev1.ResourceCPU)) + Expect(defaultedInitResources.Limits.Memory().Cmp(resource.MustParse("256Mi"))).To(Equal(0)) + + Expect(created.Spec.Resources).NotTo(BeNil()) + Expect(created.Spec.Resources.Requests.Cpu().Cmp(resource.MustParse("2"))).To(Equal(0)) + Expect(created.Spec.Resources.Limits).NotTo(HaveKey(corev1.ResourceCPU)) + Expect(created.Spec.Resources.Limits.Memory().Cmp(resource.MustParse("2Gi"))).To(Equal(0)) + }) + + It("applies Ratio only to every explicit target combination", func() { + cs := ownerClient(tnt.Spec.Owners[0].UserSpec) + + for _, targetCase := range targetCases { + By(targetCase.name) + + ns := createNamespace("target-matrix-" + targetCase.name) + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "target-matrix-" + targetCase.name}, + Spec: corev1.PodSpec{ + SecurityContext: nobodyPodSecurityContext(), + Containers: []corev1.Container{{ + Name: "app", + Image: "registry.k8s.io/pause:3.9", + ImagePullPolicy: corev1.PullIfNotPresent, + SecurityContext: restrictedContainerSecurityContext(), + Resources: corev1.ResourceRequirements{Requests: corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse("200Mi"), + }}, + }}, + InitContainers: []corev1.Container{{ + Name: "init", + Image: "registry.k8s.io/pause:3.9", + ImagePullPolicy: corev1.PullIfNotPresent, + SecurityContext: restrictedContainerSecurityContext(), + Resources: corev1.ResourceRequirements{Requests: corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse("400Mi"), + }}, + }}, + Resources: &corev1.ResourceRequirements{Requests: corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse("1Gi"), + }}, + }, + } + + created := createPodAndExpectAllowed(cs, ns.Name, pod) + targeted := make(map[rules.WorkloadValidationTarget]bool, len(targetCase.targets)) + for _, target := range targetCase.targets { + targeted[target] = true + } + + assertLimit := func( + target rules.WorkloadValidationTarget, + resources corev1.ResourceRequirements, + expected string, + ) { + limit, found := resources.Limits[corev1.ResourceMemory] + Expect(found).To(Equal(targeted[target]), "target %q in case %q", target, targetCase.name) + if found { + Expect(limit.Cmp(resource.MustParse(expected))).To(Equal(0), "target %q in case %q", target, targetCase.name) + } + } + + assertLimit(rules.ValidateContainers, created.Spec.Containers[0].Resources, "300Mi") + assertLimit(rules.ValidateInitContainers, created.Spec.InitContainers[0].Resources, "600Mi") + assertLimit(rules.ValidatePod, *created.Spec.Resources, "1536Mi") + } + }) + + It("defaults ephemeral storage only on explicitly targeted containers", func() { + ns := createNamespace("ephemeral-managed") + cs := ownerClient(tnt.Spec.Owners[0].UserSpec) + pod := newPod("managed-ephemeral-storage", "") + pod.Spec.Containers = append(pod.Spec.Containers, corev1.Container{ + Name: "sidecar", + Image: "registry.k8s.io/pause:3.9", + ImagePullPolicy: corev1.PullIfNotPresent, + SecurityContext: restrictedContainerSecurityContext(), + Resources: corev1.ResourceRequirements{Requests: corev1.ResourceList{ + corev1.ResourceEphemeralStorage: resource.MustParse("1536Mi"), + }}, + }) + pod.Spec.InitContainers = []corev1.Container{{ + Name: "init-with-limit", + Image: "registry.k8s.io/pause:3.9", + ImagePullPolicy: corev1.PullIfNotPresent, + SecurityContext: restrictedContainerSecurityContext(), + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceEphemeralStorage: resource.MustParse("1Gi")}, + Limits: corev1.ResourceList{corev1.ResourceEphemeralStorage: resource.MustParse("3Gi")}, + }, + }, { + Name: "init-defaulted", + Image: "registry.k8s.io/pause:3.9", + ImagePullPolicy: corev1.PullIfNotPresent, + SecurityContext: restrictedContainerSecurityContext(), + }} + + created := createPodAndExpectAllowed(cs, ns.Name, pod) + + Expect(created.Spec.Containers[0].Resources.Requests.StorageEphemeral().Cmp(resource.MustParse("1Gi"))).To(Equal(0)) + Expect(created.Spec.Containers[0].Resources.Limits.StorageEphemeral().Cmp(resource.MustParse("2Gi"))).To(Equal(0)) + Expect(created.Spec.Containers[1].Resources.Requests.StorageEphemeral().Cmp(resource.MustParse("1536Mi"))).To(Equal(0)) + Expect(created.Spec.Containers[1].Resources.Limits.StorageEphemeral().Cmp(resource.MustParse("2Gi"))).To(Equal(0)) + Expect(created.Spec.InitContainers[0].Resources.Requests.StorageEphemeral().Cmp(resource.MustParse("1Gi"))).To(Equal(0)) + Expect(created.Spec.InitContainers[0].Resources.Limits.StorageEphemeral().Cmp(resource.MustParse("3Gi"))).To(Equal(0)) + Expect(created.Spec.InitContainers[1].Resources.Requests.StorageEphemeral().Cmp(resource.MustParse("1Gi"))).To(Equal(0)) + Expect(created.Spec.InitContainers[1].Resources.Limits.StorageEphemeral().Cmp(resource.MustParse("2Gi"))).To(Equal(0)) + Expect(created.Spec.Resources).To(BeNil()) + }) + + It("applies Preserve, Remove, and Default only to an explicitly targeted init container", func() { + ns := createNamespace("init-only") + cs := ownerClient(tnt.Spec.Owners[0].UserSpec) + pod := newPod("init-only-resources", "2Gi") + pod.Spec.Containers[0].Resources.Requests[corev1.ResourceCPU] = resource.MustParse("250m") + pod.Spec.InitContainers = []corev1.Container{{ + Name: "init", + Image: "registry.k8s.io/pause:3.9", + ImagePullPolicy: corev1.PullIfNotPresent, + SecurityContext: restrictedContainerSecurityContext(), + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("100m"), + corev1.ResourceMemory: resource.MustParse("64Mi"), + corev1.ResourceEphemeralStorage: resource.MustParse("1Gi"), + }, + Limits: corev1.ResourceList{corev1.ResourceEphemeralStorage: resource.MustParse("3Gi")}, + }, + }} + + created := createPodAndExpectAllowed(cs, ns.Name, pod) + + containerResources := created.Spec.Containers[0].Resources + Expect(containerResources.Requests.Cpu().Cmp(resource.MustParse("250m"))).To(Equal(0)) + Expect(containerResources.Limits.Cpu().Cmp(resource.MustParse("1"))).To(Equal(0)) + Expect(containerResources.Limits.Memory().Cmp(resource.MustParse("2Gi"))).To(Equal(0)) + + initResources := created.Spec.InitContainers[0].Resources + Expect(initResources.Requests).NotTo(HaveKey(corev1.ResourceCPU)) + Expect(initResources.Requests.Memory().Cmp(resource.MustParse("64Mi"))).To(Equal(0)) + Expect(initResources.Limits).NotTo(HaveKey(corev1.ResourceCPU)) + Expect(initResources.Limits.Memory().Cmp(resource.MustParse("256Mi"))).To(Equal(0)) + Expect(initResources.Requests.StorageEphemeral().Cmp(resource.MustParse("1Gi"))).To(Equal(0)) + Expect(initResources.Limits.StorageEphemeral().Cmp(resource.MustParse("3Gi"))).To(Equal(0)) + }) + + It("accepts MatchRequest after Kubernetes defaults a missing request from its limit", func() { + ns := createNamespace("managed") + cs := ownerClient(tnt.Spec.Owners[0].UserSpec) + pod := newPod("match-request-missing", "1Gi") + delete(pod.Spec.Containers[0].Resources.Requests, corev1.ResourceMemory) + delete(pod.Spec.Containers[0].Resources.Limits, corev1.ResourceCPU) + + created := createPodAndExpectAllowed(cs, ns.Name, pod) + Expect(created.Spec.Containers[0].Resources.Requests.Memory().Cmp(resource.MustParse("1Gi"))).To(Equal(0)) + Expect(created.Spec.Containers[0].Resources.Limits.Memory().Cmp(resource.MustParse("1Gi"))).To(Equal(0)) + }) + + It("defaults a missing limit from Ratio", func() { + ns := createNamespace("ratio") + cs := ownerClient(tnt.Spec.Owners[0].UserSpec) + pod := newPod("ratio-default", "") + + var created *corev1.Pod + EventuallyCreation(func() error { + var err error + created, err = cs.CoreV1().Pods(ns.Name).Create(context.Background(), pod, metav1.CreateOptions{}) + + return err + }).Should(Succeed()) + + Expect(created.Spec.Containers[0].Resources.Limits.Memory().Cmp(resource.MustParse("1536Mi"))).To(Equal(0)) + }) + + It("preserves a compliant explicit Ratio limit and ignores untargeted locations", func() { + ns := createNamespace("ratio") + cs := ownerClient(tnt.Spec.Owners[0].UserSpec) + pod := newPod("ratio-target-isolation", "1280Mi") + pod.Spec.InitContainers = []corev1.Container{{ + Name: "init", + Image: "registry.k8s.io/pause:3.9", + ImagePullPolicy: corev1.PullIfNotPresent, + SecurityContext: restrictedContainerSecurityContext(), + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("1Gi")}, + Limits: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("2Gi")}, + }, + }} + pod.Spec.Resources = &corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("1Gi")}, + Limits: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("2Gi")}, + } + + created := createPodAndExpectAllowed(cs, ns.Name, pod) + Expect(created.Spec.Containers[0].Resources.Limits.Memory().Cmp(resource.MustParse("1280Mi"))).To(Equal(0)) + Expect(created.Spec.InitContainers[0].Resources.Limits.Memory().Cmp(resource.MustParse("2Gi"))).To(Equal(0)) + Expect(created.Spec.Resources.Limits.Memory().Cmp(resource.MustParse("2Gi"))).To(Equal(0)) + }) + + It("denies an explicitly excessive Ratio limit", func() { + ns := createNamespace("ratio") + cs := ownerClient(tnt.Spec.Owners[0].UserSpec) + + createPodAndExpectDenied( + cs, + ns.Name, + newPod("ratio-denied", "2Gi"), + "violates policy Ratio", + "must not exceed 1536Mi", + ) + }) + + It("denies Ratio when the targeted resource request is missing", func() { + ns := createNamespace("ratio") + cs := ownerClient(tnt.Spec.Owners[0].UserSpec) + pod := newPod("ratio-missing-request", "") + delete(pod.Spec.Containers[0].Resources.Requests, corev1.ResourceMemory) + + createPodAndExpectDenied( + cs, + ns.Name, + pod, + "violates policy Ratio", + "requires a request greater than zero", + ) + }) + + It("defaults and enforces Ratio at the explicit Pod target without affecting containers", func() { + ns := createNamespace("pod-ratio") + cs := ownerClient(tnt.Spec.Owners[0].UserSpec) + pod := newPod("pod-ratio-default", "") + pod.Spec.Containers[0].Resources.Requests[corev1.ResourceCPU] = resource.MustParse("100m") + pod.Spec.Containers[0].Resources.Limits = corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("200m")} + pod.Spec.Resources = &corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("500m")}, + } + + created := createPodAndExpectAllowed(cs, ns.Name, pod) + Expect(created.Spec.Resources.Limits.Cpu().Cmp(resource.MustParse("750m"))).To(Equal(0)) + Expect(created.Spec.Containers[0].Resources.Limits.Cpu().Cmp(resource.MustParse("200m"))).To(Equal(0)) + + excessive := newPod("pod-ratio-denied", "") + excessive.Spec.Resources = &corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("500m")}, + Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1")}, + } + createPodAndExpectDenied( + cs, + ns.Name, + excessive, + `spec.resources.limits["cpu"]`, + "must not exceed 750m", + ) + }) + + It("uses Ratio as an allow-list policy", func() { + ns := createNamespace("ratio-allow") + cs := ownerClient(tnt.Spec.Owners[0].UserSpec) + + createPodAndExpectAllowed(cs, ns.Name, newPod("ratio-allow-compliant", "1280Mi")) + createPodAndExpectDenied( + cs, + ns.Name, + newPod("ratio-allow-denied", "2Gi"), + "does not satisfy any allowed resource policy", + `limits["memory"]`, + ) + }) + + It("audits an excessive Ratio without blocking admission", func() { + ns := createNamespace("ratio-audit") + cs := ownerClient(tnt.Spec.Owners[0].UserSpec) + pod := newPod("ratio-audited", "2Gi") + + created := createPodAndExpectAllowed(cs, ns.Name, pod) + Expect(created.Spec.Containers[0].Resources.Limits.Memory().Cmp(resource.MustParse("2Gi"))).To(Equal(0)) + expectResourceAuditEvent( + ns.Name, + pod.Name, + "workload resource limit", + "violates policy Ratio", + "must not exceed 1536Mi", + ) + }) + + It("uses a later Allow policy to override an earlier Ratio denial", func() { + ns := createNamespace("ratio-override") + cs := ownerClient(tnt.Spec.Owners[0].UserSpec) + pod := newPod("ratio-allow-override", "1792Mi") + + created := createPodAndExpectAllowed(cs, ns.Name, pod) + Expect(created.Spec.Containers[0].Resources.Limits.Memory().Cmp(resource.MustParse("1792Mi"))).To(Equal(0)) + }) + + It("uses a later Preserve policy to clear an earlier Ratio constraint", func() { + ns := createNamespace("ratio-preserve") + cs := ownerClient(tnt.Spec.Owners[0].UserSpec) + pod := newPod("ratio-preserve-override", "2Gi") + + created := createPodAndExpectAllowed(cs, ns.Name, pod) + Expect(created.Spec.Containers[0].Resources.Limits.Memory().Cmp(resource.MustParse("2Gi"))).To(Equal(0)) + }) + + It("applies ephemeral-storage Ratio to default container targets but not Pod-level resources", func() { + ns := createNamespace("ephemeral-ratio") + cs := ownerClient(tnt.Spec.Owners[0].UserSpec) + pod := newPod("ephemeral-ratio-default", "") + pod.Spec.Containers[0].Resources.Requests[corev1.ResourceEphemeralStorage] = resource.MustParse("2Gi") + pod.Spec.InitContainers = []corev1.Container{{ + Name: "init", + Image: "registry.k8s.io/pause:3.9", + ImagePullPolicy: corev1.PullIfNotPresent, + SecurityContext: restrictedContainerSecurityContext(), + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceEphemeralStorage: resource.MustParse("1Gi")}, + }, + }} + created := createPodAndExpectAllowed(cs, ns.Name, pod) + Expect(created.Spec.Containers[0].Resources.Limits.StorageEphemeral().Cmp(resource.MustParse("3Gi"))).To(Equal(0)) + Expect(created.Spec.InitContainers[0].Resources.Limits.StorageEphemeral().Cmp(resource.MustParse("1536Mi"))).To(Equal(0)) + Expect(created.Spec.Resources).To(BeNil()) + + excessive := newPod("ephemeral-ratio-denied", "") + excessive.Spec.Containers[0].Resources.Requests[corev1.ResourceEphemeralStorage] = resource.MustParse("2Gi") + excessive.Spec.Containers[0].Resources.Limits = corev1.ResourceList{ + corev1.ResourceEphemeralStorage: resource.MustParse("4Gi"), + } + createPodAndExpectDenied( + cs, + ns.Name, + excessive, + `limits["ephemeral-storage"]`, + "must not exceed 3Gi", + ) + }) +}) + +func e2eResourceQuantity(value string) *resource.Quantity { + quantity := resource.MustParse(value) + + return &quantity +} + +func resourceRatioRule( + action rules.ActionType, + target rules.WorkloadValidationTarget, + ratio string, +) *rules.NamespaceRuleBodyNamespace { + return resourceRatioTargetsRule(action, []rules.WorkloadValidationTarget{target}, ratio) +} + +func resourceRatioTargetsRule( + action rules.ActionType, + targets []rules.WorkloadValidationTarget, + ratio string, +) *rules.NamespaceRuleBodyNamespace { + return &rules.NamespaceRuleBodyNamespace{Enforce: &rules.NamespaceRuleEnforceBody{ + Action: action, + Workloads: rules.NamespaceRuleEnforceWorkloadsBody{ + Targets: targets, + Resources: &rules.WorkloadResourceRules{ + Limits: map[corev1.ResourceName]rules.WorkloadResourceLimitPolicy{ + corev1.ResourceMemory: { + Policy: rules.WorkloadResourceLimitPolicyRatio, + Value: e2eResourceQuantity(ratio), + }, + }, + }, + }, + }} +} diff --git a/e2e/rules_quota_admission_test.go b/e2e/rules_quota_admission_test.go new file mode 100644 index 00000000..d7d29256 --- /dev/null +++ b/e2e/rules_quota_admission_test.go @@ -0,0 +1,542 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package e2e + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" + "github.com/projectcapsule/capsule/pkg/api/rules" + tenantutils "github.com/projectcapsule/capsule/pkg/tenant" +) + +var _ = Describe("rule-generated GlobalResourceQuota admission", Ordered, + Label("resourcequota", "rules", "admission", "managed", "skip-on-openshift"), func() { + const ( + tenantName = "e2e-rule-quota-admission" + quotaName = "compute" + selectorKey = "e2e.projectcapsule.dev/quota-scope" + rbacName = "e2e-rule-quota-admission-tamper" + ) + + ctx := context.Background() + owner := rbac.UserSpec{Name: tenantName, Kind: rbac.UserOwner} + tnt := &capsulev1beta2.Tenant{ + ObjectMeta: metav1.ObjectMeta{ + Name: tenantName, + Labels: map[string]string{"env": "e2e"}, + }, + Spec: capsulev1beta2.TenantSpec{ + Owners: rbac.OwnerListSpec{{CoreOwnerSpec: rbac.CoreOwnerSpec{UserSpec: owner}}}, + Rules: []*rules.NamespaceRuleBodyTenant{{ + NamespaceRuleBodyNamespace: &rules.NamespaceRuleBodyNamespace{ + Quota: []rules.ResourceQuotaRule{{ + Name: quotaName, + ResourceQuotaSpec: corev1.ResourceQuotaSpec{Hard: corev1.ResourceList{ + corev1.ResourceLimitsCPU: resource.MustParse("8"), + corev1.ResourcePods: resource.MustParse("100"), + }}, + }}, + }, + NamespaceSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{selectorKey: "application"}, + }, + }}, + }, + } + quotaKey := client.ObjectKey{Name: tenantutils.RuleGlobalResourceQuotaName(tnt, quotaName)} + tamperRole := &rbacv1.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{Name: rbacName}, + Rules: []rbacv1.PolicyRule{{ + APIGroups: []string{capsulev1beta2.GroupVersion.Group}, + Resources: []string{"globalresourcequotas"}, + Verbs: []string{"get", "create", "update", "patch", "delete"}, + }}, + } + setTenantQuota := func(selector *metav1.LabelSelector, cpu string) error { + current := &capsulev1beta2.Tenant{} + if err := k8sClient.Get(ctx, client.ObjectKey{Name: tenantName}, current); err != nil { + return err + } + + current.Spec.Rules[0].NamespaceSelector = selector + current.Spec.Rules[0].Quota[0].Hard[corev1.ResourceLimitsCPU] = resource.MustParse(cpu) + + return k8sClient.Update(ctx, current) + } + expectGeneratedQuota := func(scopeValue, hard, used string, namespaces ...string) { + Eventually(func(g Gomega) { + current := &capsulev1beta2.GlobalResourceQuota{} + g.Expect(k8sClient.Get(ctx, quotaKey, current)).To(Succeed()) + g.Expect(current.Status.ObservedGeneration).To(Equal(current.Generation)) + g.Expect(current.Status.Namespaces).To(ConsistOf(namespaces)) + + selector := current.Spec.NamespaceSelectors[0].LabelSelector + g.Expect(selector).NotTo(BeNil()) + g.Expect(selector.MatchLabels).To(HaveKeyWithValue(meta.TenantLabel, tenantName)) + if scopeValue == "" { + g.Expect(selector.MatchLabels).NotTo(HaveKey(selectorKey)) + } else { + g.Expect(selector.MatchLabels).To(HaveKeyWithValue(selectorKey, scopeValue)) + } + + hardCPU := current.Spec.Quota.Hard[corev1.ResourceLimitsCPU] + g.Expect(hardCPU.Cmp(resource.MustParse(hard))).To(Equal(0)) + usedCPU := current.Status.Total.Used[corev1.ResourceLimitsCPU] + g.Expect(usedCPU.Cmp(resource.MustParse(used))).To(Equal(0)) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + } + tamperBinding := &rbacv1.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{Name: rbacName}, + RoleRef: rbacv1.RoleRef{ + APIGroup: rbacv1.GroupName, + Kind: "ClusterRole", + Name: rbacName, + }, + Subjects: []rbacv1.Subject{{ + APIGroup: rbacv1.GroupName, + Kind: rbacv1.UserKind, + Name: owner.Name, + }}, + } + + BeforeAll(func() { + EventuallyCreation(func() error { + tamperRole.ResourceVersion = "" + + return k8sClient.Create(ctx, tamperRole) + }).Should(Succeed()) + EventuallyCreation(func() error { + tamperBinding.ResourceVersion = "" + + return k8sClient.Create(ctx, tamperBinding) + }).Should(Succeed()) + + EventuallyCreation(func() error { + tnt.ResourceVersion = "" + + return k8sClient.Create(ctx, tnt) + }).Should(Succeed()) + TenantReadyTrue(tnt) + + currentTenant := &capsulev1beta2.Tenant{} + Expect(k8sClient.Get(ctx, client.ObjectKey{Name: tenantName}, currentTenant)).To(Succeed()) + tnt.UID = currentTenant.UID + + By("waiting for the managed quota and its status to reconcile", func() { + Eventually(func(g Gomega) { + quota := &capsulev1beta2.GlobalResourceQuota{} + g.Expect(k8sClient.Get(ctx, quotaKey, quota)).To(Succeed()) + g.Expect(quota.Labels).To(HaveKeyWithValue(meta.NewManagedByCapsuleLabel, meta.ValueController)) + g.Expect(metav1.IsControlledBy(quota, currentTenant)).To(BeTrue()) + g.Expect(quota.Status.ObservedGeneration).To(Equal(quota.Generation)) + + condition := quota.Status.Conditions.GetConditionByType(meta.ReadyCondition) + g.Expect(condition).NotTo(BeNil()) + g.Expect(condition.Status).To(Equal(metav1.ConditionTrue)) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + }) + }) + + AfterAll(func() { + controllerClient := impersonationClient(ControllerServiceAccountFull, nil) + forged := &capsulev1beta2.GlobalResourceQuota{ObjectMeta: metav1.ObjectMeta{Name: rbacName}} + Expect(ignoreNotFound(controllerClient.Delete(ctx, forged))).To(Succeed()) + + EventuallyDeletion(tnt) + EventuallyDeletion(tamperBinding) + EventuallyDeletion(tamperRole) + }) + + It("rejects a Tenant rule update that decreases or removes a hard limit while changing scope", func() { + for _, test := range []struct { + name string + mutate func(corev1.ResourceList) + message string + }{ + { + name: "decrease", + mutate: func(hard corev1.ResourceList) { + hard[corev1.ResourceLimitsCPU] = resource.MustParse("0") + }, + message: `rules[0].quota[0].hard["limits.cpu"] cannot be reduced from 8 to 0 while namespace selectors are changing`, + }, + { + name: "removal", + mutate: func(hard corev1.ResourceList) { + delete(hard, corev1.ResourceLimitsCPU) + }, + message: `rules[0].quota[0].hard["limits.cpu"] cannot be removed while namespace selectors are changing`, + }, + } { + By(test.name, func() { + Eventually(func() error { + current := &capsulev1beta2.Tenant{} + if err := k8sClient.Get(ctx, client.ObjectKey{Name: tenantName}, current); err != nil { + return err + } + + updated := current.DeepCopy() + updated.Spec.Rules[0].NamespaceSelector = nil + test.mutate(updated.Spec.Rules[0].Quota[0].Hard) + + return k8sClient.Update(ctx, updated) + }, defaultTimeoutInterval, defaultPollInterval).Should(MatchError(ContainSubstring(test.message))) + }) + } + + persisted := &capsulev1beta2.Tenant{} + Expect(k8sClient.Get(ctx, client.ObjectKey{Name: tenantName}, persisted)).To(Succeed()) + Expect(persisted.Spec.Rules[0].NamespaceSelector).NotTo(BeNil()) + Expect(persisted.Spec.Rules[0].NamespaceSelector.MatchLabels).To(HaveKeyWithValue(selectorKey, "application")) + persistedLimit := persisted.Spec.Rules[0].Quota[0].Hard[corev1.ResourceLimitsCPU] + Expect(persistedLimit.Cmp(resource.MustParse("8"))).To(Equal(0)) + }) + + It("rejects the same unsafe scope and hard-limit changes on the generated quota", func() { + controllerClient := impersonationClient(ControllerServiceAccountFull, nil) + for _, test := range []struct { + name string + mutate func(corev1.ResourceList) + message string + }{ + { + name: "decrease", + mutate: func(hard corev1.ResourceList) { + hard[corev1.ResourceLimitsCPU] = resource.MustParse("0") + }, + message: `spec.quota.hard["limits.cpu"] cannot be reduced from 8 to 0 while namespace selectors are changing`, + }, + { + name: "removal", + mutate: func(hard corev1.ResourceList) { + delete(hard, corev1.ResourceLimitsCPU) + }, + message: `spec.quota.hard["limits.cpu"] cannot be removed while namespace selectors are changing`, + }, + } { + By(test.name, func() { + Eventually(func() error { + current := &capsulev1beta2.GlobalResourceQuota{} + if err := controllerClient.Get(ctx, quotaKey, current); err != nil { + return err + } + + updated := current.DeepCopy() + delete(updated.Spec.NamespaceSelectors[0].LabelSelector.MatchLabels, selectorKey) + test.mutate(updated.Spec.Quota.Hard) + + return controllerClient.Update(ctx, updated) + }, defaultTimeoutInterval, defaultPollInterval).Should(MatchError(ContainSubstring(test.message))) + }) + } + + persisted := &capsulev1beta2.GlobalResourceQuota{} + Expect(k8sClient.Get(ctx, quotaKey, persisted)).To(Succeed()) + Expect(persisted.Spec.NamespaceSelectors[0].LabelSelector.MatchLabels).To(HaveKeyWithValue(selectorKey, "application")) + persistedLimit := persisted.Spec.Quota.Hard[corev1.ResourceLimitsCPU] + Expect(persistedLimit.Cmp(resource.MustParse("8"))).To(Equal(0)) + }) + + It("allows equal or increased limits with a scope change and a later same-scope decrease", func() { + Eventually(func() error { + return setTenantQuota(nil, "8") + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + expectGeneratedQuota("", "8", "0") + + applicationSelector := &metav1.LabelSelector{MatchLabels: map[string]string{selectorKey: "application"}} + Eventually(func() error { + return setTenantQuota(applicationSelector, "9") + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + expectGeneratedQuota("application", "9", "0") + + Eventually(func() error { + return setTenantQuota(applicationSelector, "8") + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + expectGeneratedQuota("application", "8", "0") + }) + + It("allows a scope-first transition, observes newly selected usage, and then rejects an unsafe decrease", func() { + ns := NewNamespace("", map[string]string{meta.TenantLabel: tenantName}) + NamespaceCreation(ns, owner, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) + + cs := ownerClient(owner) + pod := MakePod(ns.Name, "scope-first-usage", nil, nil, "registry.k8s.io/pause:3.10", "", "") + pod.Spec.Containers[0].Resources.Limits = corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("300m"), + } + EventuallyCreation(func() error { + _, err := cs.CoreV1().Pods(ns.Name).Create(ctx, pod, metav1.CreateOptions{}) + + return err + }).Should(Succeed()) + + Eventually(func() error { + return setTenantQuota(nil, "8") + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + expectGeneratedQuota("", "8", "300m", ns.Name) + + Eventually(func() error { + return setTenantQuota(nil, "0") + }, defaultTimeoutInterval, defaultPollInterval).Should(MatchError(ContainSubstring( + `rules[0].quota[0].hard["limits.cpu"] cannot be reduced to 0 while 300m is allocated`, + ))) + + Eventually(func() error { + current := &capsulev1beta2.Tenant{} + if err := k8sClient.Get(ctx, client.ObjectKey{Name: tenantName}, current); err != nil { + return err + } + + delete(current.Spec.Rules[0].Quota[0].Hard, corev1.ResourceLimitsCPU) + + return k8sClient.Update(ctx, current) + }, defaultTimeoutInterval, defaultPollInterval).Should(MatchError(ContainSubstring( + `rules[0].quota[0].hard["limits.cpu"] cannot be removed while 300m is allocated`, + ))) + + applicationSelector := &metav1.LabelSelector{MatchLabels: map[string]string{selectorKey: "application"}} + Eventually(func() error { + return setTenantQuota(applicationSelector, "8") + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + expectGeneratedQuota("application", "8", "0") + }) + + It("denies authorized non-admin creation, updates, and deletes of managed quotas", func() { + tamperClient := impersonationClient(owner.Name, withDefaultGroups(nil)) + current := &capsulev1beta2.GlobalResourceQuota{} + Eventually(func() error { + return tamperClient.Get(ctx, quotaKey, current) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + + forged := &capsulev1beta2.GlobalResourceQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: rbacName, + Labels: map[string]string{meta.NewManagedByCapsuleLabel: meta.ValueController}, + }, + Spec: capsulev1beta2.GlobalResourceQuotaSpec{ + Quota: corev1.ResourceQuotaSpec{Hard: corev1.ResourceList{ + corev1.ResourceLimitsCPU: resource.MustParse("1"), + }}, + }, + } + err := tamperClient.Create(ctx, forged) + Expect(err).To(MatchError(ContainSubstring( + "Labeling resources as controller managed can only be done by the controller or administrators", + ))) + + updated := current.DeepCopy() + updated.Annotations = map[string]string{"e2e.projectcapsule.dev/tampered": "true"} + err = tamperClient.Update(ctx, updated) + Expect(err).To(MatchError(ContainSubstring( + "Labeling resources as controller managed can only be done by the controller or administrators", + ))) + + labelRemoval := current.DeepCopy() + delete(labelRemoval.Labels, meta.NewManagedByCapsuleLabel) + err = tamperClient.Update(ctx, labelRemoval) + Expect(err).To(MatchError(ContainSubstring( + "Labeling resources as controller managed can only be done by the controller or administrators", + ))) + + err = tamperClient.Delete(ctx, current) + Expect(err).To(MatchError(ContainSubstring( + "Labeling resources as controller managed can only be done by the controller or administrators", + ))) + + persisted := &capsulev1beta2.GlobalResourceQuota{} + Expect(k8sClient.Get(ctx, quotaKey, persisted)).To(Succeed()) + Expect(persisted.Annotations).NotTo(HaveKey("e2e.projectcapsule.dev/tampered")) + Expect(persisted.Labels).To(HaveKeyWithValue(meta.NewManagedByCapsuleLabel, meta.ValueController)) + }) + + It("does not apply managed protection to an unlabeled GlobalResourceQuota", func() { + tamperClient := impersonationClient(owner.Name, withDefaultGroups(nil)) + unmanaged := &capsulev1beta2.GlobalResourceQuota{ + ObjectMeta: metav1.ObjectMeta{Name: rbacName}, + Spec: capsulev1beta2.GlobalResourceQuotaSpec{ + Quota: corev1.ResourceQuotaSpec{Hard: corev1.ResourceList{ + corev1.ResourcePods: resource.MustParse("10"), + }}, + }, + } + Expect(tamperClient.Create(ctx, unmanaged)).To(Succeed()) + + current := &capsulev1beta2.GlobalResourceQuota{} + Expect(tamperClient.Get(ctx, client.ObjectKey{Name: unmanaged.Name}, current)).To(Succeed()) + current.Annotations = map[string]string{"e2e.projectcapsule.dev/updated": "true"} + Expect(tamperClient.Update(ctx, current)).To(Succeed()) + Expect(tamperClient.Delete(ctx, current)).To(Succeed()) + Eventually(func() bool { + err := k8sClient.Get(ctx, client.ObjectKey{Name: unmanaged.Name}, &capsulev1beta2.GlobalResourceQuota{}) + + return apierrors.IsNotFound(err) + }, defaultTimeoutInterval, defaultPollInterval).Should(BeTrue()) + }) + + It("uses the owner reference to restore controller drift and keeps status reconciliation working", func() { + controllerClient := impersonationClient(ControllerServiceAccountFull, nil) + current := &capsulev1beta2.GlobalResourceQuota{} + Expect(controllerClient.Get(ctx, quotaKey, current)).To(Succeed()) + + scopeDrift := current.DeepCopy() + delete(scopeDrift.Spec.NamespaceSelectors[0].LabelSelector.MatchLabels, selectorKey) + Expect(controllerClient.Update(ctx, scopeDrift)).To(Succeed()) + Eventually(func(g Gomega) { + reconciled := &capsulev1beta2.GlobalResourceQuota{} + g.Expect(k8sClient.Get(ctx, quotaKey, reconciled)).To(Succeed()) + g.Expect(reconciled.Spec.NamespaceSelectors[0].LabelSelector.MatchLabels). + To(HaveKeyWithValue(selectorKey, "application")) + g.Expect(reconciled.Status.ObservedGeneration).To(Equal(reconciled.Generation)) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + + Expect(controllerClient.Get(ctx, quotaKey, current)).To(Succeed()) + originalGeneration := current.Generation + + drifted := current.DeepCopy() + drifted.Spec.Quota.Hard[corev1.ResourceLimitsCPU] = resource.MustParse("9") + Expect(controllerClient.Update(ctx, drifted)).To(Succeed()) + + Eventually(func(g Gomega) { + reconciled := &capsulev1beta2.GlobalResourceQuota{} + g.Expect(k8sClient.Get(ctx, quotaKey, reconciled)).To(Succeed()) + g.Expect(reconciled.Generation).To(BeNumerically(">", originalGeneration)) + reconciledLimit := reconciled.Spec.Quota.Hard[corev1.ResourceLimitsCPU] + g.Expect(reconciledLimit.Cmp(resource.MustParse("8"))).To(Equal(0)) + g.Expect(reconciled.Status.ObservedGeneration).To(Equal(reconciled.Generation)) + + condition := reconciled.Status.Conditions.GetConditionByType(meta.ReadyCondition) + g.Expect(condition).NotTo(BeNil()) + g.Expect(condition.Status).To(Equal(metav1.ConditionTrue)) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + }) + + It("deletes the managed quota through the controller before the Tenant finalizes", func() { + current := &capsulev1beta2.Tenant{} + Expect(k8sClient.Get(ctx, client.ObjectKey{Name: tenantName}, current)).To(Succeed()) + Expect(current.Finalizers).To(ContainElement(meta.ControllerFinalizer)) + + EventuallyDeletion(tnt) + Eventually(func() bool { + err := k8sClient.Get(ctx, quotaKey, &capsulev1beta2.GlobalResourceQuota{}) + + return apierrors.IsNotFound(err) + }, defaultTimeoutInterval, defaultPollInterval).Should(BeTrue()) + }) + }) + +var _ = Describe("managed GlobalResourceQuota administrator admission", Ordered, + Label("config", "resourcequota", "admission", "managed", "skip-on-openshift"), func() { + const ( + adminName = "e2e-managed-global-quota-admin" + quotaName = "e2e-managed-global-quota-admin" + ) + + ctx := context.Background() + administrator := rbac.UserSpec{Name: adminName, Kind: rbac.UserOwner} + originConfig := &capsulev1beta2.CapsuleConfiguration{} + adminRole := &rbacv1.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{Name: adminName}, + Rules: []rbacv1.PolicyRule{{ + APIGroups: []string{capsulev1beta2.GroupVersion.Group}, + Resources: []string{"globalresourcequotas"}, + Verbs: []string{"get", "create", "update", "patch", "delete"}, + }}, + } + adminBinding := &rbacv1.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{Name: adminName}, + RoleRef: rbacv1.RoleRef{ + APIGroup: rbacv1.GroupName, + Kind: "ClusterRole", + Name: adminName, + }, + Subjects: []rbacv1.Subject{{ + APIGroup: rbacv1.GroupName, + Kind: rbacv1.UserKind, + Name: adminName, + }}, + } + + BeforeAll(func() { + Expect(k8sClient.Get(ctx, client.ObjectKey{Name: defaultConfigurationName}, originConfig)).To(Succeed()) + + EventuallyCreation(func() error { + adminRole.ResourceVersion = "" + + return k8sClient.Create(ctx, adminRole) + }).Should(Succeed()) + EventuallyCreation(func() error { + adminBinding.ResourceVersion = "" + + return k8sClient.Create(ctx, adminBinding) + }).Should(Succeed()) + + ModifyCapsuleConfigurationOpts(func(configuration *capsulev1beta2.CapsuleConfiguration) { + configuration.Spec.Administrators = append(configuration.Spec.Administrators, administrator) + }) + }) + + AfterAll(func() { + controllerClient := impersonationClient(ControllerServiceAccountFull, nil) + quota := &capsulev1beta2.GlobalResourceQuota{ObjectMeta: metav1.ObjectMeta{Name: quotaName}} + Expect(ignoreNotFound(controllerClient.Delete(ctx, quota))).To(Succeed()) + + Eventually(func() error { + current := &capsulev1beta2.CapsuleConfiguration{} + if err := k8sClient.Get(ctx, client.ObjectKey{Name: originConfig.Name}, current); err != nil { + return err + } + + current.Spec = originConfig.Spec + + return k8sClient.Update(ctx, current) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + + EventuallyDeletion(adminBinding) + EventuallyDeletion(adminRole) + }) + + It("allows a configured administrator to create, update, and delete a managed quota", func() { + adminClient := impersonationClient(administrator.Name, nil) + quota := &capsulev1beta2.GlobalResourceQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: quotaName, + Labels: map[string]string{meta.NewManagedByCapsuleLabel: meta.ValueController}, + }, + Spec: capsulev1beta2.GlobalResourceQuotaSpec{ + Quota: corev1.ResourceQuotaSpec{Hard: corev1.ResourceList{ + corev1.ResourcePods: resource.MustParse("10"), + }}, + }, + } + EventuallyCreation(func() error { + quota.ResourceVersion = "" + + return adminClient.Create(ctx, quota) + }).Should(Succeed()) + + current := &capsulev1beta2.GlobalResourceQuota{} + Expect(adminClient.Get(ctx, client.ObjectKey{Name: quotaName}, current)).To(Succeed()) + current.Annotations = map[string]string{"e2e.projectcapsule.dev/admin-updated": "true"} + Expect(adminClient.Update(ctx, current)).To(Succeed()) + Expect(adminClient.Delete(ctx, current)).To(Succeed()) + Eventually(func() bool { + err := k8sClient.Get(ctx, client.ObjectKey{Name: quotaName}, &capsulev1beta2.GlobalResourceQuota{}) + + return apierrors.IsNotFound(err) + }, defaultTimeoutInterval, defaultPollInterval).Should(BeTrue()) + }) + }) diff --git a/e2e/rules_quota_test.go b/e2e/rules_quota_test.go index bd4aaadb..cc365a74 100644 --- a/e2e/rules_quota_test.go +++ b/e2e/rules_quota_test.go @@ -391,8 +391,13 @@ var _ = Describe("rule-generated GlobalResourceQuota", Ordered, Label("resourceq It("rejects invalid rule limits and projects valid changes to every namespace", func() { quotaKey := clientKey("", tenantutils.RuleGlobalResourceQuotaName(tnt, "service-count")) - marker := "e2e.projectcapsule.dev/stable-identity" resourceName := corev1.ResourceServices + initialQuota := &capsulev1beta2.GlobalResourceQuota{} + Eventually(func() error { + return k8sClient.Get(ctx, quotaKey, initialQuota) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + stableUID := initialQuota.UID + setServiceLimit := func(limit string) error { current := &capsulev1beta2.Tenant{} if err := k8sClient.Get(ctx, clientKey("", tenantName), current); err != nil { @@ -424,7 +429,7 @@ var _ = Describe("rule-generated GlobalResourceQuota", Ordered, Label("resourceq Eventually(func(g Gomega) { current := &capsulev1beta2.GlobalResourceQuota{} g.Expect(k8sClient.Get(ctx, quotaKey, current)).To(Succeed()) - g.Expect(current.Annotations).To(HaveKeyWithValue(marker, "preserved")) + g.Expect(current.UID).To(Equal(stableUID)) g.Expect(current.Labels).To(HaveKeyWithValue(meta.RuleQuotaLabel, "service-count")) g.Expect(current.Status.ObservedGeneration).To(Equal(current.Generation)) @@ -457,19 +462,6 @@ var _ = Describe("rule-generated GlobalResourceQuota", Ordered, Label("resourceq } } - Eventually(func() error { - current := &capsulev1beta2.GlobalResourceQuota{} - if err := k8sClient.Get(ctx, quotaKey, current); err != nil { - return err - } - if current.Annotations == nil { - current.Annotations = map[string]string{} - } - current.Annotations[marker] = "preserved" - - return k8sClient.Update(ctx, current) - }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) - // The concurrent admission scenario immediately before this test creates // exactly five Services. Wait for native quota accounting before changing // the shared limit. diff --git a/go.mod b/go.mod index 26e4f98c..f6f453c8 100644 --- a/go.mod +++ b/go.mod @@ -29,6 +29,7 @@ require ( golang.org/x/sync v0.22.0 gomodules.xyz/jsonpatch/v2 v2.5.0 google.golang.org/grpc v1.83.0 + gopkg.in/inf.v0 v0.9.1 k8s.io/api v0.36.3 k8s.io/apiextensions-apiserver v0.36.3 k8s.io/apimachinery v0.36.3 @@ -128,7 +129,6 @@ require ( google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect - gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/cli-runtime v0.36.3 // indirect k8s.io/component-base v0.36.3 // indirect diff --git a/internal/controllers/rbac/manager.go b/internal/controllers/rbac/manager.go index eb782185..4080e2e7 100644 --- a/internal/controllers/rbac/manager.go +++ b/internal/controllers/rbac/manager.go @@ -42,10 +42,14 @@ type Manager struct { Log logr.Logger Client client.Client Configuration configuration.Configuration + + reader client.Reader } //nolint:revive func (r *Manager) SetupWithManager(ctx context.Context, mgr ctrl.Manager, ctrlConfig utils.ControllerOptions) (err error) { + r.reader = mgr.GetAPIReader() + namesPredicate := predicates.LabelsMatching(map[string]string{ meta.CreatedByCapsuleLabel: controllerManager, }) @@ -186,7 +190,16 @@ func (r *Manager) EnsureClusterRoleBindingsProvisioner(ctx context.Context) erro listStarted := time.Now() - if err := r.Client.List(ctx, saList, client.MatchingLabels{ + reader := r.reader + if reader == nil { + reader = r.Client + } + + // ServiceAccount events are sourced from the metadata-only cache. + // Reading full objects from the regular cache here can observe the + // previous label value and permanently lose a promotion/demotion + // reconcile. The API reader provides an authoritative snapshot. + if err := reader.List(ctx, saList, client.MatchingLabels{ meta.OwnerPromotionLabel: meta.ValueTrue, }); err != nil { logOperationDuration(log, "list promoted ServiceAccounts", listStarted) diff --git a/internal/controllers/rbac/manager_test.go b/internal/controllers/rbac/manager_test.go new file mode 100644 index 00000000..12992d55 --- /dev/null +++ b/internal/controllers/rbac/manager_test.go @@ -0,0 +1,136 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package rbac + +import ( + "context" + "testing" + + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/runtime/configuration" +) + +func TestEnsureClusterRoleBindingsProvisionerUsesAuthoritativePromotionState(t *testing.T) { + t.Parallel() + + const ( + configurationName = "capsule" + provisionerRole = "capsule-namespace-provisioner" + namespace = "tenant-a" + serviceAccount = "builder" + ) + + ctx := context.Background() + scheme := runtime.NewScheme() + for _, addToScheme := range []func(*runtime.Scheme) error{ + corev1.AddToScheme, + rbacv1.AddToScheme, + capsulev1beta2.AddToScheme, + } { + if err := addToScheme(scheme); err != nil { + t.Fatal(err) + } + } + + configurationObject := &capsulev1beta2.CapsuleConfiguration{ + ObjectMeta: metav1.ObjectMeta{Name: configurationName}, + Spec: capsulev1beta2.CapsuleConfigurationSpec{ + AllowServiceAccountPromotion: true, + RBAC: &capsulev1beta2.RBACConfiguration{ + ProvisionerClusterRole: provisionerRole, + }, + }, + } + promoted := &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: serviceAccount, + Namespace: namespace, + Labels: map[string]string{ + meta.OwnerPromotionLabel: meta.ValueTrue, + }, + }, + } + + // The cached client deliberately remains stale after demotion, reproducing + // the ordering between the metadata-only watch and full-object cache. + cachedClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(configurationObject, promoted.DeepCopy()). + Build() + authoritativeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(promoted.DeepCopy()). + Build() + cfg := configuration.NewCapsuleConfiguration( + ctx, + cachedClient, + cachedClient, + nil, + configurationName, + ) + manager := &Manager{ + Log: logr.Discard(), + Client: cachedClient, + Configuration: cfg, + reader: authoritativeClient, + } + + if err := manager.EnsureClusterRoleBindingsProvisioner(ctx); err != nil { + t.Fatalf("initial provisioner binding reconciliation: %v", err) + } + assertServiceAccountSubject(t, cachedClient, provisionerRole, namespace, serviceAccount, true) + + latest := &corev1.ServiceAccount{} + if err := authoritativeClient.Get(ctx, client.ObjectKeyFromObject(promoted), latest); err != nil { + t.Fatalf("get authoritative ServiceAccount: %v", err) + } + latest.Labels[meta.OwnerPromotionLabel] = "false" + if err := authoritativeClient.Update(ctx, latest); err != nil { + t.Fatalf("demote authoritative ServiceAccount: %v", err) + } + + if err := manager.EnsureClusterRoleBindingsProvisioner(ctx); err != nil { + t.Fatalf("demotion provisioner binding reconciliation: %v", err) + } + assertServiceAccountSubject(t, cachedClient, provisionerRole, namespace, serviceAccount, false) +} + +func assertServiceAccountSubject( + t *testing.T, + kubeClient client.Client, + bindingName string, + namespace string, + serviceAccount string, + want bool, +) { + t.Helper() + + binding := &rbacv1.ClusterRoleBinding{} + if err := kubeClient.Get(context.Background(), client.ObjectKey{Name: bindingName}, binding); err != nil { + t.Fatalf("get ClusterRoleBinding: %v", err) + } + + found := false + for _, subject := range binding.Subjects { + if subject.Kind == rbacv1.ServiceAccountKind && + subject.Namespace == namespace && + subject.Name == serviceAccount { + found = true + break + } + } + + if found != want { + t.Fatalf("ServiceAccount subject presence = %t, want %t; subjects: %#v", found, want, binding.Subjects) + } +} diff --git a/internal/controllers/tenant/globalresourcequotas.go b/internal/controllers/tenant/globalresourcequotas.go index 382b5f01..60d2c250 100644 --- a/internal/controllers/tenant/globalresourcequotas.go +++ b/internal/controllers/tenant/globalresourcequotas.go @@ -96,7 +96,13 @@ func (r *Manager) pruneGlobalResourceQuotas( meta.NewManagedByCapsuleLabel: meta.ValueController, meta.NewTenantLabel: tnt.Name, }) - if err := r.List(ctx, list, &client.ListOptions{LabelSelector: selector}); err != nil { + + reader := client.Reader(r.Client) + if r.reader != nil { + reader = r.reader + } + + if err := reader.List(ctx, list, &client.ListOptions{LabelSelector: selector}); err != nil { return err } @@ -117,3 +123,13 @@ func (r *Manager) pruneGlobalResourceQuotas( return nil } + +func hasRuleGlobalResourceQuotas(tnt *capsulev1beta2.Tenant) bool { + for _, rule := range tnt.Spec.Rules { + if rule != nil && rule.NamespaceRuleBodyNamespace != nil && len(rule.Quota) > 0 { + return true + } + } + + return false +} diff --git a/internal/controllers/tenant/globalresourcequotas_test.go b/internal/controllers/tenant/globalresourcequotas_test.go index af8a6c9f..945d7de7 100644 --- a/internal/controllers/tenant/globalresourcequotas_test.go +++ b/internal/controllers/tenant/globalresourcequotas_test.go @@ -59,6 +59,16 @@ func TestSyncGlobalResourceQuotasGeneratesAndPrunesRuleQuotas(t *testing.T) { if generated.Labels[meta.RuleQuotaLabel] != "shared-compute" { t.Fatalf("rule quota label = %q, want shared-compute", generated.Labels[meta.RuleQuotaLabel]) } + if generated.Labels[meta.NewManagedByCapsuleLabel] != meta.ValueController { + t.Fatalf( + "managed-by label = %q, want %q", + generated.Labels[meta.NewManagedByCapsuleLabel], + meta.ValueController, + ) + } + if !metav1.IsControlledBy(generated, tnt) { + t.Fatalf("generated GlobalResourceQuota is not controlled by Tenant %q", tnt.Name) + } selector := generated.Spec.NamespaceSelectors[0].LabelSelector if selector.MatchLabels[meta.TenantLabel] != tnt.Name || selector.MatchLabels["tier"] != "paid" { t.Fatalf("generated selector = %#v", selector) diff --git a/internal/controllers/tenant/manager.go b/internal/controllers/tenant/manager.go index 4cf60380..dc4da2e1 100644 --- a/internal/controllers/tenant/manager.go +++ b/internal/controllers/tenant/manager.go @@ -367,6 +367,15 @@ func (r *Manager) reconcile(ctx context.Context, log logr.Logger, instance *caps errs = append(errs, fmt.Errorf("namespace(s) had reconciliation errors: %w", err)) } + // The managed-resource webhook intentionally denies deletion by the + // garbage collector. Remove rule-generated cluster-scoped children as + // the Capsule controller before releasing the Tenant finalizer. + if err = r.pruneGlobalResourceQuotas(ctx, instance, map[string]struct{}{}); err != nil { + errs = append(errs, fmt.Errorf("cannot delete rule global resource quotas: %w", err)) + + return errors.Join(errs...) + } + if err = r.ensureMetadata(ctx, instance); err != nil { errs = append(errs, fmt.Errorf("cannot ensure metadata: %w", err)) } diff --git a/internal/controllers/tenant/metadata.go b/internal/controllers/tenant/metadata.go index b2fd899a..dcd0235c 100644 --- a/internal/controllers/tenant/metadata.go +++ b/internal/controllers/tenant/metadata.go @@ -22,7 +22,11 @@ func (r *Manager) ensureMetadata(ctx context.Context, tnt *capsulev1beta2.Tenant tnt.Labels[meta.TenantNameLabel] = tnt.Name } - if len(tnt.Status.Spaces) == 0 { + // Rule-generated GlobalResourceQuotas are cluster scoped and protected from + // garbage-collector deletion by the managed-resource webhook. Keep the + // Tenant around until its controller can explicitly remove those children. + keepForRuleQuotas := tnt.DeletionTimestamp == nil && hasRuleGlobalResourceQuotas(tnt) + if len(tnt.Status.Spaces) == 0 && !keepForRuleQuotas { controllerutil.RemoveFinalizer(tnt, meta.ControllerFinalizer) } else { controllerutil.AddFinalizer(tnt, meta.ControllerFinalizer) diff --git a/internal/controllers/tenant/metadata_test.go b/internal/controllers/tenant/metadata_test.go new file mode 100644 index 00000000..6f495190 --- /dev/null +++ b/internal/controllers/tenant/metadata_test.go @@ -0,0 +1,86 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package tenant + +import ( + "context" + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rules" +) + +func TestEnsureMetadataKeepsFinalizerForRuleGlobalResourceQuotas(t *testing.T) { + t.Parallel() + + tnt := &capsulev1beta2.Tenant{ + ObjectMeta: metav1.ObjectMeta{Name: "tenant-a"}, + Spec: capsulev1beta2.TenantSpec{Rules: []*rules.NamespaceRuleBodyTenant{{ + NamespaceRuleBodyNamespace: &rules.NamespaceRuleBodyNamespace{ + Quota: []rules.ResourceQuotaRule{{Name: "compute"}}, + }, + }}}, + } + manager := &Manager{} + + if err := manager.ensureMetadata(context.Background(), tnt); err != nil { + t.Fatalf("ensureMetadata() error = %v", err) + } + if !controllerutil.ContainsFinalizer(tnt, meta.ControllerFinalizer) { + t.Fatal("Tenant with rule-generated GlobalResourceQuota is missing the controller finalizer") + } + + now := metav1.Now() + tnt.DeletionTimestamp = &now + if err := manager.ensureMetadata(context.Background(), tnt); err != nil { + t.Fatalf("ensureMetadata() while deleting error = %v", err) + } + if controllerutil.ContainsFinalizer(tnt, meta.ControllerFinalizer) { + t.Fatal("Tenant controller finalizer was retained after managed child cleanup") + } +} + +func TestHasRuleGlobalResourceQuotas(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + rules []*rules.NamespaceRuleBodyTenant + want bool + }{ + {name: "no rules"}, + {name: "nil rule", rules: []*rules.NamespaceRuleBodyTenant{nil}}, + {name: "rule without namespace body", rules: []*rules.NamespaceRuleBodyTenant{{}}}, + { + name: "rule without quota", + rules: []*rules.NamespaceRuleBodyTenant{{ + NamespaceRuleBodyNamespace: &rules.NamespaceRuleBodyNamespace{}, + }}, + }, + { + name: "rule with quota", + rules: []*rules.NamespaceRuleBodyTenant{{ + NamespaceRuleBodyNamespace: &rules.NamespaceRuleBodyNamespace{ + Quota: []rules.ResourceQuotaRule{{Name: "compute"}}, + }, + }}, + want: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + tnt := &capsulev1beta2.Tenant{Spec: capsulev1beta2.TenantSpec{Rules: test.rules}} + if got := hasRuleGlobalResourceQuotas(tnt); got != test.want { + t.Fatalf("hasRuleGlobalResourceQuotas() = %t, want %t", got, test.want) + } + }) + } +} diff --git a/internal/controllers/tls/manager.go b/internal/controllers/tls/manager.go index f7141c11..8d94b2e1 100644 --- a/internal/controllers/tls/manager.go +++ b/internal/controllers/tls/manager.go @@ -165,51 +165,24 @@ func (r *Reconciler) ReconcileCertificates( "ipAddresses", cert.IPsToStrings(sans.IPAddrs), ) - ca, caBundle, rotateServingCert, err := r.ensureCertificateMaterial(log, certSecret, sans) - if err != nil { + if err := r.reconcileTLSSecret(ctx, log, certSecret, sans); err != nil { return err } - log.V(4).Info( - "certificate requires rotation", - "rotation", rotateServingCert, - ) - - if rotateServingCert { - if ca == nil { - return fmt.Errorf("cannot rotate serving certificate without CA private key") - } - - crt, key, err := ca.GenerateCertificate(cert.NewCertOpts( - time.Now().Add(certificateValidity), - sans, - )) - if err != nil { - log.Error(err, "cannot generate serving TLS certificate") - - return err - } - - certSecret.Data[corev1.TLSCertKey] = crt.Bytes() - certSecret.Data[corev1.TLSPrivateKeyKey] = key.Bytes() - - if err := r.validateSecretCertificate(certSecret, sans); err != nil { - return err - } - - if err := r.upsertTLSSecret(ctx, certSecret); err != nil { - return err - } - } - - caBundle = certSecret.Data[corev1.ServiceAccountRootCAKey] + caBundle := certSecret.Data[corev1.ServiceAccountRootCAKey] if len(caBundle) == 0 { return fmt.Errorf("missing %q field in %q secret", corev1.ServiceAccountRootCAKey, r.Configuration.TLSSecretName()) } - log.V(5).Info("Patching caBundle in managed CRD conversions") + log.V(5).Info("Patching caBundle in admission webhooks and managed CRD conversions") patchGroup, groupCtx := errgroup.WithContext(ctx) + patchGroup.Go(func() error { + return r.patchMutatingWebhookConfigurationCABundle(groupCtx, caBundle) + }) + patchGroup.Go(func() error { + return r.patchValidatingWebhookConfigurationCABundle(groupCtx, caBundle) + }) for key, managed := range r.conversionManagedCRDs() { patchGroup.Go(func() error { @@ -224,15 +197,108 @@ func (r *Reconciler) ReconcileCertificates( return patchGroup.Wait() } +// reconcileTLSSecret calculates certificate material from the latest persisted +// Secret inside an optimistic-concurrency retry. This is deliberately not based +// on the object supplied by the caller: every controller replica performs the +// startup reconciliation before leader election, so that object may already be +// stale by the time certificate generation finishes. +func (r *Reconciler) reconcileTLSSecret( + ctx context.Context, + log logr.Logger, + certSecret *corev1.Secret, + sans cert.CertificateSANs, +) error { + key := client.ObjectKeyFromObject(certSecret) + if key.Name == "" { + key.Name = r.Configuration.TLSSecretName() + } + + if key.Namespace == "" { + key.Namespace = r.Namespace + } + + err := retry.OnError(retry.DefaultBackoff, func(err error) bool { + return apierrors.IsAlreadyExists(err) || apierrors.IsConflict(err) + }, func() error { + // Use the TLS type when Capsule creates the Secret itself. If the Secret + // already exists, CreateOrUpdate loads its persisted type into desired; + // preserve it because Secret type is immutable. In particular, the Helm + // chart pre-creates an empty Opaque Secret so the controller Pod can mount + // it before this reconciliation supplies the certificate data. + desired := &corev1.Secret{ + ObjectMeta: *certSecret.ObjectMeta.DeepCopy(), + Type: corev1.SecretTypeTLS, + } + desired.Name = key.Name + desired.Namespace = key.Namespace + + _, err := controllerutil.CreateOrUpdate(ctx, r.Client, desired, func() error { + if desired.Labels == nil { + desired.Labels = map[string]string{} + } + + if desired.Annotations == nil { + desired.Annotations = map[string]string{} + } + + ca, _, rotateServingCert, err := r.ensureCertificateMaterial(log, desired, sans) + if err != nil { + return err + } + + log.V(4).Info( + "certificate requires rotation", + "rotation", rotateServingCert, + ) + + if rotateServingCert { + if ca == nil { + return fmt.Errorf("cannot rotate serving certificate without CA private key") + } + + crt, key, err := ca.GenerateCertificate(cert.NewCertOpts( + time.Now().Add(certificateValidity), + sans, + )) + if err != nil { + return fmt.Errorf("generate serving TLS certificate: %w", err) + } + + desired.Data[corev1.TLSCertKey] = crt.Bytes() + desired.Data[corev1.TLSPrivateKeyKey] = key.Bytes() + } + + return r.validateSecretCertificate(desired, sans) + }) + if err != nil { + return err + } + + certSecret.ObjectMeta = desired.ObjectMeta + certSecret.Type = desired.Type + certSecret.Immutable = desired.Immutable + certSecret.Data = copySecretData(desired.Data) + + return nil + }) + if err != nil { + r.Log.Error(err, "cannot reconcile Capsule TLS Secret", "secret", key.String()) + } + + return err +} + // ensureCertificateMaterial ensures that the Secret contains a stable CA // certificate/key pair and decides whether the serving certificate must be // regenerated. // // Important behavior: -// - Missing Secret or missing ca.key creates a new CA. +// - Only a new, empty Secret bootstraps a CA. // - Existing valid CA is reused. // - Serving certificate renewal never rotates the CA. -// - Legacy Secrets without ca.key rotate once into the stable format. +// - Established or externally managed CA material is never replaced +// automatically, because doing so would immediately invalidate every +// published caBundle. func (r *Reconciler) ensureCertificateMaterial( log logr.Logger, certSecret *corev1.Secret, @@ -262,53 +328,51 @@ func (r *Reconciler) ensureCertificateMaterial( case hasCABundle && hasCAKey: loadedCA, err := cert.NewCertificateAuthorityFromBytes(caBundle, caKey) if err != nil { - log.V(3).Info( - "Existing CA material is invalid, generating new CA", - "error", err.Error(), + return nil, nil, false, fmt.Errorf( + "TLS Secret %s contains invalid CA certificate/key material; refusing automatic CA replacement: %w", + client.ObjectKeyFromObject(certSecret).String(), + err, ) - - generatedCA, generatedCABundle, generatedCAKey, err := generateCertificateAuthorityMaterial() - if err != nil { - return nil, nil, false, err - } - - ca = generatedCA - caBundle = generatedCABundle - - certSecret.Data[corev1.ServiceAccountRootCAKey] = generatedCABundle - certSecret.Data["ca.key"] = generatedCAKey - - rotateServingCert = true - } else { - ca = loadedCA } + ca = loadedCA + case hasCABundle && !hasCAKey: - // A CA bundle without its private key is typical of an externally - // managed Secret. Once the Capsule TLS controller is enabled, it must - // take ownership of the complete certificate lifecycle; otherwise a - // SAN change or certificate renewal cannot be recovered. + // This is an externally managed or legacy Secret. It is safe to keep + // serving while its certificate remains valid, but Capsule cannot renew + // it without the CA key. Replacing that CA in-place would make the API + // server distrust one or more running webhook replicas during rollout. + if err := r.validateSecretCertificate(certSecret, sans); err != nil { + return nil, nil, false, fmt.Errorf( + "TLS Secret %s has no CA private key and its serving certificate needs renewal; refusing automatic CA replacement: %w", + client.ObjectKeyFromObject(certSecret).String(), + err, + ) + } + log.V(3).Info( - "TLS Secret contains CA bundle but no CA private key, rotating into controller-managed TLS material", + "Keeping externally managed TLS material without a CA private key", "secret", client.ObjectKeyFromObject(certSecret).String(), ) - generatedCA, generatedCABundle, generatedCAKey, err := generateCertificateAuthorityMaterial() - if err != nil { - return nil, nil, false, err - } + return nil, caBundle, false, nil - ca = generatedCA - caBundle = generatedCABundle - - certSecret.Data[corev1.ServiceAccountRootCAKey] = generatedCABundle - certSecret.Data["ca.key"] = generatedCAKey - - rotateServingCert = true + case !hasCABundle && hasCAKey: + return nil, nil, false, fmt.Errorf( + "TLS Secret %s contains a CA private key but no CA certificate; refusing automatic CA replacement", + client.ObjectKeyFromObject(certSecret).String(), + ) default: + if len(certSecret.Data[corev1.TLSCertKey]) > 0 || len(certSecret.Data[corev1.TLSPrivateKeyKey]) > 0 { + return nil, nil, false, fmt.Errorf( + "TLS Secret %s contains serving certificate material but no CA certificate; refusing automatic CA replacement", + client.ObjectKeyFromObject(certSecret).String(), + ) + } + log.V(10).Info( - "TLS Secret is missing CA material, generating new CA", + "TLS Secret is empty, generating initial CA", "secret", client.ObjectKeyFromObject(certSecret).String(), ) @@ -411,35 +475,103 @@ func generateCertificateAuthorityMaterial() (*cert.CapsuleCA, []byte, []byte, er return ca, caCrt.Bytes(), caKey.Bytes(), nil } -func (r *Reconciler) upsertTLSSecret(ctx context.Context, certSecret *corev1.Secret) error { - desired := &corev1.Secret{ - ObjectMeta: certSecret.ObjectMeta, - Type: corev1.SecretTypeTLS, - } +func (r *Reconciler) patchMutatingWebhookConfigurationCABundle(ctx context.Context, caBundle []byte) error { + return r.patchAdmissionConfigurationCABundle( + ctx, + r.Configuration.MutatingWebhookConfigurationName(), + caBundle, + func() client.Object { return &admissionregistrationv1.MutatingWebhookConfiguration{} }, + updateMutatingWebhookCABundles, + ) +} - _, err := controllerutil.CreateOrUpdate(ctx, r.Client, desired, func() error { - if desired.Labels == nil { - desired.Labels = map[string]string{} - } - - if desired.Annotations == nil { - desired.Annotations = map[string]string{} - } - - desired.Data = copySecretData(certSecret.Data) +func (r *Reconciler) patchValidatingWebhookConfigurationCABundle(ctx context.Context, caBundle []byte) error { + return r.patchAdmissionConfigurationCABundle( + ctx, + r.Configuration.ValidatingWebhookConfigurationName(), + caBundle, + func() client.Object { return &admissionregistrationv1.ValidatingWebhookConfiguration{} }, + updateValidatingWebhookCABundles, + ) +} +func (r *Reconciler) patchAdmissionConfigurationCABundle( + ctx context.Context, + name string, + caBundle []byte, + newConfiguration func() client.Object, + updateCABundles func(client.Object, []byte) (bool, error), +) error { + if name == "" { return nil - }) - if err != nil { - r.Log.Error(err, "cannot update Capsule TLS Secret") - - return err } - certSecret.ObjectMeta = desired.ObjectMeta - certSecret.Data = copySecretData(desired.Data) + return retry.RetryOnConflict(retry.DefaultBackoff, func() error { + configuration := newConfiguration() + if err := r.Get(ctx, types.NamespacedName{Name: name}, configuration); err != nil { + if apierrors.IsNotFound(err) { + return nil + } - return nil + return err + } + + before, ok := configuration.DeepCopyObject().(client.Object) + if !ok { + return fmt.Errorf("admission configuration %q cannot be deep-copied as a client object", name) + } + + changed, err := updateCABundles(configuration, caBundle) + if err != nil { + return err + } + + if !changed { + return nil + } + + return r.Patch(ctx, configuration, client.MergeFrom(before)) + }) +} + +func updateMutatingWebhookCABundles(object client.Object, caBundle []byte) (bool, error) { + configuration, ok := object.(*admissionregistrationv1.MutatingWebhookConfiguration) + if !ok { + return false, fmt.Errorf("expected MutatingWebhookConfiguration, got %T", object) + } + + changed := false + + for index := range configuration.Webhooks { + if bytes.Equal(configuration.Webhooks[index].ClientConfig.CABundle, caBundle) { + continue + } + + configuration.Webhooks[index].ClientConfig.CABundle = append([]byte(nil), caBundle...) + changed = true + } + + return changed, nil +} + +func updateValidatingWebhookCABundles(object client.Object, caBundle []byte) (bool, error) { + configuration, ok := object.(*admissionregistrationv1.ValidatingWebhookConfiguration) + if !ok { + return false, fmt.Errorf("expected ValidatingWebhookConfiguration, got %T", object) + } + + changed := false + + for index := range configuration.Webhooks { + if bytes.Equal(configuration.Webhooks[index].ClientConfig.CABundle, caBundle) { + continue + } + + configuration.Webhooks[index].ClientConfig.CABundle = append([]byte(nil), caBundle...) + changed = true + } + + return changed, nil } func (r *Reconciler) validateSecretCertificate( diff --git a/internal/controllers/tls/manager_test.go b/internal/controllers/tls/manager_test.go index 2c116fa1..7fd42a05 100644 --- a/internal/controllers/tls/manager_test.go +++ b/internal/controllers/tls/manager_test.go @@ -6,10 +6,12 @@ package tls import ( "bytes" "context" + "strings" "testing" "time" "github.com/go-logr/logr" + admissionregistrationv1 "k8s.io/api/admissionregistration/v1" corev1 "k8s.io/api/core/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -18,123 +20,298 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/fake" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + apiMeta "github.com/projectcapsule/capsule/pkg/api/meta" + runtimeadmission "github.com/projectcapsule/capsule/pkg/runtime/admission" "github.com/projectcapsule/capsule/pkg/runtime/cert" "github.com/projectcapsule/capsule/pkg/runtime/configuration" ) -func TestReconcileCertificatesMigratesExternalTLSSecret(t *testing.T) { +const ( + testNamespace = "capsule-system" + testSecretName = "capsule-tls" + testServiceName = "capsule-webhook-service" + testMutatingConfiguration = "capsule-mutating-webhook-configuration" + testValidatingConfiguration = "capsule-validating-webhook-configuration" +) + +func TestReconcileCertificatesPreservesValidExternalTLSSecret(t *testing.T) { t.Parallel() - const ( - namespace = "capsule-system" - secretName = "capsule-tls" - serviceName = "capsule-webhook-service" - ) + externalCABundle, externalCertificate, externalKey := generateTestTLSMaterial(t, testWebhookSANs()) + secret := testTLSSecret(externalCABundle, externalCertificate, externalKey) + reconciler, kubeClient := newTestTLSReconciler(t, secret) + + if err := reconciler.ReconcileCertificates(context.Background(), logr.Discard(), secret.DeepCopy()); err != nil { + t.Fatalf("ReconcileCertificates() error = %v", err) + } + + updated := getTestTLSSecret(t, kubeClient) + if len(updated.Data["ca.key"]) != 0 { + t.Fatal("external TLS Secret unexpectedly gained a CA private key") + } + + assertTLSDataEqual(t, updated, externalCABundle, externalCertificate, externalKey) +} + +func TestReconcileCertificatesRejectsUnsafeExternalCARotation(t *testing.T) { + t.Parallel() - ctx := context.Background() externalCABundle, externalCertificate, externalKey := generateTestTLSMaterial(t, cert.CertificateSANs{ - DNSNames: []string{serviceName + "." + namespace + ".svc"}, + DNSNames: []string{testServiceName + "." + testNamespace + ".svc"}, }) + secret := testTLSSecret(externalCABundle, externalCertificate, externalKey) + reconciler, kubeClient := newTestTLSReconciler(t, secret) - secret := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: secretName, - Namespace: namespace, - }, - Type: corev1.SecretTypeTLS, - Data: map[string][]byte{ - corev1.ServiceAccountRootCAKey: externalCABundle, - corev1.TLSCertKey: externalCertificate, - corev1.TLSPrivateKeyKey: externalKey, + err := reconciler.ReconcileCertificates(context.Background(), logr.Discard(), secret.DeepCopy()) + if err == nil { + t.Fatal("ReconcileCertificates() unexpectedly replaced externally managed CA material") + } + if !strings.Contains(err.Error(), "refusing automatic CA replacement") { + t.Fatalf("ReconcileCertificates() error = %v, want safe CA replacement refusal", err) + } + + updated := getTestTLSSecret(t, kubeClient) + if len(updated.Data["ca.key"]) != 0 { + t.Fatal("rejected external TLS Secret unexpectedly gained a CA private key") + } + + assertTLSDataEqual(t, updated, externalCABundle, externalCertificate, externalKey) +} + +func TestReconcileCertificatesConcurrentReplicasAdoptPersistedWinner(t *testing.T) { + t.Parallel() + + secret := testTLSSecret(nil, nil, nil) + reconciler, kubeClient := newTestTLSReconciler(t, secret) + firstReplica := secret.DeepCopy() + secondReplica := secret.DeepCopy() + start := make(chan struct{}) + errors := make(chan error, 2) + + for _, replica := range []*corev1.Secret{firstReplica, secondReplica} { + go func(replica *corev1.Secret) { + <-start + errors <- reconciler.ReconcileCertificates(context.Background(), logr.Discard(), replica) + }(replica) + } + + close(start) + for range 2 { + if err := <-errors; err != nil { + t.Fatalf("concurrent ReconcileCertificates() error = %v", err) + } + } + + finalPersisted := getTestTLSSecret(t, kubeClient) + if !bytes.Equal( + firstReplica.Data[corev1.ServiceAccountRootCAKey], + finalPersisted.Data[corev1.ServiceAccountRootCAKey], + ) { + t.Fatal("first replica did not adopt the persisted CA") + } + if !bytes.Equal( + secondReplica.Data[corev1.ServiceAccountRootCAKey], + finalPersisted.Data[corev1.ServiceAccountRootCAKey], + ) { + t.Fatal("second replica did not adopt the persisted CA") + } + if !bytes.Equal(firstReplica.Data[corev1.TLSCertKey], finalPersisted.Data[corev1.TLSCertKey]) || + !bytes.Equal(secondReplica.Data[corev1.TLSCertKey], finalPersisted.Data[corev1.TLSCertKey]) { + t.Fatal("concurrent replicas did not adopt the persisted serving certificate") + } +} + +func TestReconcileCertificatesPopulatesChartCreatedOpaqueSecret(t *testing.T) { + t.Parallel() + + // The Helm chart must create this empty Secret before the controller Pod can + // mount it. Kubernetes does not allow changing a Secret's type afterwards, + // so reconciliation must preserve Opaque while adding valid TLS material. + secret := testTLSSecret(nil, nil, nil) + secret.Type = corev1.SecretTypeOpaque + reconciler, kubeClient := newTestTLSReconciler(t, secret) + + if err := reconciler.ReconcileCertificates(context.Background(), logr.Discard(), secret.DeepCopy()); err != nil { + t.Fatalf("ReconcileCertificates() error = %v", err) + } + + updated := getTestTLSSecret(t, kubeClient) + if updated.Type != corev1.SecretTypeOpaque { + t.Fatalf("reconciled Secret type = %q, want %q", updated.Type, corev1.SecretTypeOpaque) + } + if len(updated.Data[corev1.ServiceAccountRootCAKey]) == 0 || + len(updated.Data[corev1.TLSCertKey]) == 0 || + len(updated.Data[corev1.TLSPrivateKeyKey]) == 0 { + t.Fatal("reconciled Secret is missing generated TLS material") + } +} + +func TestReconcileCertificatesPatchesEveryAdmissionCABundle(t *testing.T) { + t.Parallel() + + oldCA, _, _ := generateTestTLSMaterial(t, testWebhookSANs()) + secret := testTLSSecret(nil, nil, nil) + mutating := &admissionregistrationv1.MutatingWebhookConfiguration{ + ObjectMeta: metav1.ObjectMeta{Name: testMutatingConfiguration}, + Webhooks: []admissionregistrationv1.MutatingWebhook{ + {Name: "first.mutating.projectcapsule.dev", ClientConfig: admissionregistrationv1.WebhookClientConfig{CABundle: oldCA}}, + {Name: "second.mutating.projectcapsule.dev"}, }, } + validating := &admissionregistrationv1.ValidatingWebhookConfiguration{ + ObjectMeta: metav1.ObjectMeta{Name: testValidatingConfiguration}, + Webhooks: []admissionregistrationv1.ValidatingWebhook{ + {Name: "first.validating.projectcapsule.dev", ClientConfig: admissionregistrationv1.WebhookClientConfig{CABundle: oldCA}}, + {Name: "second.validating.projectcapsule.dev"}, + }, + } + reconciler, kubeClient := newTestTLSReconciler(t, secret, mutating, validating) + + if err := reconciler.ReconcileCertificates(context.Background(), logr.Discard(), secret.DeepCopy()); err != nil { + t.Fatalf("ReconcileCertificates() error = %v", err) + } + + wantCA := getTestTLSSecret(t, kubeClient).Data[corev1.ServiceAccountRootCAKey] + updatedMutating := &admissionregistrationv1.MutatingWebhookConfiguration{} + if err := kubeClient.Get( + context.Background(), + client.ObjectKey{Name: testMutatingConfiguration}, + updatedMutating, + ); err != nil { + t.Fatalf("get mutating webhook configuration: %v", err) + } + for index := range updatedMutating.Webhooks { + if !bytes.Equal(updatedMutating.Webhooks[index].ClientConfig.CABundle, wantCA) { + t.Fatalf("mutating webhook %q has stale caBundle", updatedMutating.Webhooks[index].Name) + } + } + + updatedValidating := &admissionregistrationv1.ValidatingWebhookConfiguration{} + if err := kubeClient.Get( + context.Background(), + client.ObjectKey{Name: testValidatingConfiguration}, + updatedValidating, + ); err != nil { + t.Fatalf("get validating webhook configuration: %v", err) + } + for index := range updatedValidating.Webhooks { + if !bytes.Equal(updatedValidating.Webhooks[index].ClientConfig.CABundle, wantCA) { + t.Fatalf("validating webhook %q has stale caBundle", updatedValidating.Webhooks[index].Name) + } + } +} + +func newTestTLSReconciler(t *testing.T, objects ...client.Object) (*Reconciler, client.Client) { + t.Helper() configurationObject := &capsulev1beta2.CapsuleConfiguration{ ObjectMeta: metav1.ObjectMeta{Name: "capsule"}, Spec: capsulev1beta2.CapsuleConfigurationSpec{ EnableTLSReconciler: true, CapsuleResources: capsulev1beta2.CapsuleResources{ - TLSSecretName: secretName, + TLSSecretName: testSecretName, }, Admission: capsulev1beta2.DynamicAdmission{ - ServiceName: serviceName, - Mutating: &capsulev1beta2.DynamicMutatingAdmissionConfig{}, - Validating: &capsulev1beta2.DynamicValidatingAdmissionConfig{}, + ServiceName: testServiceName, + Mutating: &capsulev1beta2.DynamicMutatingAdmissionConfig{ + DynamicAdmissionConfig: runtimeadmission.DynamicAdmissionConfig{ + Name: apiMeta.RFC1123Name(testMutatingConfiguration), + }, + }, + Validating: &capsulev1beta2.DynamicValidatingAdmissionConfig{ + DynamicAdmissionConfig: runtimeadmission.DynamicAdmissionConfig{ + Name: apiMeta.RFC1123Name(testValidatingConfiguration), + }, + }, }, }, } scheme := runtime.NewScheme() - if err := corev1.AddToScheme(scheme); err != nil { - t.Fatal(err) + for _, addToScheme := range []func(*runtime.Scheme) error{ + corev1.AddToScheme, + admissionregistrationv1.AddToScheme, + apiextensionsv1.AddToScheme, + capsulev1beta2.AddToScheme, + } { + if err := addToScheme(scheme); err != nil { + t.Fatal(err) + } } - if err := apiextensionsv1.AddToScheme(scheme); err != nil { - t.Fatal(err) + objects = append(objects, configurationObject) + kubeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build() + cfg := configuration.NewCapsuleConfiguration( + context.Background(), + kubeClient, + kubeClient, + nil, + configurationObject.Name, + ) + + return &Reconciler{ + Client: kubeClient, + Log: logr.Discard(), + Namespace: testNamespace, + Configuration: cfg, + }, kubeClient +} + +func testTLSSecret(caBundle, certificate, key []byte) *corev1.Secret { + data := map[string][]byte{} + if len(caBundle) > 0 { + data[corev1.ServiceAccountRootCAKey] = caBundle + } + if len(certificate) > 0 { + data[corev1.TLSCertKey] = certificate + } + if len(key) > 0 { + data[corev1.TLSPrivateKeyKey] = key } - if err := capsulev1beta2.AddToScheme(scheme); err != nil { - t.Fatal(err) + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: testSecretName, Namespace: testNamespace}, + Type: corev1.SecretTypeTLS, + Data: data, } +} - kubeClient := fake.NewClientBuilder(). - WithScheme(scheme). - WithObjects(secret, configurationObject). - Build() +func getTestTLSSecret(t *testing.T, kubeClient client.Client) *corev1.Secret { + t.Helper() - reconciler := &Reconciler{ - Client: kubeClient, - Log: logr.Discard(), - Namespace: namespace, - Configuration: configuration.NewCapsuleConfiguration( - ctx, - kubeClient, - kubeClient, - nil, - configurationObject.Name, - ), - } - - desiredSANs, err := reconciler.desiredWebhookSANs(ctx) - if err != nil { - t.Fatalf("desiredWebhookSANs() error = %v", err) - } - - if err := reconciler.validateSecretCertificate(secret, desiredSANs); err == nil { - t.Fatal("external serving certificate unexpectedly satisfies all desired SANs") - } - - if err := reconciler.ReconcileCertificates(ctx, logr.Discard(), secret.DeepCopy()); err != nil { - t.Fatalf("ReconcileCertificates() error = %v", err) - } - - updated := &corev1.Secret{} - if err := kubeClient.Get(ctx, client.ObjectKeyFromObject(secret), updated); err != nil { + secret := &corev1.Secret{} + if err := kubeClient.Get( + context.Background(), + client.ObjectKey{Namespace: testNamespace, Name: testSecretName}, + secret, + ); err != nil { t.Fatalf("get reconciled TLS Secret: %v", err) } - if len(updated.Data["ca.key"]) == 0 { - t.Fatal("reconciled TLS Secret does not contain ca.key") - } + return secret +} - if bytes.Equal(updated.Data[corev1.ServiceAccountRootCAKey], externalCABundle) { - t.Fatal("reconciled TLS Secret retained the external CA bundle") - } +func assertTLSDataEqual(t *testing.T, secret *corev1.Secret, caBundle, certificate, key []byte) { + t.Helper() - if bytes.Equal(updated.Data[corev1.TLSCertKey], externalCertificate) { - t.Fatal("reconciled TLS Secret retained the external serving certificate") + if !bytes.Equal(secret.Data[corev1.ServiceAccountRootCAKey], caBundle) { + t.Fatal("TLS Secret CA bundle changed") } + if !bytes.Equal(secret.Data[corev1.TLSCertKey], certificate) { + t.Fatal("TLS Secret serving certificate changed") + } + if !bytes.Equal(secret.Data[corev1.TLSPrivateKeyKey], key) { + t.Fatal("TLS Secret serving private key changed") + } +} - if _, err := cert.NewCertificateAuthorityFromBytes( - updated.Data[corev1.ServiceAccountRootCAKey], - updated.Data["ca.key"], - ); err != nil { - t.Fatalf("reconciled CA certificate/key pair is invalid: %v", err) - } - - if err := reconciler.validateSecretCertificate(updated, desiredSANs); err != nil { - t.Fatalf("reconciled serving certificate is invalid: %v", err) - } +func testWebhookSANs() cert.CertificateSANs { + return cert.CertificateSANs{DNSNames: []string{ + testServiceName, + testServiceName + "." + testNamespace, + testServiceName + "." + testNamespace + ".svc", + testServiceName + "." + testNamespace + ".svc.cluster.local", + }} } func generateTestTLSMaterial( diff --git a/internal/controllers/tls/readiness.go b/internal/controllers/tls/readiness.go new file mode 100644 index 00000000..e852caed --- /dev/null +++ b/internal/controllers/tls/readiness.go @@ -0,0 +1,229 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package tls + +import ( + "bytes" + "context" + "crypto/tls" + "crypto/x509" + "encoding/pem" + "fmt" + "net/http" + + admissionregistrationv1 "k8s.io/api/admissionregistration/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/healthz" + + "github.com/projectcapsule/capsule/pkg/runtime/configuration" +) + +type certificateGetter func(*tls.ClientHelloInfo) (*tls.Certificate, error) + +// WebhookCertificateReadinessCheck prevents a webhook Pod from becoming ready +// while the certificate it is actually serving is not trusted by an admission +// configuration managed by Capsule. This closes the window between Secret +// publication, certwatcher reload, and caBundle reconciliation. +func WebhookCertificateReadinessCheck( + reader client.Reader, + cfg configuration.Configuration, + namespace string, + getCertificate certificateGetter, +) healthz.Checker { + return func(request *http.Request) error { + servingCertificate, err := getCertificate(nil) + if err != nil { + return fmt.Errorf("load webhook serving certificate: %w", err) + } + + if servingCertificate == nil || len(servingCertificate.Certificate) == 0 { + return fmt.Errorf("webhook serving certificate is empty") + } + + leaf, err := x509.ParseCertificate(servingCertificate.Certificate[0]) + if err != nil { + return fmt.Errorf("parse webhook serving certificate: %w", err) + } + + secret := &corev1.Secret{} + + secretKey := types.NamespacedName{Namespace: namespace, Name: cfg.TLSSecretName()} + if err := reader.Get(request.Context(), secretKey, secret); err != nil { + return fmt.Errorf("get webhook TLS Secret %s: %w", secretKey.String(), err) + } + + persistedLeaf, err := certificateFromPEM(secret.Data[corev1.TLSCertKey]) + if err != nil { + return fmt.Errorf("parse serving certificate in TLS Secret %s: %w", secretKey.String(), err) + } + + if !bytes.Equal(leaf.Raw, persistedLeaf.Raw) { + return fmt.Errorf("loaded webhook serving certificate is stale relative to TLS Secret %s", secretKey.String()) + } + + admission := cfg.Admission() + trusts := make([]admissionWebhookTrust, 0) + + if admission.Mutating != nil && len(admission.Mutating.Webhooks) > 0 { + loaded, err := loadAdmissionWebhookTrust( + request.Context(), + reader, + "mutating", + string(admission.Mutating.Name), + func() client.Object { return &admissionregistrationv1.MutatingWebhookConfiguration{} }, + mutatingWebhookTrust, + ) + if err != nil { + return err + } + + trusts = append(trusts, loaded...) + } + + if admission.Validating != nil && len(admission.Validating.Webhooks) > 0 { + loaded, err := loadAdmissionWebhookTrust( + request.Context(), + reader, + "validating", + string(admission.Validating.Name), + func() client.Object { return &admissionregistrationv1.ValidatingWebhookConfiguration{} }, + validatingWebhookTrust, + ) + if err != nil { + return err + } + + trusts = append(trusts, loaded...) + } + + for _, trust := range trusts { + if err := verifyCertificateAgainstCABundle(leaf, trust.caBundle); err != nil { + return fmt.Errorf( + "%s webhook configuration %q webhook %q does not trust the serving certificate: %w", + trust.kind, + trust.configurationName, + trust.webhookName, + err, + ) + } + } + + return nil + } +} + +func certificateFromPEM(certificatePEM []byte) (*x509.Certificate, error) { + block, _ := pem.Decode(certificatePEM) + if block == nil { + return nil, fmt.Errorf("invalid certificate PEM") + } + + return x509.ParseCertificate(block.Bytes) +} + +type admissionWebhookTrust struct { + kind string + configurationName string + webhookName string + caBundle []byte +} + +func loadAdmissionWebhookTrust( + ctx context.Context, + reader client.Reader, + kind string, + name string, + newConfiguration func() client.Object, + extractTrust func(client.Object, string, string) ([]admissionWebhookTrust, error), +) ([]admissionWebhookTrust, error) { + if name == "" { + return nil, fmt.Errorf("%s webhook configuration name is empty", kind) + } + + webhookConfiguration := newConfiguration() + if err := reader.Get(ctx, types.NamespacedName{Name: name}, webhookConfiguration); err != nil { + return nil, fmt.Errorf("get %s webhook configuration %q: %w", kind, name, err) + } + + trusts, err := extractTrust(webhookConfiguration, kind, name) + if err != nil { + return nil, err + } + + if len(trusts) == 0 { + return nil, fmt.Errorf("%s webhook configuration %q contains no webhooks", kind, name) + } + + return trusts, nil +} + +func mutatingWebhookTrust( + object client.Object, + kind string, + configurationName string, +) ([]admissionWebhookTrust, error) { + configuration, ok := object.(*admissionregistrationv1.MutatingWebhookConfiguration) + if !ok { + return nil, fmt.Errorf("expected MutatingWebhookConfiguration, got %T", object) + } + + trusts := make([]admissionWebhookTrust, 0, len(configuration.Webhooks)) + for index := range configuration.Webhooks { + trusts = append(trusts, admissionWebhookTrust{ + kind: kind, + configurationName: configurationName, + webhookName: configuration.Webhooks[index].Name, + caBundle: configuration.Webhooks[index].ClientConfig.CABundle, + }) + } + + return trusts, nil +} + +func validatingWebhookTrust( + object client.Object, + kind string, + configurationName string, +) ([]admissionWebhookTrust, error) { + configuration, ok := object.(*admissionregistrationv1.ValidatingWebhookConfiguration) + if !ok { + return nil, fmt.Errorf("expected ValidatingWebhookConfiguration, got %T", object) + } + + trusts := make([]admissionWebhookTrust, 0, len(configuration.Webhooks)) + for index := range configuration.Webhooks { + trusts = append(trusts, admissionWebhookTrust{ + kind: kind, + configurationName: configurationName, + webhookName: configuration.Webhooks[index].Name, + caBundle: configuration.Webhooks[index].ClientConfig.CABundle, + }) + } + + return trusts, nil +} + +func verifyCertificateAgainstCABundle(leaf *x509.Certificate, caBundle []byte) error { + if len(caBundle) == 0 { + return fmt.Errorf("caBundle is empty") + } + + roots := x509.NewCertPool() + if !roots.AppendCertsFromPEM(caBundle) { + return fmt.Errorf("caBundle is not valid PEM certificate data") + } + + if _, err := leaf.Verify(x509.VerifyOptions{ + Roots: roots, + KeyUsages: []x509.ExtKeyUsage{ + x509.ExtKeyUsageServerAuth, + }, + }); err != nil { + return err + } + + return nil +} diff --git a/internal/controllers/tls/readiness_test.go b/internal/controllers/tls/readiness_test.go new file mode 100644 index 00000000..67bb10f9 --- /dev/null +++ b/internal/controllers/tls/readiness_test.go @@ -0,0 +1,170 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package tls + +import ( + "context" + cryptotls "crypto/tls" + "net/http/httptest" + "strings" + "testing" + + admissionregistrationv1 "k8s.io/api/admissionregistration/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + runtimeadmission "github.com/projectcapsule/capsule/pkg/runtime/admission" +) + +func TestWebhookCertificateReadinessCheckAcceptsPublishedCA(t *testing.T) { + t.Parallel() + + caBundle, certificatePEM, keyPEM := generateTestTLSMaterial(t, testWebhookSANs()) + validating := testValidatingWebhookConfiguration(caBundle) + reconciler, kubeClient := newTestTLSReconciler( + t, + testTLSSecret(caBundle, certificatePEM, keyPEM), + validating, + ) + enableTestValidatingWebhook(t, kubeClient) + servingCertificate := parseTestServingCertificate(t, certificatePEM, keyPEM) + check := WebhookCertificateReadinessCheck( + kubeClient, + reconciler.Configuration, + testNamespace, + func(*cryptotls.ClientHelloInfo) (*cryptotls.Certificate, error) { + return &servingCertificate, nil + }, + ) + + if err := check(httptest.NewRequest("GET", "/readyz", nil)); err != nil { + t.Fatalf("readiness check rejected matching serving certificate and caBundle: %v", err) + } +} + +func TestWebhookCertificateReadinessCheckRejectsMismatchedPublishedCA(t *testing.T) { + t.Parallel() + + caBundle, certificatePEM, keyPEM := generateTestTLSMaterial(t, testWebhookSANs()) + wrongCABundle, _, _ := generateTestTLSMaterial(t, testWebhookSANs()) + validating := testValidatingWebhookConfiguration(wrongCABundle) + reconciler, kubeClient := newTestTLSReconciler( + t, + testTLSSecret(caBundle, certificatePEM, keyPEM), + validating, + ) + enableTestValidatingWebhook(t, kubeClient) + servingCertificate := parseTestServingCertificate(t, certificatePEM, keyPEM) + check := WebhookCertificateReadinessCheck( + kubeClient, + reconciler.Configuration, + testNamespace, + func(*cryptotls.ClientHelloInfo) (*cryptotls.Certificate, error) { + return &servingCertificate, nil + }, + ) + + err := check(httptest.NewRequest("GET", "/readyz", nil)) + if err == nil { + t.Fatal("readiness check accepted a caBundle that does not trust the serving certificate") + } + if !strings.Contains(err.Error(), "does not trust the serving certificate") { + t.Fatalf("readiness error = %v, want certificate trust failure", err) + } +} + +func TestWebhookCertificateReadinessCheckRejectsStaleLoadedCertificate(t *testing.T) { + t.Parallel() + + caBundle, persistedCertificate, persistedKey := generateTestTLSMaterial(t, testWebhookSANs()) + _, staleCertificate, staleKey := generateTestTLSMaterial(t, testWebhookSANs()) + reconciler, kubeClient := newTestTLSReconciler( + t, + testTLSSecret(caBundle, persistedCertificate, persistedKey), + ) + loadedCertificate := parseTestServingCertificate(t, staleCertificate, staleKey) + check := WebhookCertificateReadinessCheck( + kubeClient, + reconciler.Configuration, + testNamespace, + func(*cryptotls.ClientHelloInfo) (*cryptotls.Certificate, error) { + return &loadedCertificate, nil + }, + ) + + err := check(httptest.NewRequest("GET", "/readyz", nil)) + if err == nil { + t.Fatal("readiness check accepted a stale certificate loaded by certwatcher") + } + if !strings.Contains(err.Error(), "is stale relative to TLS Secret") { + t.Fatalf("readiness error = %v, want stale loaded certificate failure", err) + } +} + +func TestWebhookCertificateReadinessCheckRequiresManagedConfiguration(t *testing.T) { + t.Parallel() + + caBundle, certificatePEM, keyPEM := generateTestTLSMaterial(t, testWebhookSANs()) + reconciler, kubeClient := newTestTLSReconciler(t, testTLSSecret(caBundle, certificatePEM, keyPEM)) + enableTestValidatingWebhook(t, kubeClient) + servingCertificate := parseTestServingCertificate(t, certificatePEM, keyPEM) + check := WebhookCertificateReadinessCheck( + kubeClient, + reconciler.Configuration, + testNamespace, + func(*cryptotls.ClientHelloInfo) (*cryptotls.Certificate, error) { + return &servingCertificate, nil + }, + ) + + err := check(httptest.NewRequest("GET", "/readyz", nil)) + if err == nil { + t.Fatal("readiness check accepted a missing managed webhook configuration") + } + if !strings.Contains(err.Error(), "get validating webhook configuration") { + t.Fatalf("readiness error = %v, want missing configuration failure", err) + } +} + +func testValidatingWebhookConfiguration(caBundle []byte) *admissionregistrationv1.ValidatingWebhookConfiguration { + return &admissionregistrationv1.ValidatingWebhookConfiguration{ + ObjectMeta: metav1.ObjectMeta{Name: testValidatingConfiguration}, + Webhooks: []admissionregistrationv1.ValidatingWebhook{{ + Name: "owners.validating.projectcapsule.dev", + ClientConfig: admissionregistrationv1.WebhookClientConfig{ + CABundle: caBundle, + }, + }}, + } +} + +func enableTestValidatingWebhook(t *testing.T, kubeClient client.Client) { + t.Helper() + + configurationObject := &capsulev1beta2.CapsuleConfiguration{} + if err := kubeClient.Get( + context.Background(), + client.ObjectKey{Name: "capsule"}, + configurationObject, + ); err != nil { + t.Fatalf("get CapsuleConfiguration: %v", err) + } + + configurationObject.Spec.Admission.Validating.Webhooks = []*runtimeadmission.ValidatingWebhook{{}} + if err := kubeClient.Update(context.Background(), configurationObject); err != nil { + t.Fatalf("update CapsuleConfiguration: %v", err) + } +} + +func parseTestServingCertificate(t *testing.T, certificatePEM, keyPEM []byte) cryptotls.Certificate { + t.Helper() + + servingCertificate, err := cryptotls.X509KeyPair(certificatePEM, keyPEM) + if err != nil { + t.Fatalf("parse test serving certificate: %v", err) + } + + return servingCertificate +} diff --git a/internal/webhook/globalresourcequota/calculation.go b/internal/webhook/globalresourcequota/calculation.go index bd14ee9e..6bf5dfac 100644 --- a/internal/webhook/globalresourcequota/calculation.go +++ b/internal/webhook/globalresourcequota/calculation.go @@ -13,6 +13,7 @@ import ( admissionv1 "k8s.io/api/admission/v1" corev1 "k8s.io/api/core/v1" + apiequality "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -115,6 +116,18 @@ func validateGlobalResourceQuotaRequest( return ad.Denyf("previous GlobalResourceQuota could not be decoded: %v", err) } + if err := runtimequota.ValidateHardLimitScopeChange( + "spec.quota.hard", + quota.Spec.Quota.Hard, + oldQuota.Spec.Quota.Hard, + !apiequality.Semantic.DeepEqual( + quota.Spec.NamespaceSelectors, + oldQuota.Spec.NamespaceSelectors, + ), + ); err != nil { + return ad.Denyf("invalid GlobalResourceQuota: %v", err) + } + if err := validateHardLimit(quota.Spec.Quota.Hard, oldQuota.Status.Total.Used); err != nil { return ad.Denyf("invalid GlobalResourceQuota: %v", err) } diff --git a/internal/webhook/globalresourcequota/calculation_test.go b/internal/webhook/globalresourcequota/calculation_test.go index 2d6c50c4..a209dc81 100644 --- a/internal/webhook/globalresourcequota/calculation_test.go +++ b/internal/webhook/globalresourcequota/calculation_test.go @@ -7,6 +7,7 @@ import ( "context" "encoding/json" "fmt" + "strings" "sync" "sync/atomic" "testing" @@ -304,6 +305,35 @@ func TestRuleManagedGlobalResourceQuotaCannotBeReducedBelowUsage(t *testing.T) { } } +func TestGlobalResourceQuotaCannotChangeScopeAndReduceHardLimit(t *testing.T) { + t.Parallel() + + oldQuota := globalQuotaForTest("shared", corev1.ResourceList{ + corev1.ResourceLimitsCPU: resource.MustParse("8"), + }) + oldQuota.Spec.NamespaceSelectors = []selectors.NamespaceSelector{{ + LabelSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"tier": "application"}}, + }} + + newQuota := oldQuota.DeepCopy() + newQuota.Spec.NamespaceSelectors = []selectors.NamespaceSelector{{ + LabelSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"tenant": "wind"}}, + }} + newQuota.Spec.Quota.Hard[corev1.ResourceLimitsCPU] = resource.MustParse("0") + + request := globalResourceQuotaUpdateRequest(t, oldQuota, newQuota) + response := validateGlobalResourceQuotaRequest(context.Background(), ledgerClient(t), request) + if response == nil || response.Allowed || response.Result == nil { + t.Fatalf("scope change with hard-limit decrease was accepted: %#v", response) + } + if !strings.Contains( + response.Result.Message, + `spec.quota.hard["limits.cpu"] cannot be reduced from 8 to 0 while namespace selectors are changing`, + ) { + t.Fatalf("denial message = %q", response.Result.Message) + } +} + func TestFormatExceededResources(t *testing.T) { t.Parallel() diff --git a/internal/webhook/rules/generic/mutation/metadata.go b/internal/webhook/rules/generic/mutation/metadata.go index c094152e..5cb805d2 100644 --- a/internal/webhook/rules/generic/mutation/metadata.go +++ b/internal/webhook/rules/generic/mutation/metadata.go @@ -9,6 +9,7 @@ import ( "fmt" "net/http" + admissionv1 "k8s.io/api/admission/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" @@ -48,7 +49,22 @@ func (*metadataRules) mutate(obj *unstructured.Unstructured, bodies []*apirules. return &response } - if !MutateMetadata(obj, gvk, bodies) { + metadataMutated := MutateMetadata(obj, gvk, bodies) + + resourcesMutated := false + + if req.Operation == admissionv1.Create { + var err error + + resourcesMutated, err = MutateWorkloadResources(obj, gvk, bodies) + if err != nil { + response := admission.Errored(http.StatusInternalServerError, err) + + return &response + } + } + + if !metadataMutated && !resourcesMutated { return nil } diff --git a/internal/webhook/rules/generic/mutation/workload_resources.go b/internal/webhook/rules/generic/mutation/workload_resources.go new file mode 100644 index 00000000..b05b817e --- /dev/null +++ b/internal/webhook/rules/generic/mutation/workload_resources.go @@ -0,0 +1,386 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package mutation + +import ( + "fmt" + "slices" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + resourcehelper "k8s.io/component-helpers/resource" + + apirules "github.com/projectcapsule/capsule/pkg/api/rules" + "github.com/projectcapsule/capsule/pkg/runtime/workloads" +) + +type workloadResourcePolicies struct { + requests map[corev1.ResourceName]apirules.WorkloadResourceRequestPolicy + limits map[corev1.ResourceName]apirules.WorkloadResourceLimitPolicy +} + +func MutateWorkloadResources( + obj *unstructured.Unstructured, + gvk schema.GroupVersionKind, + bodies []*apirules.NamespaceRuleBodyNamespace, +) (bool, error) { + if obj == nil || gvk != corev1.SchemeGroupVersion.WithKind("Pod") { + return false, nil + } + + pod := &corev1.Pod{} + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, pod); err != nil { + return false, fmt.Errorf("decode Pod resource policies: %w", err) + } + + changed, err := MutatePodResources(pod, bodies) + if err != nil || !changed { + return changed, err + } + + mutated, err := runtime.DefaultUnstructuredConverter.ToUnstructured(pod) + if err != nil { + return false, fmt.Errorf("encode Pod resource policies: %w", err) + } + + obj.Object = mutated + + return true, nil +} + +func MutatePodResources( + pod *corev1.Pod, + bodies []*apirules.NamespaceRuleBodyNamespace, +) (bool, error) { + if pod == nil { + return false, nil + } + + mutated := false + + containerPolicies := collectWorkloadResourcePolicies(bodies, apirules.ValidateContainers) + for i := range pod.Spec.Containers { + changed, err := mutateResourceRequirements(&pod.Spec.Containers[i].Resources, containerPolicies) + if err != nil { + return false, fmt.Errorf("spec.containers[%d].resources: %w", i, err) + } + + mutated = mutated || changed + } + + initContainerPolicies := collectWorkloadResourcePolicies(bodies, apirules.ValidateInitContainers) + for i := range pod.Spec.InitContainers { + changed, err := mutateResourceRequirements(&pod.Spec.InitContainers[i].Resources, initContainerPolicies) + if err != nil { + return false, fmt.Errorf("spec.initContainers[%d].resources: %w", i, err) + } + + mutated = mutated || changed + } + + podPolicies := collectWorkloadResourcePolicies(bodies, apirules.ValidatePod) + if len(podPolicies.requests) > 0 || len(podPolicies.limits) > 0 { + resources := pod.Spec.Resources + if resources == nil { + resources = &corev1.ResourceRequirements{} + } + + changed, err := mutateResourceRequirements(resources, podPolicies) + if err != nil { + return false, fmt.Errorf("spec.resources: %w", err) + } + + if changed { + pod.Spec.Resources = resources + mutated = true + } + } + + if mutated { + backfillMissingPodRequests(pod) + } + + return mutated, nil +} + +// backfillMissingPodRequests keeps Pods mutated by Capsule valid on Kubernetes +// 1.33. That release validates every aggregate container request as soon as +// spec.resources is present, treating a missing Pod-level request as zero. We +// can backfill only resources Kubernetes permits at Pod level. Kubernetes 1.34 +// and later skip missing keys; explicitly storing the effective aggregate keeps +// the same semantics on those releases. +func backfillMissingPodRequests(pod *corev1.Pod) { + if pod.Spec.Resources == nil { + return + } + + requests := resourcehelper.AggregateContainerRequests(pod, resourcehelper.PodResourcesOptions{}) + + for name, request := range requests { + if !workloads.PodLevelResourceSupported(name) { + continue + } + + if _, found := pod.Spec.Resources.Requests[name]; found { + continue + } + + if pod.Spec.Resources.Requests == nil { + pod.Spec.Resources.Requests = corev1.ResourceList{} + } + + pod.Spec.Resources.Requests[name] = request.DeepCopy() + } +} + +func collectWorkloadResourcePolicies( + bodies []*apirules.NamespaceRuleBodyNamespace, + target apirules.WorkloadValidationTarget, +) workloadResourcePolicies { + out := workloadResourcePolicies{} + + for _, body := range bodies { + if body == nil || body.Enforce == nil || body.Enforce.Workloads.Resources == nil { + continue + } + + if !resourcePoliciesTarget(body.Enforce.Workloads, target) { + continue + } + + resources := body.Enforce.Workloads.Resources + for name, policy := range resources.Requests { + if !resourcePolicySupportsTarget(name, target) { + continue + } + + if out.requests == nil { + out.requests = make(map[corev1.ResourceName]apirules.WorkloadResourceRequestPolicy) + } + + out.requests[name] = policy + } + + for name, policy := range resources.Limits { + if !resourcePolicySupportsTarget(name, target) { + continue + } + + if out.limits == nil { + out.limits = make(map[corev1.ResourceName]apirules.WorkloadResourceLimitPolicy) + } + + out.limits[name] = policy + } + } + + return out +} + +func resourcePoliciesTarget( + workload apirules.NamespaceRuleEnforceWorkloadsBody, + target apirules.WorkloadValidationTarget, +) bool { + if len(workload.Targets) == 0 { + return target == apirules.ValidatePod || + target == apirules.ValidateContainers || + target == apirules.ValidateInitContainers + } + + return slices.Contains(workload.Targets, target) +} + +func resourcePolicySupportsTarget( + name corev1.ResourceName, + target apirules.WorkloadValidationTarget, +) bool { + return target != apirules.ValidatePod || workloads.PodLevelResourceSupported(name) +} + +func mutateResourceRequirements( + resources *corev1.ResourceRequirements, + policies workloadResourcePolicies, +) (bool, error) { + if resources == nil { + return false, nil + } + + requestsChanged, err := mutateRequestPolicies(resources, policies.requests) + if err != nil { + return false, err + } + + limitsChanged, err := mutateLimitPolicies(resources, policies.limits) + if err != nil { + return false, err + } + + return requestsChanged || limitsChanged, nil +} + +func mutateRequestPolicies( + resources *corev1.ResourceRequirements, + policies map[corev1.ResourceName]apirules.WorkloadResourceRequestPolicy, +) (bool, error) { + changed := false + + for name, policy := range policies { + policyChanged, err := mutateRequestPolicy(resources, name, policy) + if err != nil { + return false, err + } + + changed = policyChanged || changed + } + + return changed, nil +} + +func mutateRequestPolicy( + resources *corev1.ResourceRequirements, + name corev1.ResourceName, + policy apirules.WorkloadResourceRequestPolicy, +) (bool, error) { + switch policy.Policy { + case apirules.WorkloadResourceRequestPolicyPreserve: + return false, nil + case apirules.WorkloadResourceRequestPolicyDefault: + if policy.Value == nil { + return false, fmt.Errorf("request %q Default policy has no value", name) + } + + if _, found := resources.Requests[name]; found { + return false, nil + } + + if resources.Requests == nil { + resources.Requests = corev1.ResourceList{} + } + + resources.Requests[name] = policy.Value.DeepCopy() + + return true, nil + case apirules.WorkloadResourceRequestPolicyRemove: + return removeResource(resources.Requests, name), nil + default: + return false, fmt.Errorf("request %q has unsupported policy %q", name, policy.Policy) + } +} + +func mutateLimitPolicies( + resources *corev1.ResourceRequirements, + policies map[corev1.ResourceName]apirules.WorkloadResourceLimitPolicy, +) (bool, error) { + changed := false + + for name, policy := range policies { + policyChanged, err := mutateLimitPolicy(resources, name, policy) + if err != nil { + return false, err + } + + changed = policyChanged || changed + } + + return changed, nil +} + +func mutateLimitPolicy( + resources *corev1.ResourceRequirements, + name corev1.ResourceName, + policy apirules.WorkloadResourceLimitPolicy, +) (bool, error) { + switch policy.Policy { + case apirules.WorkloadResourceLimitPolicyPreserve: + return false, nil + case apirules.WorkloadResourceLimitPolicyDefault: + if policy.Value == nil { + return false, fmt.Errorf("limit %q Default policy has no value", name) + } + + if _, found := resources.Limits[name]; found { + return false, nil + } + + if resources.Limits == nil { + resources.Limits = corev1.ResourceList{} + } + + resources.Limits[name] = policy.Value.DeepCopy() + + return true, nil + case apirules.WorkloadResourceLimitPolicyRemove: + return removeResource(resources.Limits, name), nil + case apirules.WorkloadResourceLimitPolicyMatchRequest: + return matchResourceRequest(resources, name), nil + case apirules.WorkloadResourceLimitPolicyRatio: + return defaultResourceRatio(resources, name, policy.Value) + default: + return false, fmt.Errorf("limit %q has unsupported policy %q", name, policy.Policy) + } +} + +func removeResource(resources corev1.ResourceList, name corev1.ResourceName) bool { + if _, found := resources[name]; !found { + return false + } + + delete(resources, name) + + return true +} + +func matchResourceRequest(resources *corev1.ResourceRequirements, name corev1.ResourceName) bool { + request, found := resources.Requests[name] + if !found { + return false + } + + if limit, found := resources.Limits[name]; found && limit.Cmp(request) == 0 { + return false + } + + if resources.Limits == nil { + resources.Limits = corev1.ResourceList{} + } + + resources.Limits[name] = request.DeepCopy() + + return true +} + +func defaultResourceRatio( + resources *corev1.ResourceRequirements, + name corev1.ResourceName, + ratio *resource.Quantity, +) (bool, error) { + if ratio == nil { + return false, fmt.Errorf("limit %q Ratio policy has no value", name) + } + + if _, found := resources.Limits[name]; found { + return false, nil + } + + request, found := resources.Requests[name] + if !found || request.Sign() <= 0 { + return false, nil + } + + limit, err := workloads.LimitForRatio(name, request, *ratio) + if err != nil { + return false, err + } + + if resources.Limits == nil { + resources.Limits = corev1.ResourceList{} + } + + resources.Limits[name] = limit + + return true, nil +} diff --git a/internal/webhook/rules/generic/mutation/workload_resources_test.go b/internal/webhook/rules/generic/mutation/workload_resources_test.go new file mode 100644 index 00000000..83629653 --- /dev/null +++ b/internal/webhook/rules/generic/mutation/workload_resources_test.go @@ -0,0 +1,303 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package mutation + +import ( + "testing" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + + "github.com/projectcapsule/capsule/pkg/api/rules" +) + +func TestMutatePodResourcesUsesAllDefaultTargets(t *testing.T) { + t.Parallel() + + pod := &corev1.Pod{Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: "app", + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("1Gi")}, + Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1")}, + }, + }}, + InitContainers: []corev1.Container{{ + Name: "init", + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("512Mi")}, + }, + }}, + Resources: &corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("500m")}, + Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1")}, + }, + }} + + bodies := []*rules.NamespaceRuleBodyNamespace{{Enforce: &rules.NamespaceRuleEnforceBody{ + Workloads: rules.NamespaceRuleEnforceWorkloadsBody{ + Resources: &rules.WorkloadResourceRules{ + Requests: map[corev1.ResourceName]rules.WorkloadResourceRequestPolicy{ + corev1.ResourceCPU: { + Policy: rules.WorkloadResourceRequestPolicyDefault, + Value: quantityPointer("100m"), + }, + }, + Limits: map[corev1.ResourceName]rules.WorkloadResourceLimitPolicy{ + corev1.ResourceCPU: {Policy: rules.WorkloadResourceLimitPolicyRemove}, + corev1.ResourceMemory: { + Policy: rules.WorkloadResourceLimitPolicyMatchRequest, + }, + }, + }, + }, + }}} + + changed, err := MutatePodResources(pod, bodies) + if err != nil { + t.Fatalf("MutatePodResources() error = %v", err) + } + if !changed { + t.Fatal("MutatePodResources() changed = false, want true") + } + + assertResourceQuantity(t, pod.Spec.Containers[0].Resources.Requests, corev1.ResourceCPU, "100m") + assertResourceMissing(t, pod.Spec.Containers[0].Resources.Limits, corev1.ResourceCPU) + assertResourceQuantity(t, pod.Spec.Containers[0].Resources.Limits, corev1.ResourceMemory, "1Gi") + assertResourceQuantity(t, pod.Spec.InitContainers[0].Resources.Requests, corev1.ResourceCPU, "100m") + assertResourceQuantity(t, pod.Spec.InitContainers[0].Resources.Limits, corev1.ResourceMemory, "512Mi") + + assertResourceQuantity(t, pod.Spec.Resources.Requests, corev1.ResourceCPU, "500m") + assertResourceQuantity(t, pod.Spec.Resources.Requests, corev1.ResourceMemory, "1Gi") + assertResourceMissing(t, pod.Spec.Resources.Limits, corev1.ResourceCPU) +} + +func TestMutatePodResourcesBackfillsOnlySupportedMissingPodRequests(t *testing.T) { + t.Parallel() + + pod := &corev1.Pod{Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: "app", + Resources: corev1.ResourceRequirements{Requests: corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse("1Gi"), + corev1.ResourceEphemeralStorage: resource.MustParse("2Gi"), + }}, + }, { + Name: "sidecar", + Resources: corev1.ResourceRequirements{Requests: corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse("512Mi"), + }}, + }}, + Resources: &corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("500m")}, + }, + }} + + bodies := []*rules.NamespaceRuleBodyNamespace{{Enforce: &rules.NamespaceRuleEnforceBody{ + Workloads: rules.NamespaceRuleEnforceWorkloadsBody{ + Targets: []rules.WorkloadValidationTarget{rules.ValidateContainers}, + Resources: &rules.WorkloadResourceRules{Requests: map[corev1.ResourceName]rules.WorkloadResourceRequestPolicy{ + corev1.ResourceCPU: { + Policy: rules.WorkloadResourceRequestPolicyDefault, + Value: quantityPointer("100m"), + }, + }}, + }, + }}} + + changed, err := MutatePodResources(pod, bodies) + if err != nil { + t.Fatalf("MutatePodResources() error = %v", err) + } + if !changed { + t.Fatal("MutatePodResources() changed = false, want true") + } + + assertResourceQuantity(t, pod.Spec.Resources.Requests, corev1.ResourceCPU, "500m") + assertResourceQuantity(t, pod.Spec.Resources.Requests, corev1.ResourceMemory, "1536Mi") + assertResourceMissing(t, pod.Spec.Resources.Requests, corev1.ResourceEphemeralStorage) +} + +func TestMutatePodResourcesDefaultTargetsSkipPodIncompatibleResources(t *testing.T) { + t.Parallel() + + pod := &corev1.Pod{Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Name: "app"}}, + }} + + bodies := []*rules.NamespaceRuleBodyNamespace{{Enforce: &rules.NamespaceRuleEnforceBody{ + Workloads: rules.NamespaceRuleEnforceWorkloadsBody{ + Resources: &rules.WorkloadResourceRules{ + Requests: map[corev1.ResourceName]rules.WorkloadResourceRequestPolicy{ + corev1.ResourceEphemeralStorage: { + Policy: rules.WorkloadResourceRequestPolicyDefault, + Value: quantityPointer("1Gi"), + }, + }, + }, + }, + }}} + + changed, err := MutatePodResources(pod, bodies) + if err != nil { + t.Fatalf("MutatePodResources() error = %v", err) + } + if !changed { + t.Fatal("MutatePodResources() changed = false, want true") + } + + assertResourceQuantity( + t, + pod.Spec.Containers[0].Resources.Requests, + corev1.ResourceEphemeralStorage, + "1Gi", + ) + if pod.Spec.Resources != nil { + t.Fatalf("pod.Spec.Resources = %#v, want nil", pod.Spec.Resources) + } +} + +func TestMutatePodResourcesPodRatio(t *testing.T) { + t.Parallel() + + pod := &corev1.Pod{Spec: corev1.PodSpec{ + Resources: &corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("500m")}, + }, + }} + + bodies := []*rules.NamespaceRuleBodyNamespace{{Enforce: &rules.NamespaceRuleEnforceBody{ + Workloads: rules.NamespaceRuleEnforceWorkloadsBody{ + Targets: []rules.WorkloadValidationTarget{rules.ValidatePod}, + Resources: &rules.WorkloadResourceRules{ + Limits: map[corev1.ResourceName]rules.WorkloadResourceLimitPolicy{ + corev1.ResourceCPU: { + Policy: rules.WorkloadResourceLimitPolicyRatio, + Value: quantityPointer("1.5"), + }, + }, + }, + }, + }}} + + changed, err := MutatePodResources(pod, bodies) + if err != nil { + t.Fatalf("MutatePodResources() error = %v", err) + } + if !changed { + t.Fatal("MutatePodResources() changed = false, want true") + } + + assertResourceQuantity(t, pod.Spec.Resources.Limits, corev1.ResourceCPU, "750m") +} + +func TestMutatePodResourcesPreservesExplicitRatioLimit(t *testing.T) { + t.Parallel() + + pod := &corev1.Pod{Spec: corev1.PodSpec{Containers: []corev1.Container{{ + Name: "app", + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("1Gi")}, + Limits: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("2Gi")}, + }, + }}}} + + bodies := resourceMutationBodies( + rules.WorkloadResourceLimitPolicyRatio, + quantityPointer("1.5"), + ) + + changed, err := MutatePodResources(pod, bodies) + if err != nil { + t.Fatalf("MutatePodResources() error = %v", err) + } + if changed { + t.Fatal("MutatePodResources() changed an explicitly supplied ratio limit") + } + + assertResourceQuantity(t, pod.Spec.Containers[0].Resources.Limits, corev1.ResourceMemory, "2Gi") +} + +func TestMutatePodResourcesLastPolicyWins(t *testing.T) { + t.Parallel() + + pod := &corev1.Pod{Spec: corev1.PodSpec{Containers: []corev1.Container{{ + Name: "app", + Resources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1")}, + }, + }}}} + + bodies := []*rules.NamespaceRuleBodyNamespace{ + {Enforce: &rules.NamespaceRuleEnforceBody{Workloads: rules.NamespaceRuleEnforceWorkloadsBody{ + Resources: &rules.WorkloadResourceRules{Limits: map[corev1.ResourceName]rules.WorkloadResourceLimitPolicy{ + corev1.ResourceCPU: {Policy: rules.WorkloadResourceLimitPolicyRemove}, + }}, + }}}, + {Enforce: &rules.NamespaceRuleEnforceBody{Workloads: rules.NamespaceRuleEnforceWorkloadsBody{ + Targets: []rules.WorkloadValidationTarget{rules.ValidateContainers}, + Resources: &rules.WorkloadResourceRules{Limits: map[corev1.ResourceName]rules.WorkloadResourceLimitPolicy{ + corev1.ResourceCPU: {Policy: rules.WorkloadResourceLimitPolicyPreserve}, + }}, + }}}, + } + + changed, err := MutatePodResources(pod, bodies) + if err != nil { + t.Fatalf("MutatePodResources() error = %v", err) + } + if changed { + t.Fatal("MutatePodResources() ignored the later Preserve policy") + } + + assertResourceQuantity(t, pod.Spec.Containers[0].Resources.Limits, corev1.ResourceCPU, "1") +} + +func resourceMutationBodies( + policy rules.WorkloadResourceLimitPolicyType, + value *resource.Quantity, +) []*rules.NamespaceRuleBodyNamespace { + return []*rules.NamespaceRuleBodyNamespace{{Enforce: &rules.NamespaceRuleEnforceBody{ + Workloads: rules.NamespaceRuleEnforceWorkloadsBody{ + Resources: &rules.WorkloadResourceRules{ + Limits: map[corev1.ResourceName]rules.WorkloadResourceLimitPolicy{ + corev1.ResourceMemory: {Policy: policy, Value: value}, + }, + }, + }, + }}} +} + +func quantityPointer(value string) *resource.Quantity { + quantity := resource.MustParse(value) + + return &quantity +} + +func assertResourceQuantity( + t *testing.T, + resources corev1.ResourceList, + name corev1.ResourceName, + want string, +) { + t.Helper() + + got, found := resources[name] + if !found { + t.Fatalf("resource %q is missing", name) + } + + wantQuantity := resource.MustParse(want) + if got.Cmp(wantQuantity) != 0 { + t.Fatalf("resource %q = %s, want %s", name, got.String(), wantQuantity.String()) + } +} + +func assertResourceMissing(t *testing.T, resources corev1.ResourceList, name corev1.ResourceName) { + t.Helper() + + if value, found := resources[name]; found { + t.Fatalf("resource %q = %s, want missing", name, value.String()) + } +} diff --git a/internal/webhook/rules/pods/validation/factory.go b/internal/webhook/rules/pods/validation/factory.go index c4559ae2..13714af3 100644 --- a/internal/webhook/rules/pods/validation/factory.go +++ b/internal/webhook/rules/pods/validation/factory.go @@ -38,10 +38,13 @@ func evaluatePodRules[R any]( ) } -type podRuleValidator func( - *corev1.Pod, - []*apirules.NamespaceRuleEnforceBody, -) (*ruleengine.Evaluation, error) +type podRuleValidator struct { + evaluate func( + *corev1.Pod, + []*apirules.NamespaceRuleEnforceBody, + ) (*ruleengine.Evaluation, error) + includeSubresources bool +} type podRules struct { rules []podRuleValidator @@ -67,9 +70,10 @@ func PodRules( } h.rules = []podRuleValidator{ - h.validateSchedulers, - h.validateQoSClasses, - h.validateRegistries, + {evaluate: h.validateResources}, + {evaluate: h.validateSchedulers, includeSubresources: true}, + {evaluate: h.validateQoSClasses, includeSubresources: true}, + {evaluate: h.validateRegistries, includeSubresources: true}, } return h @@ -138,8 +142,12 @@ func (h *podRules) validatePodRules( recorder events.EventRecorder, enforceBodies []*apirules.NamespaceRuleEnforceBody, ) error { - for _, evaluate := range h.rules { - evaluation, err := evaluate(pod, enforceBodies) + for _, rule := range h.rules { + if req.SubResource != "" && !rule.includeSubresources { + continue + } + + evaluation, err := rule.evaluate(pod, enforceBodies) if err != nil { return err } diff --git a/internal/webhook/rules/pods/validation/factory_test.go b/internal/webhook/rules/pods/validation/factory_test.go index 7b99d6f2..cb11cf6e 100644 --- a/internal/webhook/rules/pods/validation/factory_test.go +++ b/internal/webhook/rules/pods/validation/factory_test.go @@ -3,10 +3,57 @@ package validation -import "github.com/projectcapsule/capsule/internal/cache" +import ( + "context" + "testing" + + admissionv1 "k8s.io/api/admission/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/internal/cache" + apirules "github.com/projectcapsule/capsule/pkg/api/rules" +) func podRulesForTest() *podRules { return &podRules{ regexCache: cache.NewRegexCache(), } } + +func TestPodResourceRulesSkipSubresources(t *testing.T) { + t.Parallel() + + h := podRulesForTest() + h.rules = []podRuleValidator{{evaluate: h.validateResources}} + + ratio := resource.MustParse("1.5") + enforce := []*apirules.NamespaceRuleEnforceBody{{ + Action: apirules.ActionTypeDeny, + Workloads: apirules.NamespaceRuleEnforceWorkloadsBody{ + Resources: &apirules.WorkloadResourceRules{ + Limits: map[corev1.ResourceName]apirules.WorkloadResourceLimitPolicy{ + corev1.ResourceMemory: { + Policy: apirules.WorkloadResourceLimitPolicyRatio, + Value: &ratio, + }, + }, + }, + }, + }} + pod := resourceValidationPod("1Gi", "2Gi") + req := admission.Request{AdmissionRequest: admissionv1.AdmissionRequest{SubResource: "ephemeralcontainers"}} + + if err := h.validatePodRules( + context.Background(), + req, + pod, + &capsulev1beta2.Tenant{}, + nil, + enforce, + ); err != nil { + t.Fatalf("validatePodRules() error = %v, resource rules must skip subresources", err) + } +} diff --git a/internal/webhook/rules/pods/validation/resources.go b/internal/webhook/rules/pods/validation/resources.go new file mode 100644 index 00000000..fa4bb4dd --- /dev/null +++ b/internal/webhook/rules/pods/validation/resources.go @@ -0,0 +1,517 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package validation + +import ( + "fmt" + "slices" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + + apirules "github.com/projectcapsule/capsule/pkg/api/rules" + "github.com/projectcapsule/capsule/pkg/ruleengine" + "github.com/projectcapsule/capsule/pkg/runtime/events" + "github.com/projectcapsule/capsule/pkg/runtime/workloads" +) + +type workloadResourceLocation struct { + target apirules.WorkloadValidationTarget + path string + resources *corev1.ResourceRequirements +} + +type workloadResourceConstraint struct { + action apirules.ActionType + policy string + value *resource.Quantity +} + +type workloadResourceField string + +const ( + workloadResourceRequests workloadResourceField = "request" + workloadResourceLimits workloadResourceField = "limit" +) + +func (h *podRules) validateResources( + pod *corev1.Pod, + enforceBodies []*apirules.NamespaceRuleEnforceBody, +) (*ruleengine.Evaluation, error) { + if pod == nil || len(enforceBodies) == 0 { + return nil, nil + } + + out := &ruleengine.Evaluation{} + + for _, location := range workloadResourceLocations(pod, enforceBodies) { + requests, limits := workloadResourceNames(enforceBodies, location.target) + + for _, name := range requests { + evaluation, err := evaluateWorkloadResourceField( + location, + name, + workloadResourceRequests, + workloadResourceConstraints(enforceBodies, location.target, name, workloadResourceRequests), + ) + if err != nil { + return out, err + } + + out.Append(evaluation) + + if evaluation != nil && evaluation.Blocking != nil { + return out, nil + } + } + + for _, name := range limits { + evaluation, err := evaluateWorkloadResourceField( + location, + name, + workloadResourceLimits, + workloadResourceConstraints(enforceBodies, location.target, name, workloadResourceLimits), + ) + if err != nil { + return out, err + } + + out.Append(evaluation) + + if evaluation != nil && evaluation.Blocking != nil { + return out, nil + } + } + } + + return out, nil +} + +func workloadResourceLocations( + pod *corev1.Pod, + enforceBodies []*apirules.NamespaceRuleEnforceBody, +) []workloadResourceLocation { + out := make([]workloadResourceLocation, 0, len(pod.Spec.Containers)+len(pod.Spec.InitContainers)+1) + + if hasWorkloadResourcePolicies(enforceBodies, apirules.ValidatePod) { + resources := pod.Spec.Resources + if resources == nil { + resources = &corev1.ResourceRequirements{} + } + + out = append(out, workloadResourceLocation{ + target: apirules.ValidatePod, + path: "spec.resources", + resources: resources, + }) + } + + if hasWorkloadResourcePolicies(enforceBodies, apirules.ValidateInitContainers) { + for i := range pod.Spec.InitContainers { + out = append(out, workloadResourceLocation{ + target: apirules.ValidateInitContainers, + path: fmt.Sprintf("spec.initContainers[%d].resources", i), + resources: &pod.Spec.InitContainers[i].Resources, + }) + } + } + + if hasWorkloadResourcePolicies(enforceBodies, apirules.ValidateContainers) { + for i := range pod.Spec.Containers { + out = append(out, workloadResourceLocation{ + target: apirules.ValidateContainers, + path: fmt.Sprintf("spec.containers[%d].resources", i), + resources: &pod.Spec.Containers[i].Resources, + }) + } + } + + return out +} + +func hasWorkloadResourcePolicies( + bodies []*apirules.NamespaceRuleEnforceBody, + target apirules.WorkloadValidationTarget, +) bool { + for _, enforce := range bodies { + if enforce == nil || enforce.Workloads.Resources == nil { + continue + } + + if !validationResourcePoliciesTarget(enforce.Workloads, target) { + continue + } + + for name := range enforce.Workloads.Resources.Requests { + if validationResourcePolicySupportsTarget(name, target) { + return true + } + } + + for name := range enforce.Workloads.Resources.Limits { + if validationResourcePolicySupportsTarget(name, target) { + return true + } + } + } + + return false +} + +func validationResourcePoliciesTarget( + workload apirules.NamespaceRuleEnforceWorkloadsBody, + target apirules.WorkloadValidationTarget, +) bool { + if len(workload.Targets) == 0 { + return target == apirules.ValidatePod || + target == apirules.ValidateContainers || + target == apirules.ValidateInitContainers + } + + return slices.Contains(workload.Targets, target) +} + +func validationResourcePolicySupportsTarget( + name corev1.ResourceName, + target apirules.WorkloadValidationTarget, +) bool { + return target != apirules.ValidatePod || workloads.PodLevelResourceSupported(name) +} + +func workloadResourceNames( + bodies []*apirules.NamespaceRuleEnforceBody, + target apirules.WorkloadValidationTarget, +) (requests []corev1.ResourceName, limits []corev1.ResourceName) { + requestSet := map[corev1.ResourceName]struct{}{} + limitSet := map[corev1.ResourceName]struct{}{} + + for _, enforce := range bodies { + if enforce == nil || enforce.Workloads.Resources == nil || + !validationResourcePoliciesTarget(enforce.Workloads, target) { + continue + } + + for name, policy := range enforce.Workloads.Resources.Requests { + if !validationResourcePolicySupportsTarget(name, target) { + continue + } + + if policy.Policy == apirules.WorkloadResourceRequestPolicyRemove { + requestSet[name] = struct{}{} + } + } + + for name, policy := range enforce.Workloads.Resources.Limits { + if !validationResourcePolicySupportsTarget(name, target) { + continue + } + + switch policy.Policy { + case apirules.WorkloadResourceLimitPolicyPreserve, + apirules.WorkloadResourceLimitPolicyDefault: + case apirules.WorkloadResourceLimitPolicyRemove, + apirules.WorkloadResourceLimitPolicyMatchRequest, + apirules.WorkloadResourceLimitPolicyRatio: + limitSet[name] = struct{}{} + } + } + } + + requests = sortedResourceNames(requestSet) + limits = sortedResourceNames(limitSet) + + return requests, limits +} + +func sortedResourceNames(set map[corev1.ResourceName]struct{}) []corev1.ResourceName { + out := make([]corev1.ResourceName, 0, len(set)) + for name := range set { + out = append(out, name) + } + + slices.Sort(out) + + return out +} + +func workloadResourceConstraints( + bodies []*apirules.NamespaceRuleEnforceBody, + target apirules.WorkloadValidationTarget, + name corev1.ResourceName, + field workloadResourceField, +) []workloadResourceConstraint { + out := make([]workloadResourceConstraint, 0, 1) + + for _, enforce := range bodies { + if enforce == nil || enforce.Workloads.Resources == nil || + !validationResourcePoliciesTarget(enforce.Workloads, target) { + continue + } + + action := enforce.Action.OrDefault() + + switch field { + case workloadResourceRequests: + policy, found := enforce.Workloads.Resources.Requests[name] + if !found { + continue + } + + switch policy.Policy { + case apirules.WorkloadResourceRequestPolicyPreserve, + apirules.WorkloadResourceRequestPolicyDefault: + out = out[:0] + case apirules.WorkloadResourceRequestPolicyRemove: + out = append(out, workloadResourceConstraint{action: action, policy: string(policy.Policy)}) + } + case workloadResourceLimits: + policy, found := enforce.Workloads.Resources.Limits[name] + if !found { + continue + } + + switch policy.Policy { + case apirules.WorkloadResourceLimitPolicyPreserve, + apirules.WorkloadResourceLimitPolicyDefault: + out = out[:0] + case apirules.WorkloadResourceLimitPolicyRemove, + apirules.WorkloadResourceLimitPolicyMatchRequest, + apirules.WorkloadResourceLimitPolicyRatio: + out = append(out, workloadResourceConstraint{ + action: action, + policy: string(policy.Policy), + value: policy.Value, + }) + } + } + } + + return out +} + +func evaluateWorkloadResourceField( + location workloadResourceLocation, + name corev1.ResourceName, + field workloadResourceField, + constraints []workloadResourceConstraint, +) (*ruleengine.Evaluation, error) { + if len(constraints) == 0 { + return nil, nil + } + + evaluation := &ruleengine.Evaluation{} + hasAllow := false + + var lastDecision *ruleengine.Decision + + for _, constraint := range constraints { + compliant, detail, err := workloadResourceCompliant(location.resources, name, field, constraint) + if err != nil { + return evaluation, err + } + + switch constraint.action { + case apirules.ActionTypeAllow: + hasAllow = true + + if compliant { + lastDecision = workloadResourceDecision(location, name, field, constraint, detail, true) + } + case apirules.ActionTypeDeny: + if !compliant { + lastDecision = workloadResourceDecision(location, name, field, constraint, detail, false) + } + case apirules.ActionTypeAudit: + if !compliant { + evaluation.Audits = append( + evaluation.Audits, + workloadResourceDecision(location, name, field, constraint, detail, false), + ) + } + default: + return evaluation, fmt.Errorf("workload resources: unsupported rule action %q", constraint.action) + } + } + + if lastDecision != nil { + evaluation.Final = lastDecision + if lastDecision.Action == apirules.ActionTypeDeny { + evaluation.Blocking = lastDecision + } + + return evaluation, nil + } + + if hasAllow { + path := workloadResourcePath(location.path, field, name) + evaluation.Blocking = &ruleengine.Decision{ + SetName: "workload resource", + EventReason: events.ReasonForbiddenPodResources, + Action: apirules.ActionTypeDeny, + Value: ruleengine.Value{ + Value: workloadResourceValue(location.resources, name, field), + Path: path, + }, + Message: fmt.Sprintf( + "workload resource %s %q at %s does not satisfy any allowed resource policy", + field, + name, + path, + ), + } + } + + return evaluation, nil +} + +func workloadResourceCompliant( + resources *corev1.ResourceRequirements, + name corev1.ResourceName, + field workloadResourceField, + constraint workloadResourceConstraint, +) (bool, string, error) { + if resources == nil { + resources = &corev1.ResourceRequirements{} + } + + switch field { + case workloadResourceRequests: + _, present := resources.Requests[name] + + return !present, fmt.Sprintf("request must be undefined for policy %s", constraint.policy), nil + case workloadResourceLimits: + limit, limitPresent := resources.Limits[name] + + switch apirules.WorkloadResourceLimitPolicyType(constraint.policy) { + case apirules.WorkloadResourceLimitPolicyPreserve, + apirules.WorkloadResourceLimitPolicyDefault: + return false, "", fmt.Errorf("limit %q policy %q is not an enforcement constraint", name, constraint.policy) + case apirules.WorkloadResourceLimitPolicyRemove: + return !limitPresent, "limit must be undefined for policy Remove", nil + case apirules.WorkloadResourceLimitPolicyMatchRequest: + request, requestPresent := resources.Requests[name] + if !requestPresent { + return !limitPresent, "limit and request must both be undefined or equal for policy MatchRequest", nil + } + + if !limitPresent { + return false, fmt.Sprintf("limit is undefined while request is %s", request.String()), nil + } + + return limit.Cmp(request) == 0, + fmt.Sprintf("limit %s must equal request %s", limit.String(), request.String()), + nil + case apirules.WorkloadResourceLimitPolicyRatio: + if constraint.value == nil { + return false, "", fmt.Errorf("limit %q Ratio policy has no value", name) + } + + request, requestPresent := resources.Requests[name] + if !requestPresent || request.Sign() <= 0 { + return false, "Ratio requires a request greater than zero", nil + } + + if !limitPresent { + return false, "limit is undefined", nil + } + + maximum, err := workloads.LimitForRatio(name, request, *constraint.value) + if err != nil { + return false, "", err + } + + return limit.Cmp(maximum) <= 0, + fmt.Sprintf( + "limit %s must not exceed %s (request %s x ratio %s)", + limit.String(), + maximum.String(), + request.String(), + constraint.value.String(), + ), + nil + default: + return false, "", fmt.Errorf("limit %q has unsupported policy %q", name, constraint.policy) + } + default: + return false, "", fmt.Errorf("unsupported workload resource field %q", field) + } +} + +func workloadResourceDecision( + location workloadResourceLocation, + name corev1.ResourceName, + field workloadResourceField, + constraint workloadResourceConstraint, + detail string, + compliant bool, +) *ruleengine.Decision { + path := workloadResourcePath(location.path, field, name) + + verb := "violates" + + if compliant { + verb = "satisfies" + } + + message := fmt.Sprintf( + "workload resource %s %q at %s %s policy %s", + field, + name, + path, + verb, + constraint.policy, + ) + + if detail != "" { + message += ": " + detail + } + + return &ruleengine.Decision{ + SetName: "workload resource", + EventReason: events.ReasonForbiddenPodResources, + Action: constraint.action, + Value: ruleengine.Value{ + Value: workloadResourceValue(location.resources, name, field), + Path: path, + }, + MatchedRule: constraint.policy, + MatchDetail: detail, + Message: message, + } +} + +func workloadResourcePath( + base string, + field workloadResourceField, + name corev1.ResourceName, +) string { + fieldName := "requests" + if field == workloadResourceLimits { + fieldName = "limits" + } + + return fmt.Sprintf("%s.%s[%q]", base, fieldName, name) +} + +func workloadResourceValue( + resources *corev1.ResourceRequirements, + name corev1.ResourceName, + field workloadResourceField, +) string { + if resources == nil { + return "" + } + + list := resources.Requests + if field == workloadResourceLimits { + list = resources.Limits + } + + value, found := list[name] + if !found { + return "" + } + + return value.String() +} diff --git a/internal/webhook/rules/pods/validation/resources_test.go b/internal/webhook/rules/pods/validation/resources_test.go new file mode 100644 index 00000000..54c7fdaa --- /dev/null +++ b/internal/webhook/rules/pods/validation/resources_test.go @@ -0,0 +1,288 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package validation + +import ( + "strings" + "testing" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + + apirules "github.com/projectcapsule/capsule/pkg/api/rules" +) + +func TestValidateResourcesDenyRatioViolation(t *testing.T) { + t.Parallel() + + h := podRulesForTest() + pod := resourceValidationPod("1Gi", "2Gi") + + evaluation, err := h.validateResources(pod, []*apirules.NamespaceRuleEnforceBody{ + resourceEnforcement(apirules.ActionTypeDeny, "1.5"), + }) + if err != nil { + t.Fatalf("validateResources() error = %v", err) + } + if evaluation == nil || evaluation.Blocking == nil { + t.Fatal("validateResources() did not block a ratio violation") + } + if !strings.Contains(evaluation.Blocking.Message, "must not exceed 1536Mi") { + t.Fatalf("blocking message = %q", evaluation.Blocking.Message) + } +} + +func TestValidateResourcesAllowsCompliantRatio(t *testing.T) { + t.Parallel() + + h := podRulesForTest() + pod := resourceValidationPod("1Gi", "1280Mi") + + evaluation, err := h.validateResources(pod, []*apirules.NamespaceRuleEnforceBody{ + resourceEnforcement(apirules.ActionTypeDeny, "1.5"), + }) + if err != nil { + t.Fatalf("validateResources() error = %v", err) + } + if evaluation == nil || evaluation.Blocking != nil { + t.Fatalf("validateResources() evaluation = %#v, want non-blocking", evaluation) + } +} + +func TestValidateResourcesAllowUsesPolicyAsAllowList(t *testing.T) { + t.Parallel() + + h := podRulesForTest() + + compliant, err := h.validateResources(resourceValidationPod("1Gi", "1280Mi"), []*apirules.NamespaceRuleEnforceBody{ + resourceEnforcement(apirules.ActionTypeAllow, "1.5"), + }) + if err != nil { + t.Fatalf("validateResources(compliant) error = %v", err) + } + if compliant == nil || compliant.Blocking != nil { + t.Fatalf("validateResources(compliant) = %#v, want allowed", compliant) + } + + violating, err := h.validateResources(resourceValidationPod("1Gi", "2Gi"), []*apirules.NamespaceRuleEnforceBody{ + resourceEnforcement(apirules.ActionTypeAllow, "1.5"), + }) + if err != nil { + t.Fatalf("validateResources(violating) error = %v", err) + } + if violating == nil || violating.Blocking == nil || + !strings.Contains(violating.Blocking.Message, "does not satisfy any allowed resource policy") { + t.Fatalf("validateResources(violating) = %#v, want allow-list miss", violating) + } +} + +func TestValidateResourcesLaterAllowOverridesDeny(t *testing.T) { + t.Parallel() + + h := podRulesForTest() + pod := resourceValidationPod("1Gi", "1792Mi") + + evaluation, err := h.validateResources(pod, []*apirules.NamespaceRuleEnforceBody{ + resourceEnforcement(apirules.ActionTypeDeny, "1.5"), + resourceEnforcement(apirules.ActionTypeAllow, "2"), + }) + if err != nil { + t.Fatalf("validateResources() error = %v", err) + } + if evaluation == nil || evaluation.Blocking != nil || evaluation.Final == nil || + evaluation.Final.Action != apirules.ActionTypeAllow { + t.Fatalf("validateResources() = %#v, want later allow decision", evaluation) + } +} + +func TestValidateResourcesLaterPreserveClearsConstraint(t *testing.T) { + t.Parallel() + + h := podRulesForTest() + pod := resourceValidationPod("1Gi", "2Gi") + preserve := &apirules.NamespaceRuleEnforceBody{ + Action: apirules.ActionTypeAllow, + Workloads: apirules.NamespaceRuleEnforceWorkloadsBody{ + Targets: []apirules.WorkloadValidationTarget{apirules.ValidateContainers}, + Resources: &apirules.WorkloadResourceRules{ + Limits: map[corev1.ResourceName]apirules.WorkloadResourceLimitPolicy{ + corev1.ResourceMemory: {Policy: apirules.WorkloadResourceLimitPolicyPreserve}, + }, + }, + }, + } + + evaluation, err := h.validateResources(pod, []*apirules.NamespaceRuleEnforceBody{ + resourceEnforcement(apirules.ActionTypeDeny, "1.5"), + preserve, + }) + if err != nil { + t.Fatalf("validateResources() error = %v", err) + } + if evaluation == nil || evaluation.Blocking != nil { + t.Fatalf("validateResources() = %#v, want later Preserve to clear the constraint", evaluation) + } +} + +func TestValidateResourcesAuditsViolation(t *testing.T) { + t.Parallel() + + h := podRulesForTest() + pod := resourceValidationPod("1Gi", "2Gi") + + evaluation, err := h.validateResources(pod, []*apirules.NamespaceRuleEnforceBody{ + resourceEnforcement(apirules.ActionTypeAudit, "1.5"), + }) + if err != nil { + t.Fatalf("validateResources() error = %v", err) + } + if evaluation == nil || evaluation.Blocking != nil || len(evaluation.Audits) != 1 { + t.Fatalf("validateResources() = %#v, want one non-blocking audit", evaluation) + } +} + +func TestValidateResourcesRequiresRequestForRatio(t *testing.T) { + t.Parallel() + + h := podRulesForTest() + pod := resourceValidationPod("", "1Gi") + + evaluation, err := h.validateResources(pod, []*apirules.NamespaceRuleEnforceBody{ + resourceEnforcement(apirules.ActionTypeDeny, "1.5"), + }) + if err != nil { + t.Fatalf("validateResources() error = %v", err) + } + if evaluation == nil || evaluation.Blocking == nil || + !strings.Contains(evaluation.Blocking.Message, "requires a request greater than zero") { + t.Fatalf("validateResources() = %#v, want missing-request violation", evaluation) + } +} + +func TestValidateResourcesHonorsTarget(t *testing.T) { + t.Parallel() + + h := podRulesForTest() + pod := &corev1.Pod{Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: "app", + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("1Gi")}, + Limits: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("1Gi")}, + }, + }}, + InitContainers: []corev1.Container{{ + Name: "init", + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("1Gi")}, + Limits: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("2Gi")}, + }, + }}, + }} + + enforce := resourceEnforcement(apirules.ActionTypeDeny, "1.5") + enforce.Workloads.Targets = []apirules.WorkloadValidationTarget{apirules.ValidateContainers} + + evaluation, err := h.validateResources(pod, []*apirules.NamespaceRuleEnforceBody{enforce}) + if err != nil { + t.Fatalf("validateResources() error = %v", err) + } + if evaluation == nil || evaluation.Blocking != nil { + t.Fatalf("validateResources() = %#v, init container should be out of scope", evaluation) + } +} + +func TestValidateResourcesDefaultTargetsIncludePodResources(t *testing.T) { + t.Parallel() + + h := podRulesForTest() + pod := resourceValidationPod("1Gi", "1Gi") + pod.Spec.Resources = &corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("1Gi")}, + Limits: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("2Gi")}, + } + + enforce := resourceEnforcement(apirules.ActionTypeDeny, "1.5") + enforce.Workloads.Targets = nil + + evaluation, err := h.validateResources(pod, []*apirules.NamespaceRuleEnforceBody{enforce}) + if err != nil { + t.Fatalf("validateResources() error = %v", err) + } + if evaluation == nil || evaluation.Blocking == nil { + t.Fatal("validateResources() did not enforce the omitted target on Pod-level resources") + } + if !strings.Contains(evaluation.Blocking.Value.Path, "spec.resources") { + t.Fatalf("blocking path = %q, want spec.resources", evaluation.Blocking.Value.Path) + } +} + +func TestValidateResourcesDefaultTargetsSkipPodIncompatibleResources(t *testing.T) { + t.Parallel() + + h := podRulesForTest() + pod := &corev1.Pod{Spec: corev1.PodSpec{Containers: []corev1.Container{{ + Name: "app", + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceEphemeralStorage: resource.MustParse("1Gi")}, + Limits: corev1.ResourceList{corev1.ResourceEphemeralStorage: resource.MustParse("1Gi")}, + }, + }}}} + ratio := resource.MustParse("1.5") + enforce := &apirules.NamespaceRuleEnforceBody{ + Action: apirules.ActionTypeDeny, + Workloads: apirules.NamespaceRuleEnforceWorkloadsBody{ + Resources: &apirules.WorkloadResourceRules{ + Limits: map[corev1.ResourceName]apirules.WorkloadResourceLimitPolicy{ + corev1.ResourceEphemeralStorage: { + Policy: apirules.WorkloadResourceLimitPolicyRatio, + Value: &ratio, + }, + }, + }, + }, + } + + evaluation, err := h.validateResources(pod, []*apirules.NamespaceRuleEnforceBody{enforce}) + if err != nil { + t.Fatalf("validateResources() error = %v", err) + } + if evaluation == nil || evaluation.Blocking != nil { + t.Fatalf("validateResources() = %#v, want pod-incompatible resource skipped at Pod level", evaluation) + } +} + +func resourceValidationPod(request, limit string) *corev1.Pod { + resources := corev1.ResourceRequirements{} + if request != "" { + resources.Requests = corev1.ResourceList{corev1.ResourceMemory: resource.MustParse(request)} + } + if limit != "" { + resources.Limits = corev1.ResourceList{corev1.ResourceMemory: resource.MustParse(limit)} + } + + return &corev1.Pod{Spec: corev1.PodSpec{Containers: []corev1.Container{{ + Name: "app", + Resources: resources, + }}}} +} + +func resourceEnforcement(action apirules.ActionType, ratio string) *apirules.NamespaceRuleEnforceBody { + value := resource.MustParse(ratio) + + return &apirules.NamespaceRuleEnforceBody{ + Action: action, + Workloads: apirules.NamespaceRuleEnforceWorkloadsBody{ + Targets: []apirules.WorkloadValidationTarget{apirules.ValidateContainers}, + Resources: &apirules.WorkloadResourceRules{ + Limits: map[corev1.ResourceName]apirules.WorkloadResourceLimitPolicy{ + corev1.ResourceMemory: { + Policy: apirules.WorkloadResourceLimitPolicyRatio, + Value: &value, + }, + }, + }, + }, + } +} diff --git a/internal/webhook/tenant/validation/rule_quota_test.go b/internal/webhook/tenant/validation/rule_quota_test.go index 55b34f41..cc8bb941 100644 --- a/internal/webhook/tenant/validation/rule_quota_test.go +++ b/internal/webhook/tenant/validation/rule_quota_test.go @@ -23,7 +23,7 @@ import ( tenantutils "github.com/projectcapsule/capsule/pkg/tenant" ) -func TestValidateRuleQuotaUpdatesRejectsHardBelowUsageOrAllocation(t *testing.T) { +func TestValidateRuleQuotaUpdates(t *testing.T) { t.Setenv(configuration.EnvironmentControllerNamespace, "capsule-system") scheme := runtime.NewScheme() @@ -32,11 +32,13 @@ func TestValidateRuleQuotaUpdatesRejectsHardBelowUsageOrAllocation(t *testing.T) } oldTenant := ruleQuotaTenant("5") + desiredGlobalQuota := tenantutils.RuleGlobalResourceQuota(oldTenant, 0, 0) globalQuota := &capsulev1beta2.GlobalResourceQuota{ ObjectMeta: metav1.ObjectMeta{ Name: tenantutils.RuleGlobalResourceQuotaName(oldTenant, "services"), UID: types.UID("global-quota-uid"), }, + Spec: desiredGlobalQuota.Spec, Status: capsulev1beta2.GlobalResourceQuotaStatus{ Total: capsulev1beta2.GlobalResourceQuotaUsage{Used: corev1.ResourceList{ corev1.ResourceServices: resource.MustParse("4"), @@ -71,6 +73,23 @@ func TestValidateRuleQuotaUpdatesRejectsHardBelowUsageOrAllocation(t *testing.T) } }) + t.Run("scope change with decrease", func(t *testing.T) { + oldScoped := ruleQuotaTenant("8") + oldScoped.Spec.Rules[0].NamespaceSelector = &metav1.LabelSelector{ + MatchLabels: map[string]string{"company.example/tier": "application"}, + } + quota := tenantutils.RuleGlobalResourceQuota(oldScoped, 0, 0) + quota.UID = types.UID("scoped-global-quota-uid") + reader := fake.NewClientBuilder().WithScheme(scheme).WithObjects(quota).Build() + + response := validateRuleQuotaUpdates(context.Background(), reader, ruleQuotaTenant("0"), oldScoped) + assertRuleQuotaDenied( + t, + response, + `cannot be reduced from 8 to 0 while namespace selectors are changing`, + ) + }) + t.Run("unchanged", func(t *testing.T) { reader := &failingReader{} if response := validateRuleQuotaUpdates(context.Background(), reader, oldTenant.DeepCopy(), oldTenant); response != nil { diff --git a/internal/webhook/tenant/validation/rule_validator.go b/internal/webhook/tenant/validation/rule_validator.go index 20dbc476..6cd71f89 100644 --- a/internal/webhook/tenant/validation/rule_validator.go +++ b/internal/webhook/tenant/validation/rule_validator.go @@ -134,6 +134,20 @@ func validateRuleQuotaUpdates( } path := fmt.Sprintf("rules[%d].quota[%d].hard", ruleIndex, quotaIndex) + + desiredQuota := tenantutils.RuleGlobalResourceQuota(tnt, ruleIndex, quotaIndex) + if err := runtimequota.ValidateHardLimitScopeChange( + path, + quota.Hard, + globalQuota.Spec.Quota.Hard, + !apiequality.Semantic.DeepEqual( + desiredQuota.Spec.NamespaceSelectors, + globalQuota.Spec.NamespaceSelectors, + ), + ); err != nil { + return ad.Deny(err.Error()) + } + if err := runtimequota.ValidateHardLimit(path, quota.Hard, globalQuota.Status.Total.Used); err != nil { return ad.Deny(err.Error()) } diff --git a/internal/webhook/tenant/validation/warnings.go b/internal/webhook/tenant/validation/warnings.go index 7aeedb8f..0428518c 100644 --- a/internal/webhook/tenant/validation/warnings.go +++ b/internal/webhook/tenant/validation/warnings.go @@ -145,21 +145,21 @@ func deprecatedTenantFieldWarnings(tnt *capsulev1beta2.Tenant) (warnings []strin //nolint:staticcheck if tnt.Spec.ResourceQuota.Scope != "" || len(tnt.Spec.ResourceQuota.Items) > 0 { warnings = append(warnings, - "The field `resourceQuotas` is deprecated and will be removed in a future release. Please migrate to rules quotas. See: https://projectcapsule.dev/docs/tenants/rules/#quotas.", + "The field `resourceQuotas` is deprecated and will be removed in a future release. Please migrate to rules quotas. See: https://projectcapsule.dev/docs/tenants/rules/#migration.", ) } //nolint:staticcheck if tnt.Spec.ServiceOptions != nil { warnings = append(warnings, - "The field `serviceOptions` is deprecated and will be removed in a future release. Please migrate to rules metadata. See: https://projectcapsule.dev/docs/rules/enforcement/metadata/.", + "The field `serviceOptions` is deprecated and will be removed in a future release. Please migrate to rules metadata. See: https://projectcapsule.devdocs/rules/enforcement/metadata/#migrate-service-metadata.", ) } //nolint:staticcheck if tnt.Spec.PodOptions != nil { warnings = append(warnings, - "The field `podOptions` is deprecated and will be removed in a future release. Please migrate to rules metadata. See: https://projectcapsule.dev/docs/rules/enforcement/metadata/.", + "The field `podOptions` is deprecated and will be removed in a future release. Please migrate to rules metadata. See: https://projectcapsule.dev/docs/rules/enforcement/metadata/#migrate-pod-metadata.", ) } @@ -174,21 +174,21 @@ func deprecatedNamespaceOptionWarnings(tnt *capsulev1beta2.Tenant) (warnings []s //nolint:staticcheck if tnt.Spec.NamespaceOptions.AdditionalMetadata != nil { warnings = append(warnings, - "The field `additionalMetadata` is deprecated and will be removed in a future release. Please migrate to rules metadata. See: https://projectcapsule.dev/docs/rules/enforcement/metadata/#namespace.", + "The field `additionalMetadata` is deprecated and will be removed in a future release. Please migrate to rules metadata. See: https://projectcapsule.dev/docs/rules/enforcement/metadata/#migrate-namespace-metadata.", ) } //nolint:staticcheck if len(tnt.Spec.NamespaceOptions.AdditionalMetadataList) > 0 { warnings = append(warnings, - "The field `additionalMetadataList` is deprecated and will be removed in a future release. Please migrate to rules metadata. See: https://projectcapsule.dev/docs/rules/enforcement/metadata/#namespace.", + "The field `additionalMetadataList` is deprecated and will be removed in a future release. Please migrate to rules metadata. See: https://projectcapsule.dev/docs/rules/enforcement/metadata/#migrate-namespace-metadata.", ) } //nolint:staticcheck if tnt.Spec.NamespaceOptions.RequiredMetadata != nil { warnings = append(warnings, - "The field `requiredMetadata` is deprecated and will be removed in a future release. Please migrate to rules metadata. See: https://projectcapsule.dev/docs/rules/enforcement/metadata/#namespace.", + "The field `requiredMetadata` is deprecated and will be removed in a future release. Please migrate to rules metadata. See: https://projectcapsule.dev/docs/rules/enforcement/metadata/#migrate-namespace-metadata.", ) } @@ -196,7 +196,7 @@ func deprecatedNamespaceOptionWarnings(tnt *capsulev1beta2.Tenant) (warnings []s if len(tnt.Spec.NamespaceOptions.ForbiddenLabels.Exact) > 0 || tnt.Spec.NamespaceOptions.ForbiddenLabels.Regex != "" { warnings = append(warnings, - "The field `forbiddenLabels` is deprecated and will be removed in a future release. Please migrate to rules metadata. See: https://projectcapsule.dev/docs/rules/enforcement/metadata/#namespace.", + "The field `forbiddenLabels` is deprecated and will be removed in a future release. Please migrate to rules metadata. See: https://projectcapsule.dev/docs/rules/enforcement/metadata/#migrate-namespace-metadata.", ) } @@ -204,7 +204,7 @@ func deprecatedNamespaceOptionWarnings(tnt *capsulev1beta2.Tenant) (warnings []s if len(tnt.Spec.NamespaceOptions.ForbiddenAnnotations.Exact) > 0 || tnt.Spec.NamespaceOptions.ForbiddenAnnotations.Regex != "" { warnings = append(warnings, - "The field `forbiddenAnnotations` is deprecated and will be removed in a future release. Please migrate to rules metadata. See: https://projectcapsule.dev/docs/rules/enforcement/metadata/#namespace.", + "The field `forbiddenAnnotations` is deprecated and will be removed in a future release. Please migrate to rules metadata. See: https://projectcapsule.dev/docs/rules/enforcement/metadata/#migrate-namespace-metadata.", ) } diff --git a/pkg/api/rules/enforce_workloads_types.go b/pkg/api/rules/enforce_workloads_types.go index f862d4ed..5644cc8c 100644 --- a/pkg/api/rules/enforce_workloads_types.go +++ b/pkg/api/rules/enforce_workloads_types.go @@ -5,16 +5,18 @@ package rules import ( corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" "github.com/projectcapsule/capsule/pkg/api/runtime" ) -// +kubebuilder:validation:Enum=pod/initcontainers;pod/ephemeralcontainers;pod/containers;pod/volumes +// +kubebuilder:validation:Enum=pod;pod/initcontainers;pod/ephemeralcontainers;pod/containers;pod/volumes type WorkloadValidationTarget string const ( DeprecatedValidateImages WorkloadValidationTarget = "pod/images" + ValidatePod WorkloadValidationTarget = "pod" ValidateInitContainers WorkloadValidationTarget = "pod/initcontainers" ValidateEphemeralContainers WorkloadValidationTarget = "pod/ephemeralcontainers" ValidateContainers WorkloadValidationTarget = "pod/containers" @@ -28,6 +30,20 @@ type NamespaceRuleEnforceWorkloadsBody struct { // +optional Targets []WorkloadValidationTarget `json:"targets,omitempty"` + // Resources defines mutation and enforcement policies for Pod and container + // resource requests and limits. The workload targets select where the + // policies apply. With no targets, resource policies apply to all compatible + // locations: Pod-level resources, regular containers, and init containers. + // Resource names unsupported at Pod level still apply to compatible container + // locations. + // Mutation is applied when a Pod is created. Remove and MatchRequest manage + // explicit values, Default fills an absent value, and Ratio fills an absent + // limit from its request. An explicit Ratio violation is then handled by the + // enclosing allow, deny, or audit action. + // + // +optional + Resources *WorkloadResourceRules `json:"resources,omitempty"` + // Define Pod QoS classes matched by this enforcement rule. // Supported values are Guaranteed, Burstable and BestEffort. // +optional @@ -46,3 +62,70 @@ type NamespaceRuleEnforceWorkloadsBody struct { // +optional Schedulers []runtime.ExpressionMatch `json:"schedulers,omitempty"` } + +type WorkloadResourceRequestPolicyType string + +const ( + WorkloadResourceRequestPolicyPreserve WorkloadResourceRequestPolicyType = "Preserve" + WorkloadResourceRequestPolicyDefault WorkloadResourceRequestPolicyType = "Default" + WorkloadResourceRequestPolicyRemove WorkloadResourceRequestPolicyType = "Remove" +) + +type WorkloadResourceLimitPolicyType string + +const ( + WorkloadResourceLimitPolicyPreserve WorkloadResourceLimitPolicyType = "Preserve" + WorkloadResourceLimitPolicyDefault WorkloadResourceLimitPolicyType = "Default" + WorkloadResourceLimitPolicyRemove WorkloadResourceLimitPolicyType = "Remove" + WorkloadResourceLimitPolicyMatchRequest WorkloadResourceLimitPolicyType = "MatchRequest" + WorkloadResourceLimitPolicyRatio WorkloadResourceLimitPolicyType = "Ratio" +) + +// WorkloadResourceRules defines policies keyed by Kubernetes resource name. +// +// +kubebuilder:object:generate=true +// +kubebuilder:validation:XValidation:rule="has(self.requests) || has(self.limits)",message="at least one of requests or limits must be set" +type WorkloadResourceRules struct { + // Requests defines policies for resource requests. + // +optional + // +kubebuilder:validation:MinProperties=1 + Requests map[corev1.ResourceName]WorkloadResourceRequestPolicy `json:"requests,omitempty"` + + // Limits defines policies for resource limits. + // +optional + // +kubebuilder:validation:MinProperties=1 + Limits map[corev1.ResourceName]WorkloadResourceLimitPolicy `json:"limits,omitempty"` +} + +// WorkloadResourceRequestPolicy defines how a resource request is mutated. +// +// +kubebuilder:object:generate=true +// +kubebuilder:validation:XValidation:rule="self.policy == 'Default' ? has(self.value) : !has(self.value)",message="value must be set only for the Default policy" +type WorkloadResourceRequestPolicy struct { + // Policy selects how the request is handled: Preserve leaves it unchanged, + // Default fills an absent request, and Remove deletes it. + // +kubebuilder:validation:Enum=Preserve;Default;Remove + Policy WorkloadResourceRequestPolicyType `json:"policy"` + + // Value is the quantity applied by the Default policy. + // +optional + Value *resource.Quantity `json:"value,omitempty"` +} + +// WorkloadResourceLimitPolicy defines how a resource limit is mutated and enforced. +// +// +kubebuilder:object:generate=true +// +kubebuilder:validation:XValidation:rule="self.policy == 'Default' || self.policy == 'Ratio' ? has(self.value) : !has(self.value)",message="value must be set only for the Default and Ratio policies" +type WorkloadResourceLimitPolicy struct { + // Policy selects how the limit is handled: Preserve leaves it unchanged, + // Default fills an absent limit, Remove deletes it, MatchRequest manages it + // to equal the request, and Ratio defaults an absent limit and enforces the + // maximum multiplier against explicitly supplied limits. + // +kubebuilder:validation:Enum=Preserve;Default;Remove;MatchRequest;Ratio + Policy WorkloadResourceLimitPolicyType `json:"policy"` + + // Value is the quantity applied by Default or the maximum limit-to-request + // multiplier applied by Ratio. + // +optional + Value *resource.Quantity `json:"value,omitempty"` +} diff --git a/pkg/api/rules/zz_generated.deepcopy.go b/pkg/api/rules/zz_generated.deepcopy.go index 8db5eb6e..30871b2a 100644 --- a/pkg/api/rules/zz_generated.deepcopy.go +++ b/pkg/api/rules/zz_generated.deepcopy.go @@ -244,6 +244,11 @@ func (in *NamespaceRuleEnforceWorkloadsBody) DeepCopyInto(out *NamespaceRuleEnfo *out = make([]WorkloadValidationTarget, len(*in)) copy(*out, *in) } + if in.Resources != nil { + in, out := &in.Resources, &out.Resources + *out = new(WorkloadResourceRules) + (*in).DeepCopyInto(*out) + } if in.QoSClasses != nil { in, out := &in.QoSClasses, &out.QoSClasses *out = make([]v1.PodQOSClass, len(*in)) @@ -446,3 +451,72 @@ func (in *ServiceNodePortRule) DeepCopy() *ServiceNodePortRule { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *WorkloadResourceLimitPolicy) DeepCopyInto(out *WorkloadResourceLimitPolicy) { + *out = *in + if in.Value != nil { + in, out := &in.Value, &out.Value + x := (*in).DeepCopy() + *out = &x + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WorkloadResourceLimitPolicy. +func (in *WorkloadResourceLimitPolicy) DeepCopy() *WorkloadResourceLimitPolicy { + if in == nil { + return nil + } + out := new(WorkloadResourceLimitPolicy) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *WorkloadResourceRequestPolicy) DeepCopyInto(out *WorkloadResourceRequestPolicy) { + *out = *in + if in.Value != nil { + in, out := &in.Value, &out.Value + x := (*in).DeepCopy() + *out = &x + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WorkloadResourceRequestPolicy. +func (in *WorkloadResourceRequestPolicy) DeepCopy() *WorkloadResourceRequestPolicy { + if in == nil { + return nil + } + out := new(WorkloadResourceRequestPolicy) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *WorkloadResourceRules) DeepCopyInto(out *WorkloadResourceRules) { + *out = *in + if in.Requests != nil { + in, out := &in.Requests, &out.Requests + *out = make(map[v1.ResourceName]WorkloadResourceRequestPolicy, len(*in)) + for key, val := range *in { + (*out)[key] = *val.DeepCopy() + } + } + if in.Limits != nil { + in, out := &in.Limits, &out.Limits + *out = make(map[v1.ResourceName]WorkloadResourceLimitPolicy, len(*in)) + for key, val := range *in { + (*out)[key] = *val.DeepCopy() + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WorkloadResourceRules. +func (in *WorkloadResourceRules) DeepCopy() *WorkloadResourceRules { + if in == nil { + return nil + } + out := new(WorkloadResourceRules) + in.DeepCopyInto(out) + return out +} diff --git a/pkg/ruleengine/validate.go b/pkg/ruleengine/validate.go index 92f0792a..c0c36d8c 100644 --- a/pkg/ruleengine/validate.go +++ b/pkg/ruleengine/validate.go @@ -9,12 +9,15 @@ import ( "regexp" "strings" + corev1 "k8s.io/api/core/v1" k8smeta "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/runtime/schema" k8svalidation "k8s.io/apimachinery/pkg/util/validation" "github.com/projectcapsule/capsule/pkg/api/rules" "github.com/projectcapsule/capsule/pkg/api/runtime" + workloadruntime "github.com/projectcapsule/capsule/pkg/runtime/workloads" ) func ValidateRuleStatusBody( @@ -172,6 +175,28 @@ func validateWorkloadRules( ruleIndex int, workloads rules.NamespaceRuleEnforceWorkloadsBody, ) error { + for j, target := range workloads.Targets { + switch target { + case rules.DeprecatedValidateImages, + rules.ValidatePod, + rules.ValidateInitContainers, + rules.ValidateEphemeralContainers, + rules.ValidateContainers, + rules.ValidateVolumes: + default: + return fmt.Errorf( + "rules[%d].enforce.workloads.targets[%d] %q is invalid: unsupported workload target", + ruleIndex, + j, + target, + ) + } + } + + if err := validateWorkloadResourceRules(ruleIndex, workloads); err != nil { + return err + } + for j, registry := range workloads.Registries { if err := validateExpression( registry.Expression, @@ -193,6 +218,173 @@ func validateWorkloadRules( return nil } +func validateWorkloadResourceRules( + ruleIndex int, + workloads rules.NamespaceRuleEnforceWorkloadsBody, +) error { + resources := workloads.Resources + if resources == nil { + return nil + } + + path := fmt.Sprintf("rules[%d].enforce.workloads.resources", ruleIndex) + if len(resources.Requests) == 0 && len(resources.Limits) == 0 { + return fmt.Errorf("%s is invalid: at least one request or limit policy is required", path) + } + + podTarget, err := validateWorkloadResourceTargets(path, workloads.Targets) + if err != nil { + return err + } + + if err := validateWorkloadRequestPolicies(path, resources.Requests, podTarget); err != nil { + return err + } + + return validateWorkloadLimitPolicies(path, resources, podTarget) +} + +func validateWorkloadResourceTargets( + path string, + targets []rules.WorkloadValidationTarget, +) (bool, error) { + podTarget := false + + for _, target := range targets { + switch target { + case rules.ValidatePod: + podTarget = true + case rules.ValidateContainers, rules.ValidateInitContainers: + case rules.ValidateEphemeralContainers, rules.ValidateVolumes, rules.DeprecatedValidateImages: + return false, fmt.Errorf( + "%s is invalid: workload target %q does not support resource policies", + path, + target, + ) + } + } + + return podTarget, nil +} + +func validateWorkloadRequestPolicies( + path string, + policies map[corev1.ResourceName]rules.WorkloadResourceRequestPolicy, + podTarget bool, +) error { + for name, policy := range policies { + policyPath := fmt.Sprintf("%s.requests[%q]", path, name) + if err := validateWorkloadResourceName(name, podTarget); err != nil { + return fmt.Errorf("%s is invalid: %w", policyPath, err) + } + + switch policy.Policy { + case rules.WorkloadResourceRequestPolicyPreserve, + rules.WorkloadResourceRequestPolicyRemove: + if policy.Value != nil { + return fmt.Errorf("%s.value is invalid: value is only supported by the Default policy", policyPath) + } + case rules.WorkloadResourceRequestPolicyDefault: + if err := validateDefaultResourceQuantity(policyPath, policy.Value); err != nil { + return err + } + default: + return fmt.Errorf("%s.policy %q is invalid: unsupported request policy", policyPath, policy.Policy) + } + } + + return nil +} + +func validateWorkloadLimitPolicies( + path string, + resources *rules.WorkloadResourceRules, + podTarget bool, +) error { + one := resource.MustParse("1") + + for name, policy := range resources.Limits { + policyPath := fmt.Sprintf("%s.limits[%q]", path, name) + if err := validateWorkloadResourceName(name, podTarget); err != nil { + return fmt.Errorf("%s is invalid: %w", policyPath, err) + } + + switch policy.Policy { + case rules.WorkloadResourceLimitPolicyPreserve, + rules.WorkloadResourceLimitPolicyRemove, + rules.WorkloadResourceLimitPolicyMatchRequest: + if policy.Value != nil { + return fmt.Errorf( + "%s.value is invalid: value is only supported by the Default and Ratio policies", + policyPath, + ) + } + case rules.WorkloadResourceLimitPolicyDefault: + if err := validateDefaultResourceQuantity(policyPath, policy.Value); err != nil { + return err + } + case rules.WorkloadResourceLimitPolicyRatio: + if policy.Value == nil { + return fmt.Errorf("%s.value is invalid: Ratio requires a value", policyPath) + } + + if !workloadruntime.RatioSupportedResource(name) { + return fmt.Errorf( + "%s.policy is invalid: Ratio is only supported for cpu, memory, and ephemeral-storage", + policyPath, + ) + } + + if policy.Value.Cmp(one) < 0 { + return fmt.Errorf("%s.value is invalid: Ratio must be greater than or equal to 1", policyPath) + } + default: + return fmt.Errorf("%s.policy %q is invalid: unsupported limit policy", policyPath, policy.Policy) + } + + if requestPolicy, found := resources.Requests[name]; found && + requestPolicy.Policy == rules.WorkloadResourceRequestPolicyRemove && + (policy.Policy == rules.WorkloadResourceLimitPolicyMatchRequest || + policy.Policy == rules.WorkloadResourceLimitPolicyRatio) { + return fmt.Errorf( + "%s is invalid: %s requires a request which is removed by the request policy", + policyPath, + policy.Policy, + ) + } + } + + return nil +} + +func validateDefaultResourceQuantity(path string, value *resource.Quantity) error { + if value == nil { + return fmt.Errorf("%s.value is invalid: Default requires a value", path) + } + + if value.Sign() < 0 { + return fmt.Errorf("%s.value is invalid: quantity must not be negative", path) + } + + return nil +} + +func validateWorkloadResourceName(name corev1.ResourceName, podTarget bool) error { + if errs := k8svalidation.IsQualifiedName(string(name)); len(errs) > 0 { + return fmt.Errorf("resource name is invalid: %s", strings.Join(errs, "; ")) + } + + if !podTarget { + return nil + } + + if workloadruntime.PodLevelResourceSupported(name) { + return nil + } + + return fmt.Errorf("resource %q is not supported by pod-level resources", name) +} + func validateServiceRules( ruleIndex int, services rules.NamespaceRuleEnforceServicesBody, diff --git a/pkg/ruleengine/validate_resources_test.go b/pkg/ruleengine/validate_resources_test.go new file mode 100644 index 00000000..12a84891 --- /dev/null +++ b/pkg/ruleengine/validate_resources_test.go @@ -0,0 +1,145 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package ruleengine + +import ( + "strings" + "testing" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + + "github.com/projectcapsule/capsule/pkg/api/rules" +) + +func TestValidateRuleStatusBodyWorkloadResources(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + body *rules.NamespaceRuleBodyNamespace + wantErr string + }{ + { + name: "valid", + body: workloadResourceRuleForValidation( + []rules.WorkloadValidationTarget{rules.ValidateContainers}, + rules.WorkloadResourceRequestPolicy{Policy: rules.WorkloadResourceRequestPolicyDefault, Value: validationQuantity("1Gi")}, + rules.WorkloadResourceLimitPolicy{Policy: rules.WorkloadResourceLimitPolicyRatio, Value: validationQuantity("1.5")}, + ), + }, + { + name: "omitted target permits container-only resource names", + body: workloadResourceRuleForValidationWithName( + nil, + corev1.ResourceEphemeralStorage, + rules.WorkloadResourceRequestPolicy{Policy: rules.WorkloadResourceRequestPolicyPreserve}, + rules.WorkloadResourceLimitPolicy{Policy: rules.WorkloadResourceLimitPolicyRatio, Value: validationQuantity("1.5")}, + ), + }, + { + name: "ratio below one", + body: workloadResourceRuleForValidation( + nil, + rules.WorkloadResourceRequestPolicy{Policy: rules.WorkloadResourceRequestPolicyPreserve}, + rules.WorkloadResourceLimitPolicy{Policy: rules.WorkloadResourceLimitPolicyRatio, Value: validationQuantity("0.5")}, + ), + wantErr: "Ratio must be greater than or equal to 1", + }, + { + name: "ratio missing value", + body: workloadResourceRuleForValidation( + nil, + rules.WorkloadResourceRequestPolicy{Policy: rules.WorkloadResourceRequestPolicyPreserve}, + rules.WorkloadResourceLimitPolicy{Policy: rules.WorkloadResourceLimitPolicyRatio}, + ), + wantErr: "Ratio requires a value", + }, + { + name: "unsupported target", + body: workloadResourceRuleForValidation( + []rules.WorkloadValidationTarget{rules.ValidateEphemeralContainers}, + rules.WorkloadResourceRequestPolicy{Policy: rules.WorkloadResourceRequestPolicyPreserve}, + rules.WorkloadResourceLimitPolicy{Policy: rules.WorkloadResourceLimitPolicyRatio, Value: validationQuantity("1.5")}, + ), + wantErr: "does not support resource policies", + }, + { + name: "pod-level ephemeral storage", + body: workloadResourceRuleForValidation( + []rules.WorkloadValidationTarget{rules.ValidatePod}, + rules.WorkloadResourceRequestPolicy{Policy: rules.WorkloadResourceRequestPolicyPreserve}, + rules.WorkloadResourceLimitPolicy{Policy: rules.WorkloadResourceLimitPolicyRatio, Value: validationQuantity("1.5")}, + ), + wantErr: "is not supported by pod-level resources", + }, + { + name: "ratio request removed", + body: workloadResourceRuleForValidation( + nil, + rules.WorkloadResourceRequestPolicy{Policy: rules.WorkloadResourceRequestPolicyRemove}, + rules.WorkloadResourceLimitPolicy{Policy: rules.WorkloadResourceLimitPolicyRatio, Value: validationQuantity("1.5")}, + ), + wantErr: "requires a request which is removed", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + err := ValidateRuleStatusBody(nil, []*rules.NamespaceRuleBodyNamespace{tt.body}) + if tt.wantErr == "" { + if err != nil { + t.Fatalf("ValidateRuleStatusBody() error = %v", err) + } + + return + } + + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("ValidateRuleStatusBody() error = %v, want containing %q", err, tt.wantErr) + } + }) + } +} + +func workloadResourceRuleForValidation( + targets []rules.WorkloadValidationTarget, + requestPolicy rules.WorkloadResourceRequestPolicy, + limitPolicy rules.WorkloadResourceLimitPolicy, +) *rules.NamespaceRuleBodyNamespace { + name := corev1.ResourceMemory + if len(targets) == 1 && targets[0] == rules.ValidatePod { + name = corev1.ResourceEphemeralStorage + } + + return workloadResourceRuleForValidationWithName(targets, name, requestPolicy, limitPolicy) +} + +func workloadResourceRuleForValidationWithName( + targets []rules.WorkloadValidationTarget, + name corev1.ResourceName, + requestPolicy rules.WorkloadResourceRequestPolicy, + limitPolicy rules.WorkloadResourceLimitPolicy, +) *rules.NamespaceRuleBodyNamespace { + + return &rules.NamespaceRuleBodyNamespace{ + Enforce: &rules.NamespaceRuleEnforceBody{ + Workloads: rules.NamespaceRuleEnforceWorkloadsBody{ + Targets: targets, + Resources: &rules.WorkloadResourceRules{ + Requests: map[corev1.ResourceName]rules.WorkloadResourceRequestPolicy{name: requestPolicy}, + Limits: map[corev1.ResourceName]rules.WorkloadResourceLimitPolicy{name: limitPolicy}, + }, + }, + }, + } +} + +func validationQuantity(value string) *resource.Quantity { + quantity := resource.MustParse(value) + + return &quantity +} diff --git a/pkg/runtime/events/reasons.go b/pkg/runtime/events/reasons.go index fddbb52e..9eb58f85 100644 --- a/pkg/runtime/events/reasons.go +++ b/pkg/runtime/events/reasons.go @@ -49,6 +49,7 @@ const ( ReasonForbiddenPullPolicy string = "ForbiddenPullPolicy" ReasonForbiddenPodQoSClass string = "ForbiddenQoSClass" ReasonForbiddenPodScheduler string = "ForbiddenScheduler" + ReasonForbiddenPodResources string = "ForbiddenPodResources" // Ingress. ReasonWildcardDenied string = "WildcardDenied" diff --git a/pkg/runtime/quota/validation.go b/pkg/runtime/quota/validation.go index e8d3941e..8245ce8a 100644 --- a/pkg/runtime/quota/validation.go +++ b/pkg/runtime/quota/validation.go @@ -41,3 +41,41 @@ func ValidateHardLimit(path string, hard, allocated corev1.ResourceList) error { return nil } + +// ValidateHardLimitScopeChange prevents a quota from reducing or removing a +// hard limit in the same update that changes its namespace selection. Usage +// from newly selected namespaces is not represented by the quota's previous +// status yet, so the scope must reconcile before a safe lower bound is known. +func ValidateHardLimitScopeChange( + path string, + hard corev1.ResourceList, + previous corev1.ResourceList, + scopeChanged bool, +) error { + if !scopeChanged { + return nil + } + + for name, previousLimit := range previous { + limit, exists := hard[name] + if !exists { + return fmt.Errorf( + "%s[%q] cannot be removed while namespace selectors are changing; update the selectors first and wait for usage reconciliation", + path, + name, + ) + } + + if limit.Cmp(previousLimit) < 0 { + return fmt.Errorf( + "%s[%q] cannot be reduced from %s to %s while namespace selectors are changing; update the selectors first and wait for usage reconciliation", + path, + name, + previousLimit.String(), + limit.String(), + ) + } + } + + return nil +} diff --git a/pkg/runtime/quota/validation_test.go b/pkg/runtime/quota/validation_test.go new file mode 100644 index 00000000..25662882 --- /dev/null +++ b/pkg/runtime/quota/validation_test.go @@ -0,0 +1,86 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package quota + +import ( + "strings" + "testing" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" +) + +func TestValidateHardLimitScopeChange(t *testing.T) { + t.Parallel() + + previous := corev1.ResourceList{ + corev1.ResourceLimitsCPU: resource.MustParse("8"), + } + + tests := []struct { + name string + hard corev1.ResourceList + scopeChanged bool + wantErr string + }{ + { + name: "rejects decrease while scope changes", + hard: corev1.ResourceList{ + corev1.ResourceLimitsCPU: resource.MustParse("0"), + }, + scopeChanged: true, + wantErr: "cannot be reduced from 8 to 0 while namespace selectors are changing", + }, + { + name: "rejects removal while scope changes", + hard: corev1.ResourceList{}, + scopeChanged: true, + wantErr: "cannot be removed while namespace selectors are changing", + }, + { + name: "allows equal limit while scope changes", + hard: corev1.ResourceList{ + corev1.ResourceLimitsCPU: resource.MustParse("8"), + }, + scopeChanged: true, + }, + { + name: "allows increase while scope changes", + hard: corev1.ResourceList{ + corev1.ResourceLimitsCPU: resource.MustParse("10"), + }, + scopeChanged: true, + }, + { + name: "defers unchanged-scope decrease to usage validation", + hard: corev1.ResourceList{ + corev1.ResourceLimitsCPU: resource.MustParse("0"), + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + err := ValidateHardLimitScopeChange( + "spec.quota.hard", + test.hard, + previous, + test.scopeChanged, + ) + if test.wantErr == "" { + if err != nil { + t.Fatalf("ValidateHardLimitScopeChange() error = %v", err) + } + + return + } + + if err == nil || !strings.Contains(err.Error(), test.wantErr) { + t.Fatalf("ValidateHardLimitScopeChange() error = %v, want containing %q", err, test.wantErr) + } + }) + } +} diff --git a/pkg/runtime/workloads/resource_quantity.go b/pkg/runtime/workloads/resource_quantity.go new file mode 100644 index 00000000..62175161 --- /dev/null +++ b/pkg/runtime/workloads/resource_quantity.go @@ -0,0 +1,62 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package workloads + +import ( + "fmt" + "strings" + + inf "gopkg.in/inf.v0" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" +) + +// PodLevelResourceSupported reports whether Kubernetes permits the resource +// name in Pod-level resource requirements. +func PodLevelResourceSupported(name corev1.ResourceName) bool { + return name == corev1.ResourceCPU || + name == corev1.ResourceMemory || + strings.HasPrefix(string(name), corev1.ResourceHugePagesPrefix) +} + +// RatioSupportedResource reports whether Capsule can safely calculate a +// limit-to-request ratio for the resource. CPU is rounded down to milliCPU; +// byte-based resources are rounded down to whole bytes. +func RatioSupportedResource(name corev1.ResourceName) bool { + return name == corev1.ResourceCPU || + name == corev1.ResourceMemory || + name == corev1.ResourceEphemeralStorage +} + +// LimitForRatio calculates request * ratio without floating-point arithmetic. +// The result is rounded down so it never exceeds the configured maximum ratio. +func LimitForRatio( + name corev1.ResourceName, + request resource.Quantity, + ratio resource.Quantity, +) (resource.Quantity, error) { + if !RatioSupportedResource(name) { + return resource.Quantity{}, fmt.Errorf("ratio is not supported for resource %q", name) + } + + if request.Sign() <= 0 { + return resource.Quantity{}, fmt.Errorf("request for resource %q must be greater than zero", name) + } + + if ratio.Cmp(resource.MustParse("1")) < 0 { + return resource.Quantity{}, fmt.Errorf("ratio for resource %q must be greater than or equal to 1", name) + } + + product := new(inf.Dec).Mul(request.AsDec(), ratio.AsDec()) + + scale := inf.Scale(0) + + if name == corev1.ResourceCPU { + scale = inf.Scale(3) + } + + rounded := new(inf.Dec).Round(product, scale, inf.RoundDown) + + return *resource.NewDecimalQuantity(*rounded, request.Format), nil +} diff --git a/pkg/runtime/workloads/resource_quantity_test.go b/pkg/runtime/workloads/resource_quantity_test.go new file mode 100644 index 00000000..f0719daf --- /dev/null +++ b/pkg/runtime/workloads/resource_quantity_test.go @@ -0,0 +1,113 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package workloads + +import ( + "testing" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" +) + +func TestPodLevelResourceSupported(t *testing.T) { + t.Parallel() + + tests := []struct { + name corev1.ResourceName + supported bool + }{ + {name: corev1.ResourceCPU, supported: true}, + {name: corev1.ResourceMemory, supported: true}, + {name: corev1.ResourceName("hugepages-2Mi"), supported: true}, + {name: corev1.ResourceEphemeralStorage, supported: false}, + {name: corev1.ResourceName("example.com/gpu"), supported: false}, + } + + for _, tt := range tests { + t.Run(string(tt.name), func(t *testing.T) { + t.Parallel() + + if got := PodLevelResourceSupported(tt.name); got != tt.supported { + t.Fatalf("PodLevelResourceSupported(%q) = %t, want %t", tt.name, got, tt.supported) + } + }) + } +} + +func TestLimitForRatio(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + resource corev1.ResourceName + request string + ratio string + want string + }{ + { + name: "memory", + resource: corev1.ResourceMemory, + request: "1Gi", + ratio: "1.5", + want: "1536Mi", + }, + { + name: "cpu", + resource: corev1.ResourceCPU, + request: "100m", + ratio: "1.5", + want: "150m", + }, + { + name: "cpu rounds down to milliCPU", + resource: corev1.ResourceCPU, + request: "1m", + ratio: "1.5", + want: "1m", + }, + { + name: "storage rounds down to bytes", + resource: corev1.ResourceEphemeralStorage, + request: "3", + ratio: "1.5", + want: "4", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got, err := LimitForRatio( + tt.resource, + resource.MustParse(tt.request), + resource.MustParse(tt.ratio), + ) + if err != nil { + t.Fatalf("LimitForRatio() error = %v", err) + } + + want := resource.MustParse(tt.want) + if got.Cmp(want) != 0 { + t.Fatalf("LimitForRatio() = %s, want %s", got.String(), want.String()) + } + }) + } +} + +func TestLimitForRatioRejectsInvalidInputs(t *testing.T) { + t.Parallel() + + if _, err := LimitForRatio(corev1.ResourceName("example.com/gpu"), resource.MustParse("1"), resource.MustParse("1.5")); err == nil { + t.Fatal("LimitForRatio() accepted an extended resource") + } + + if _, err := LimitForRatio(corev1.ResourceCPU, resource.MustParse("0"), resource.MustParse("1.5")); err == nil { + t.Fatal("LimitForRatio() accepted a zero request") + } + + if _, err := LimitForRatio(corev1.ResourceCPU, resource.MustParse("1"), resource.MustParse("0.5")); err == nil { + t.Fatal("LimitForRatio() accepted a ratio below one") + } +} diff --git a/pkg/template/rules.go b/pkg/template/rules.go index 6f7c548c..6244a22b 100644 --- a/pkg/template/rules.go +++ b/pkg/template/rules.go @@ -6,7 +6,7 @@ package template import ( "fmt" - "go.yaml.in/yaml/v2" + "sigs.k8s.io/yaml" "github.com/projectcapsule/capsule/pkg/api/rules" ) diff --git a/pkg/template/rules_test.go b/pkg/template/rules_test.go index 48497938..fcad7881 100644 --- a/pkg/template/rules_test.go +++ b/pkg/template/rules_test.go @@ -8,6 +8,7 @@ import ( "testing" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" "github.com/projectcapsule/capsule/pkg/api/rules" "github.com/projectcapsule/capsule/pkg/api/runtime" @@ -274,6 +275,57 @@ func TestRenderNamespaceRuleBodies(t *testing.T) { } } +func TestRenderNamespaceRuleBodiesPreservesResourceQuantities(t *testing.T) { + t.Parallel() + + ratio := resource.MustParse("1.5") + request := resource.MustParse("250m") + bodies := []*rules.NamespaceRuleBodyNamespace{{ + Enforce: &rules.NamespaceRuleEnforceBody{ + Workloads: rules.NamespaceRuleEnforceWorkloadsBody{ + Resources: &rules.WorkloadResourceRules{ + Requests: map[corev1.ResourceName]rules.WorkloadResourceRequestPolicy{ + corev1.ResourceCPU: { + Policy: rules.WorkloadResourceRequestPolicyDefault, + Value: &request, + }, + }, + Limits: map[corev1.ResourceName]rules.WorkloadResourceLimitPolicy{ + corev1.ResourceCPU: { + Policy: rules.WorkloadResourceLimitPolicyRatio, + Value: &ratio, + }, + }, + }, + }, + }, + }} + + got, err := RenderNamespaceRuleBodies(nil, MissingKeyError, bodies) + if err != nil { + t.Fatalf("RenderNamespaceRuleBodies() unexpected error: %v", err) + } + + if len(got) != 1 || got[0] == nil || got[0].Enforce == nil { + t.Fatalf("RenderNamespaceRuleBodies() = %#v, want one enforce body", got) + } + + resources := got[0].Enforce.Workloads.Resources + if resources == nil { + t.Fatal("rendered resources are nil") + } + + gotRequest := resources.Requests[corev1.ResourceCPU].Value + if gotRequest == nil || gotRequest.Cmp(request) != 0 { + t.Fatalf("rendered request = %v, want %s", gotRequest, request.String()) + } + + gotRatio := resources.Limits[corev1.ResourceCPU].Value + if gotRatio == nil || gotRatio.Cmp(ratio) != 0 { + t.Fatalf("rendered ratio = %v, want %s", gotRatio, ratio.String()) + } +} + func TestRenderNamespaceRuleBodies_DoesNotMutateInput(t *testing.T) { t.Parallel() diff --git a/playground/installation/capsule/release.flux.yaml b/playground/installation/capsule/release.flux.yaml index 85deccb4..c35064b9 100644 --- a/playground/installation/capsule/release.flux.yaml +++ b/playground/installation/capsule/release.flux.yaml @@ -111,6 +111,8 @@ spec: leaderElection: true oidcUsernameClaim: "${OIDC_USERNAME_CLAIM}" roleBindingReflector: true + ignoredUsernames: + - "admin" extraArgs: - "--feature-gates=ProxyClusterScoped=true" service: diff --git a/playground/installation/headlamp/release.flux.yaml b/playground/installation/headlamp/release.flux.yaml index 88afe790..40c3e5ba 100644 --- a/playground/installation/headlamp/release.flux.yaml +++ b/playground/installation/headlamp/release.flux.yaml @@ -10,7 +10,6 @@ spec: targetNamespace: capsule-system serviceAccountName: kustomize-controller dependsOn: - - name: capsule - name: dex chart: spec: @@ -87,12 +86,12 @@ spec: items: - key: ca path: ca.crt - - downwardAPI: null - items: - - fieldRef: - apiVersion: v1 - fieldPath: metadata.namespace - path: namespace + - downwardAPI: + items: + - fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + path: namespace initContainers: - name: add-ca image: docker.io/alpine:3 diff --git a/playground/platform/globalresourcequotas/green-shared-compute.yaml b/playground/platform/globalresourcequotas/green-shared-compute.yaml index b57666b3..3f64b427 100644 --- a/playground/platform/globalresourcequotas/green-shared-compute.yaml +++ b/playground/platform/globalresourcequotas/green-shared-compute.yaml @@ -11,7 +11,7 @@ spec: capsule.clastix.io/tenant: green quota: hard: - limits.cpu: "8" + limits.cpu: "0" limits.memory: 16Gi requests.cpu: "8" requests.memory: 16Gi diff --git a/playground/platform/resourcepools/pool-renewable.yaml b/playground/platform/resourcepools/pool-renewable.yaml index 6300d7b0..579ed1cc 100644 --- a/playground/platform/resourcepools/pool-renewable.yaml +++ b/playground/platform/resourcepools/pool-renewable.yaml @@ -6,8 +6,8 @@ metadata: spec: quota: hard: - limits.cpu: "2" limits.memory: 2Gi + limits.cpu: "0" requests.cpu: "2" requests.memory: 2Gi requests.storage: "5Gi" diff --git a/playground/platform/resourcepools/pool-solar.yaml b/playground/platform/resourcepools/pool-solar.yaml index 34db6252..2fef728b 100644 --- a/playground/platform/resourcepools/pool-solar.yaml +++ b/playground/platform/resourcepools/pool-solar.yaml @@ -8,8 +8,8 @@ metadata: spec: quota: hard: - limits.cpu: "2" limits.memory: 2Gi + limits.cpu: "0" requests.cpu: "2" requests.memory: 2Gi requests.storage: "5Gi" diff --git a/playground/platform/tenants/green.yaml b/playground/platform/tenants/green.yaml index 043448f7..8ed63302 100644 --- a/playground/platform/tenants/green.yaml +++ b/playground/platform/tenants/green.yaml @@ -20,6 +20,16 @@ spec: - enforce: action: allow metadata: + - apiGroups: + - "v1" + kinds: + - "Namespace" + labels: + projectcapsule.dev/backup: + managed: "true" + annotations: + storagelocationtype: + managed: "s3" - kinds: - Pod annotations: diff --git a/playground/platform/tenants/wind.yaml b/playground/platform/tenants/wind.yaml index 06793049..31b3ea69 100644 --- a/playground/platform/tenants/wind.yaml +++ b/playground/platform/tenants/wind.yaml @@ -17,16 +17,24 @@ spec: - matchLabels: team: devops rules: - - namespaceSelector: - matchLabels: - company.example/tier: application - quota: + # Enforce Resource-Allocation to best-practices + - quota: - name: billable-compute hard: requests.cpu: "8" requests.memory: 16Gi - limits.cpu: "8" + limits.cpu: "0" limits.memory: 16Gi + requests.storage: "5Gi" + enforce: + action: deny + workloads: + resources: + limits: + cpu: + policy: Remove + memory: + policy: MatchRequest - permissions: bindings: - clusterRoleName: 'view' diff --git a/playground/user/workloads/kustomization.yaml b/playground/user/workloads/kustomization.yaml new file mode 100644 index 00000000..b9031ef0 --- /dev/null +++ b/playground/user/workloads/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - statefulset.yaml diff --git a/playground/user/workloads/statefulset.yaml b/playground/user/workloads/statefulset.yaml new file mode 100644 index 00000000..b28f01b7 --- /dev/null +++ b/playground/user/workloads/statefulset.yaml @@ -0,0 +1,39 @@ +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: web +spec: + selector: + matchLabels: + app: nginx # has to match .spec.template.metadata.labels + serviceName: "nginx" + replicas: 3 # by default is 1 + minReadySeconds: 10 # by default is 0 + template: + metadata: + labels: + app: nginx # has to match .spec.selector.matchLabels + spec: + terminationGracePeriodSeconds: 10 + resources: + requests: + cpu: 100m + containers: + - name: nginx + image: registry.k8s.io/nginx-slim:0.24 + ports: + - containerPort: 80 + name: web + volumeMounts: + - name: www + mountPath: /usr/share/nginx/html + volumeClaimTemplates: + - metadata: + name: www + spec: + accessModes: + - "ReadWriteOnce" + resources: + requests: + storage: 1Gi