From 6a762e789981638aebfca65535628294d1f49541 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20B=C3=A4hler?= <26610571+oliverbaehler@users.noreply.github.com> Date: Fri, 3 Jul 2026 08:21:09 +0200 Subject: [PATCH] feat: security measures and prometheusrules (#1995) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(controller): decode old object for delete requests Signed-off-by: Oliver Bähler * chore: modernize golang Signed-off-by: Oliver Bähler * chore: modernize golang Signed-off-by: Oliver Bähler * chore: modernize golang Signed-off-by: Oliver Bähler * fix: preserve ca-bundles injected from external providers Signed-off-by: Oliver Baehler * feat: add metadata enforcement Signed-off-by: Oliver Baehler * feat: add metadata enforcement Signed-off-by: Oliver Baehler * fix: add resourcepoolclaim validation Signed-off-by: Oliver Baehler * fix: add resourcepoolclaim validation Signed-off-by: Oliver Baehler * fix: add resourcepoolclaim validation Signed-off-by: Oliver Baehler * Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> * fix: add resourcepoolclaim validation Signed-off-by: Oliver Baehler * feat: security measures and prometheusrules Signed-off-by: Oliver Baehler * feat: security measures and prometheusrules Signed-off-by: Oliver Baehler * feat: security measures and prometheusrules Signed-off-by: Oliver Baehler * feat: security measures and prometheusrules Signed-off-by: Oliver Baehler * feat: security measures and prometheusrules Signed-off-by: Oliver Baehler * feat: security measures and prometheusrules Signed-off-by: Oliver Baehler --------- Signed-off-by: Oliver Bähler Signed-off-by: Oliver Baehler Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- Makefile | 4 +- charts/capsule/README.md | 7 + charts/capsule/ci/monitoring-values.yaml | 4 + charts/capsule/templates/prometheusrules.yaml | 45 +++ charts/capsule/values.schema.json | 57 +++- charts/capsule/values.yaml | 275 ++++++++++++++++++ cmd/controller/main.go | 13 +- e2e/replications_globaltenantresource_test.go | 149 ++++++++++ e2e/replications_tenantresource_test.go | 125 ++++++++ .../controllers/cfg/invalidator/manager.go | 15 +- internal/controllers/cfg/status/manager.go | 16 +- .../customquotas/custom_quota_controller.go | 1 + .../global_custom_quota_controller.go | 1 + internal/controllers/customquotas/utils.go | 8 + .../controllers/customquotas/utils_test.go | 51 ++++ internal/controllers/resources/collect.go | 27 +- .../controllers/resources/collect_test.go | 95 ++++++ internal/controllers/resources/global.go | 4 +- internal/controllers/resources/namespaced.go | 1 + internal/controllers/rulestatus/manager.go | 6 +- internal/controllers/tenantowner/manager.go | 18 +- internal/metrics/config_recorder.go | 86 ++++++ internal/metrics/customquota_recorder.go | 24 +- .../metrics/global_customquota_recorder.go | 23 +- .../metrics/global_tenantresource_recorder.go | 1 + internal/metrics/rulestatus_recorder.go | 89 ++++++ internal/metrics/tenantowner_recorder.go | 86 ++++++ internal/metrics/tenantresource_recorder.go | 1 + pkg/template/reference.go | 18 +- 29 files changed, 1206 insertions(+), 44 deletions(-) create mode 100644 charts/capsule/templates/prometheusrules.yaml create mode 100644 internal/controllers/customquotas/utils_test.go create mode 100644 internal/controllers/resources/collect_test.go create mode 100644 internal/metrics/config_recorder.go create mode 100644 internal/metrics/rulestatus_recorder.go create mode 100644 internal/metrics/tenantowner_recorder.go diff --git a/Makefile b/Makefile index 136af5a6..dcb0c859 100644 --- a/Makefile +++ b/Makefile @@ -120,8 +120,8 @@ dev-build: kind dev-destroy: kind $(KIND) delete cluster --name capsule -dev-install-deps: dev-setup-fluxcd dev-setup-cert-manager dev-install-gw-api-crds wait-for-helmreleases -dev-install-deps-openshift: dev-setup-fluxcd-openshift dev-setup-cert-manager dev-install-gw-api-crds wait-for-helmreleases +dev-install-deps: dev-setup-fluxcd dev-setup-cert-manager dev-install-gw-api-crds dev-install-prometheus-crds wait-for-helmreleases +dev-install-deps-openshift: dev-setup-fluxcd-openshift dev-setup-cert-manager dev-install-gw-api-crds dev-install-prometheus-crds wait-for-helmreleases API_GW := none API_GW_VERSION := v1.3.0 diff --git a/charts/capsule/README.md b/charts/capsule/README.md index 750fb473..e2bd7305 100644 --- a/charts/capsule/README.md +++ b/charts/capsule/README.md @@ -182,6 +182,13 @@ The following Values have changed key or Value: | monitoring.diagnostics.operator.folder | string | `""` | folder assignment for dashboard | | monitoring.diagnostics.operator.instanceSelector | object | `{}` | Selects Grafana instances for import | | monitoring.diagnostics.operator.resyncPeriod | string | `"10m"` | How often the resource is synced, defaults to 10m0s if not set | +| monitoring.prometheusRules.annotations | object | `{}` | Assign additional Annotations | +| monitoring.prometheusRules.enabled | bool | `false` | Enable PrometheusRules | +| monitoring.prometheusRules.labels | object | `{}` | Assign additional labels according to Prometheus selector matching labels | +| monitoring.prometheusRules.namespace | string | `""` | Install the PrometheusRules into a different Namespace, as the monitoring stack one (default: the release one) | +| monitoring.prometheusRules.ruleAnnotations | object | `{}` | Annotations add to all Rules | +| monitoring.prometheusRules.ruleLabels | object | `{}` | Labels add to all Rules | +| monitoring.prometheusRules.rules | list | See [values.yaml](values.yaml) | Prometheus Rules definitions. The block is directly forwarded into the PrometheusRule, so you can use whatever specification you want. | | monitoring.serviceMonitor.annotations | object | `{}` | Assign additional Annotations | | monitoring.serviceMonitor.enabled | bool | `false` | Enable ServiceMonitor | | monitoring.serviceMonitor.endpoint.interval | string | `"15s"` | Set the scrape interval for the endpoint of the serviceMonitor | diff --git a/charts/capsule/ci/monitoring-values.yaml b/charts/capsule/ci/monitoring-values.yaml index 8b6c629c..381379e5 100644 --- a/charts/capsule/ci/monitoring-values.yaml +++ b/charts/capsule/ci/monitoring-values.yaml @@ -1,4 +1,8 @@ monitoring: + prometheusRules: + enabled: true + ruleLabels: + team: platform dashboards: enabled: true annotations: diff --git a/charts/capsule/templates/prometheusrules.yaml b/charts/capsule/templates/prometheusrules.yaml new file mode 100644 index 00000000..2ca3da13 --- /dev/null +++ b/charts/capsule/templates/prometheusrules.yaml @@ -0,0 +1,45 @@ +{{- if not $.Values.crds.exclusive }} + {{- with .Values.monitoring.prometheusRules }} + {{- if .enabled }} + {{- $groups := list }} + {{- $ruleLabels := default dict .ruleLabels }} + {{- $ruleAnnotations := default dict .ruleAnnotations }} + {{- range $group := .rules }} + {{- $outGroup := deepCopy $group }} + {{- $outRules := list }} + {{- range $rule := default list $group.rules }} + {{- $outRule := deepCopy $rule }} + {{- $labels := mergeOverwrite (deepCopy $ruleLabels) (default dict $rule.labels) }} + {{- $annotations := mergeOverwrite (deepCopy $ruleAnnotations) (default dict $rule.annotations) }} + {{- if $labels }} + {{- $_ := set $outRule "labels" $labels }} + {{- end }} + {{- if $annotations }} + {{- $_ := set $outRule "annotations" $annotations }} + {{- end }} + {{- $outRules = append $outRules $outRule }} + {{- end }} + {{- $_ := set $outGroup "rules" $outRules }} + {{- $groups = append $groups $outGroup }} + {{- end }} +--- +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ include "capsule.fullname" $ }} + namespace: {{ .namespace | default $.Release.Namespace }} + labels: + {{- include "capsule.labels" $ | nindent 4 }} + {{- with .labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + groups: + {{- toYaml $groups | nindent 4 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/capsule/values.schema.json b/charts/capsule/values.schema.json index c3eeab25..fd2fba8c 100644 --- a/charts/capsule/values.schema.json +++ b/charts/capsule/values.schema.json @@ -178,7 +178,8 @@ } } } - } + }, + "additionalProperties": true }, "priorityClassName": { "description": "Set a pod priorityClassName", @@ -225,7 +226,8 @@ "runAsUser": { "type": "integer" } - } + }, + "additionalProperties": true }, "tolerations": { "description": "Set list of tolerations", @@ -326,7 +328,8 @@ "type": { "type": "string" } - } + }, + "additionalProperties": true } } }, @@ -411,7 +414,8 @@ } } } - } + }, + "additionalProperties": true }, "options": { "type": "object", @@ -619,7 +623,8 @@ } } } - } + }, + "additionalProperties": true }, "resources": { "description": "Set the resource requests/limits for the Capsule manager container", @@ -744,6 +749,42 @@ } } }, + "prometheusRules": { + "type": "object", + "properties": { + "annotations": { + "description": "Assign additional Annotations", + "type": "object" + }, + "enabled": { + "description": "Enable PrometheusRules", + "type": "boolean" + }, + "labels": { + "description": "Assign additional labels according to Prometheus selector matching labels", + "type": "object" + }, + "namespace": { + "description": "Install the PrometheusRules into a different Namespace, as the monitoring stack one (default: the release one)", + "type": "string" + }, + "ruleAnnotations": { + "description": "Annotations add to all Rules", + "type": "object" + }, + "ruleLabels": { + "description": "Labels add to all Rules", + "type": "object" + }, + "rules": { + "description": "Prometheus Rules definitions. The block is directly forwarded into the PrometheusRule, so you can use whatever specification you want.", + "type": "array", + "items": { + "type": "object" + } + } + } + }, "serviceMonitor": { "type": "object", "properties": { @@ -832,7 +873,8 @@ } } } - } + }, + "additionalProperties": true }, "ports": { "description": "Set additional ports for the deployment", @@ -918,7 +960,8 @@ "readOnlyRootFilesystem": { "type": "boolean" } - } + }, + "additionalProperties": true }, "serviceAccount": { "type": "object", diff --git a/charts/capsule/values.yaml b/charts/capsule/values.yaml index 5ef532db..f9cbd9d6 100644 --- a/charts/capsule/values.yaml +++ b/charts/capsule/values.yaml @@ -34,11 +34,15 @@ global: restartPolicy: Never # -- Sets the ttl in seconds after a finished certgen job is deleted. Set to -1 to never delete. ttlSecondsAfterFinished: 60 + # @schema type: object + # @schema additionalProperties: true # -- Security context for the job pods. podSecurityContext: enabled: true seccompProfile: type: "RuntimeDefault" + # @schema type: object + # @schema additionalProperties: true # -- Security context for the job containers. securityContext: enabled: true @@ -126,6 +130,8 @@ manager: # -- Declare ApiVersion used for Flow flowApiVersion: "flowcontrol.apiserver.k8s.io/v1" + # @schema type: object + # @schema additionalProperties: true # -- Priority level configuration. # The block is directly forwarded into the priorityLevelConfiguration, so you can use whatever specification you want. # ref: https://kubernetes.io/docs/concepts/cluster-administration/flow-control/#prioritylevelconfiguration @@ -304,12 +310,16 @@ manager: # -- Additional Environment Variables env: [] + # @schema type: object + # @schema additionalProperties: true # -- Configure the liveness probe using Deployment probe spec livenessProbe: httpGet: path: /healthz port: 10080 + # @schema type: object + # @schema additionalProperties: true # -- Configure the readiness probe using Deployment probe spec readinessProbe: httpGet: @@ -343,6 +353,8 @@ podAnnotations: {} # -- Set the priority class name of the Capsule pod priorityClassName: '' # system-cluster-critical +# @schema type: object +# @schema additionalProperties: true # -- Set the securityContext for the Capsule pod podSecurityContext: enabled: true @@ -352,6 +364,8 @@ podSecurityContext: runAsNonRoot: true runAsUser: 1002 +# @schema type: object +# @schema additionalProperties: true # -- Set the securityContext for the Capsule container securityContext: enabled: true @@ -492,6 +506,267 @@ monitoring: # -- Set relabelings for the endpoint of the serviceMonitor relabelings: [] + # PrometheusRules + prometheusRules: + # -- Enable PrometheusRules + enabled: false + # -- Install the PrometheusRules into a different Namespace, as the monitoring stack one (default: the release one) + namespace: '' + # -- Assign additional labels according to Prometheus selector matching labels + labels: {} + # -- Assign additional Annotations + annotations: {} + # -- Labels add to all Rules + ruleLabels: {} + # -- Annotations add to all Rules + ruleAnnotations: {} + # -- Prometheus Rules definitions. The block is directly forwarded into the PrometheusRule, so you can use whatever specification you want. + # @default -- See [values.yaml](values.yaml) + rules: # @schema itemProperties:{} + - name: capsule.customquotas + rules: + - alert: CapsuleGlobalCustomQuotaNotReady + expr: | + capsule_global_custom_quota_condition{condition="Ready"} != 1 + for: 10m + labels: + severity: warning + app: capsule + component: customquotas + annotations: + summary: Capsule GlobalCustomQuota {{ $labels.custom_quota }} is not ready + description: "GlobalCustomQuota {{ $labels.custom_quota }} is not in Ready state for the last 10 minutes." + - alert: CapsuleGlobalCustomQuotaHighUsageWarning + expr: | + capsule_global_custom_quota_resource_usage_percentage > 90 + for: 10m + labels: + severity: warning + app: capsule + component: customquotas + annotations: + summary: "High resource usage in GlobalCustomQuota {{ $labels.custom_quota }}" + description: "Usage for GlobalCustomQuota {{ $labels.custom_quota }} is at {{ $value }}% usage for the last 10 minutes." + - alert: CapsuleGlobalCustomQuotaHighUsageCritical + expr: | + capsule_global_custom_quota_resource_usage_percentage > 95 + for: 10m + labels: + severity: critical + app: capsule + component: customquotas + annotations: + summary: "Critical resource usage in GlobalCustomQuota {{ $labels.custom_quota }}" + description: "Usage for GlobalCustomQuota {{ $labels.custom_quota }} has exceeded 95% usage for the last 10 minutes." + - alert: CapsuleCustomQuotaNotReady + expr: | + capsule_custom_quota_condition{condition="Ready"} != 1 + for: 10m + labels: + severity: warning + app: capsule + component: customquotas + annotations: + summary: "Capsule CustomQuota {{ $labels.target_namespace }}/{{ $labels.custom_quota }} is not ready" + description: "CustomQuota {{ $labels.target_namespace }}/{{ $labels.custom_quota }} is not in Ready state for the last 10 minutes." + - alert: CapsuleCustomQuotaHighUsageWarning + expr: | + capsule_custom_quota_resource_usage_percentage > 90 + for: 10m + labels: + severity: warning + app: capsule + component: customquotas + annotations: + summary: "High resource usage in CustomQuota {{ $labels.target_namespace }}/{{ $labels.custom_quota }}" + description: "Usage for CustomQuota {{ $labels.target_namespace }}/{{ $labels.custom_quota }} is at {{ $value }}% usage for the last 10 minutes." + - alert: CapsuleCustomQuotaHighUsageCritical + expr: | + capsule_custom_quota_resource_usage_percentage > 95 + for: 10m + labels: + severity: critical + app: capsule + component: customquotas + annotations: + summary: "Critical resource usage in CustomQuota {{ $labels.target_namespace }}/{{ $labels.custom_quota }}" + description: "Usage for CustomQuota {{ $labels.target_namespace }}/{{ $labels.custom_quota }} has exceeded 95% usage for the last 10 minutes." + + - name: capsule.resourcepools + rules: + - alert: CapsuleResourcePoolNotReady + expr: | + capsule_pool_condition{condition="Ready"} != 1 + for: 10m + labels: + severity: warning + app: capsule + component: resourcepools + annotations: + summary: "Capsule ResourcePool {{ $labels.pool }} is not ready" + description: "ResourcePool {{ $labels.pool }} is not in Ready state for the last 10 minutes." + - alert: CapsuleResourcePoolClaimNotReady + expr: | + capsule_claim_condition{condition="Ready"} != 1 + for: 10m + labels: + severity: warning + app: capsule + component: resourcepools + annotations: + summary: "Capsule ResourcePoolClaim {{ $labels.target_namespace }}/{{ $labels.name }} is not ready" + description: "ResourcePoolClaim {{ $labels.target_namespace }}/{{ $labels.name }} is not in Ready state for the last 10 minutes." + - alert: CapsulePoolHighUsageWarning + expr: | + capsule_pool_usage_percentage > 90 + for: 10m + labels: + severity: warning + app: capsule + component: resourcepools + annotations: + summary: "High resource usage in Resourcepool {{ $labels.pool }}" + description: "Resource {{ $labels.resource }} in pool {{ $labels.pool }} is at {{ $value }}% usage for the last 10 minutes." + - alert: CapsulePoolHighUsageCritical + expr: | + capsule_pool_usage_percentage > 95 + for: 10m + labels: + severity: critical + app: capsule + component: resourcepools + annotations: + summary: "Critical resource usage in Resourcepool {{ $labels.pool }}" + description: "Resource {{ $labels.resource }} in pool {{ $labels.pool }} has exceeded 95% usage for the last 10 minutes." + + - name: capsule.replications + rules: + - alert: CapsuleGlobalTenantResourceNotReady + expr: | + capsule_global_resource_condition{condition="Ready"} != 1 + for: 10m + labels: + severity: warning + app: capsule + component: replications + annotations: + summary: Capsule GlobalTenantResource {{ $labels.name }} is not ready + description: "GlobalTenantResource {{ $labels.name }} is not in Ready state for the last 10 minutes." + - alert: CapsuleTenantResourceNotReady + expr: | + capsule_resource_condition{condition="Ready"} != 1 + for: 10m + labels: + severity: warning + app: capsule + component: replications + annotations: + summary: Capsule TenantResource {{ $labels.target_namespace }}/{{ $labels.name }} is not ready + description: "TenantResource {{ $labels.target_namespace }}/{{ $labels.name }} is not in Ready state for the last 10 minutes." + + - name: capsule.tenants + rules: + - alert: CapsuleTenantNamespaceQuotaExceeded + expr: | + capsule_tenant_namespaces_current >= capsule_tenant_namespaces_max + for: "5m" + labels: + severity: warning + app: capsule + component: tenants + annotations: + summary: "Capsule Tenant {{ $labels.tenant }} has reached namespace quota" + description: "Tenant {{ $labels.tenant }} has {{ $value }} namespaces and has reached or exceeded its maximum quota of {{ $labels.max_namespaces }}." + - alert: CapsuleTenantNotReady + expr: | + capsule_tenant_condition{condition="Ready"} != 1 + for: "5m" + labels: + severity: critical + app: capsule + component: tenants + annotations: + summary: "Capsule Tenant {{ $labels.tenant }} is missing or not active" + description: "Tenant {{ $labels.tenant }} is either missing or not in Active state. This may prevent users from accessing their namespaces." + - alert: CapsuleTenantNamespaceNotReady + expr: | + capsule_tenant_namespace_condition{condition="Ready"} != 1 + for: "5m" + labels: + severity: critical + app: capsule + component: tenants + annotations: + summary: "Namespace {{ $labels.target_namespace }} from Tenant {{ $labels.tenant }} is not Ready." + description: "Namespace {{ $labels.target_namespace }} from Tenant {{ $labels.tenant }} is not Ready state for the last 10 minutes." + + - name: capsule.rulestatus + rules: + - alert: CapsuleRuleStatusNotReady + expr: | + capsule_rulestatus_condition{condition="Ready"} != 1 + for: 10m + labels: + severity: warning + app: capsule + component: rulestatus + annotations: + summary: Capsule RuleStatus {{ $labels.target_namespace }}/{{ $labels.name }} is not ready + description: "RuleStatus {{ $labels.target_namespace }}/{{ $labels.name }} is not in Ready state for the last 10 minutes." + + - name: capsule.tenantowners + rules: + - alert: CapsuleTenantOwnerNotReady + expr: | + capsule_tenantowner_condition{condition="Ready"} != 1 + for: 10m + labels: + severity: warning + app: capsule + component: tenantowners + annotations: + summary: Capsule TenantOwner {{ $labels.name }} is not ready + description: "TenantOwner {{ $labels.name }} is not in Ready state for the last 10 minutes." + + - name: capsule.config + rules: + - alert: CapsuleConfigurationNotReady + expr: | + capsule_config_condition{condition="Ready"} != 1 + for: 10m + labels: + severity: warning + app: capsule + component: config + annotations: + summary: "CapsuleConfiguration {{ $labels.name }} is not ready" + description: "CapsuleConfiguration {{ $labels.name }} is not in Ready state for the last 10 minutes." + + - name: capsule.controller + rules: + # Alert when Capsule controller is down + - alert: CapsuleControllerDown + expr: | + up{job="capsule-controller-manager"} == 0 + for: "5m" + labels: + severity: critical + app: capsule + component: controller + annotations: + summary: "Capsule controller is down" + description: "The Capsule controller manager has been down for more than 5 minutes. Tenant operations may be impacted." + - alert: CapsuleWebhookHighErrorRate + expr: | + rate(capsule_webhook_requests_total{status=~"5.."}[5m]) / rate(capsule_webhook_requests_total[5m]) > 0.05 + for: "5m" + labels: + severity: warning + app: capsule + component: controller + annotations: + summary: "Capsule webhook experiencing high error rate" + description: "Capsule webhook has an error rate of {{ $value | humanizePercentage }} over the last 5 minutes. This may block tenant operations." # Conversions Webhook configurations conversions: diff --git a/cmd/controller/main.go b/cmd/controller/main.go index 3e010621..3dd67bf8 100644 --- a/cmd/controller/main.go +++ b/cmd/controller/main.go @@ -728,7 +728,7 @@ func main() { if err = (&tenantownercontroller.TenantOwnerManager{ Log: ctrl.Log.WithName("capsule.ctrl").WithName("tenantowners"), Client: manager.GetClient(), - }).SetupWithManager(manager, controllerConfig); err != nil { + }).SetupWithManager(manager, controllerConfig, metrics.MustMakeTenantOwnerRecorder()); err != nil { setupLog.Error(err, "unable to create controller", "controller", "TenantOwners") os.Exit(1) } @@ -759,18 +759,21 @@ func main() { os.Exit(1) } + configrecorder := metrics.MustMakeConfigRecorder() + if err = (&configcontroller.Manager{ Rest: manager.GetConfig(), Client: manager.GetClient(), Log: ctrl.Log.WithName("capsule.ctrl").WithName("configuration"), - }).SetupWithManager(manager, controllerConfig); err != nil { + }).SetupWithManager(manager, controllerConfig, configrecorder); err != nil { setupLog.Error(err, "unable to create controller", "controller", "CapsuleConfiguration") os.Exit(1) } if err = (&rulestatuscontroller.Manager{ - Client: manager.GetClient(), - Log: ctrl.Log.WithName("capsule.ctrl").WithName("ruleset"), + Client: manager.GetClient(), + Log: ctrl.Log.WithName("capsule.ctrl").WithName("ruleset"), + Metrics: metrics.MustMakeRuleStatusRecorder(), }).SetupWithManager(manager, controllerConfig); err != nil { setupLog.Error(err, "unable to create controller", "controller", "RuleSet") os.Exit(1) @@ -789,7 +792,7 @@ func main() { RegexCache: regexCache, } - if err := localInvalidator.SetupWithManager(manager, controllerConfig); err != nil { + if err := localInvalidator.SetupWithManager(manager, controllerConfig, configrecorder); err != nil { setupLog.Error(err, "unable to create controller", "controller", "invalidator") os.Exit(1) } diff --git a/e2e/replications_globaltenantresource_test.go b/e2e/replications_globaltenantresource_test.go index 1755d1cf..6e33f5cd 100644 --- a/e2e/replications_globaltenantresource_test.go +++ b/e2e/replications_globaltenantresource_test.go @@ -154,6 +154,22 @@ var _ = Describe("GlobalTenantResource", Ordered, Label("replications", "global" return nil }, "30s", "5s").Should(Succeed()) + Eventually(func() error { + clusterRoleList := &rbacv1.ClusterRoleList{} + labelSelector := client.MatchingLabels{"e2e.capsule.dev/test-suite": "true"} + if err := k8sClient.List(context.TODO(), clusterRoleList, labelSelector); err != nil { + return err + } + + for _, clusterRole := range clusterRoleList.Items { + if err := k8sClient.Delete(context.TODO(), &clusterRole); err != nil { + return err + } + } + + return nil + }, "30s", "5s").Should(Succeed()) + Eventually(func() error { cfg := &capsulev1beta2.CapsuleConfiguration{} if err := k8sClient.Get(ctx, client.ObjectKey{Name: originConfig.Name}, cfg); err != nil { @@ -165,6 +181,36 @@ var _ = Describe("GlobalTenantResource", Ordered, Label("replications", "global" }) + Context("cluster-scoped objects", func() { + It("applies raw cluster-scoped items", func() { + gtr := newRawClusterRoleGlobalTenantResource("gtr-cluster-raw", "gtr-cluster-raw-role") + + EventuallyCreation(func() error { return k8sClient.Create(ctx, gtr) }).Should(Succeed()) + + expectGlobalTenantResourceReady("gtr-cluster-raw") + expectClusterRoleRules("gtr-cluster-raw-role", []rbacv1.PolicyRule{{ + APIGroups: []string{""}, + Resources: []string{"configmaps"}, + Verbs: []string{"get", "list"}, + }}) + expectGlobalTenantResourceProcessedClusterRole("gtr-cluster-raw", "gtr-cluster-raw-role") + }) + + It("applies generated cluster-scoped items", func() { + gtr := newGeneratedClusterRoleGlobalTenantResource("gtr-cluster-generator", "gtr-cluster-generator-role") + + EventuallyCreation(func() error { return k8sClient.Create(ctx, gtr) }).Should(Succeed()) + + expectGlobalTenantResourceReady("gtr-cluster-generator") + expectClusterRoleRules("gtr-cluster-generator-role", []rbacv1.PolicyRule{{ + APIGroups: []string{""}, + Resources: []string{"secrets"}, + Verbs: []string{"get"}, + }}) + expectGlobalTenantResourceProcessedClusterRole("gtr-cluster-generator", "gtr-cluster-generator-role") + }) + }) + It("skips applying resources to terminating namespaces and removes them from processedItems", func() { terminatingNamespace := tenantANamespaces[2] releaseNamespace := holdNamespaceTerminating(ctx, terminatingNamespace) @@ -1324,6 +1370,85 @@ func newRawConfigMapGlobalTenantResourceWithScope(name string, scope api.Resourc return gtr } +func newRawClusterRoleGlobalTenantResource(name, clusterRoleName string) *capsulev1beta2.GlobalTenantResource { + return &capsulev1beta2.GlobalTenantResource{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "true", + }, + }, + Spec: capsulev1beta2.GlobalTenantResourceSpec{ + Scope: api.ResourceScopeNone, + TenantResourceCommonSpec: capsulev1beta2.TenantResourceCommonSpec{ + ResyncPeriod: metav1.Duration{Duration: 5 * time.Second}, + PruningOnDelete: ptr.To(true), + Resources: []capsulev1beta2.ResourceSpec{{ + RawItems: []capsulev1beta2.RawExtension{{ + RawExtension: runtime.RawExtension{ + Object: &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "rbac.authorization.k8s.io/v1", + Kind: "ClusterRole", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: clusterRoleName, + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "true", + }, + }, + Rules: []rbacv1.PolicyRule{{ + APIGroups: []string{""}, + Resources: []string{"configmaps"}, + Verbs: []string{"get", "list"}, + }}, + }, + }, + }}, + }}, + }, + }, + } +} + +func newGeneratedClusterRoleGlobalTenantResource(name, clusterRoleName string) *capsulev1beta2.GlobalTenantResource { + return &capsulev1beta2.GlobalTenantResource{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "true", + }, + }, + Spec: capsulev1beta2.GlobalTenantResourceSpec{ + Scope: api.ResourceScopeNone, + TenantResourceCommonSpec: capsulev1beta2.TenantResourceCommonSpec{ + ResyncPeriod: metav1.Duration{Duration: 5 * time.Second}, + PruningOnDelete: ptr.To(true), + Resources: []capsulev1beta2.ResourceSpec{{ + Generators: []capsulev1beta2.TemplateItemSpec{{ + MissingKey: "error", + Template: fmt.Sprintf(`--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: %s + labels: + e2e.capsule.dev/test-suite: "true" +rules: + - apiGroups: + - "" + resources: + - secrets + verbs: + - get +`, clusterRoleName), + }}, + }}, + }, + }, + } +} + func expectGlobalTenantResourceReady(name string) { Eventually(func(g Gomega) { current := &capsulev1beta2.GlobalTenantResource{} @@ -1346,3 +1471,27 @@ func expectGlobalTenantResourceFailed(name, msgContains string) { g.Expect(rdy.Message).To(ContainSubstring(msgContains)) }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) } + +func expectClusterRoleRules(name string, expected []rbacv1.PolicyRule) { + Eventually(func(g Gomega) { + clusterRole := &rbacv1.ClusterRole{} + g.Expect(k8sClient.Get(context.Background(), types.NamespacedName{Name: name}, clusterRole)).To(Succeed()) + g.Expect(clusterRole.Namespace).To(BeEmpty()) + g.Expect(clusterRole.Labels).To(HaveKeyWithValue("e2e.capsule.dev/test-suite", "true")) + g.Expect(clusterRole.Labels).To(HaveKeyWithValue(managedByLabel, meta.ValueControllerReplications)) + g.Expect(clusterRole.Rules).To(ConsistOf(expected)) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) +} + +func expectGlobalTenantResourceProcessedClusterRole(gtrName, clusterRoleName string) { + Eventually(func(g Gomega) { + current := &capsulev1beta2.GlobalTenantResource{} + g.Expect(k8sClient.Get(context.Background(), types.NamespacedName{Name: gtrName}, current)).To(Succeed()) + g.Expect(current.Status.ProcessedItems).To(ContainElement(SatisfyAll( + HaveField("Kind", "ClusterRole"), + HaveField("Name", clusterRoleName), + HaveField("Namespace", ""), + HaveField("Status", metav1.ConditionTrue), + ))) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) +} diff --git a/e2e/replications_tenantresource_test.go b/e2e/replications_tenantresource_test.go index 39ebfedd..80b7d807 100644 --- a/e2e/replications_tenantresource_test.go +++ b/e2e/replications_tenantresource_test.go @@ -135,6 +135,125 @@ var _ = Describe("TenantResource SSA", Ordered, Label("replications", "namespace }) }) + Context("cluster-scoped object protection", func() { + It("rejects cluster-scoped rawItems", func() { + clusterRoleName := "tr-raw-cluster-scoped" + defer func() { + ignoreNotFound(k8sClient.Delete(ctx, &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: clusterRoleName}})) + }() + + tr := &capsulev1beta2.TenantResource{ + ObjectMeta: metav1.ObjectMeta{ + Name: "rawitems-cluster-scoped", + Namespace: baseNamespace, + }, + Spec: capsulev1beta2.TenantResourceSpec{ + TenantResourceCommonSpec: capsulev1beta2.TenantResourceCommonSpec{ + PruningOnDelete: ptr.To(true), + ResyncPeriod: resyncPeriod, + Resources: []capsulev1beta2.ResourceSpec{{ + RawItems: []capsulev1beta2.RawExtension{{ + RawExtension: runtime.RawExtension{ + Object: &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{ + APIVersion: rbacv1.SchemeGroupVersion.String(), + Kind: "ClusterRole", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: clusterRoleName, + }, + Rules: []rbacv1.PolicyRule{{ + APIGroups: []string{""}, + Resources: []string{"configmaps"}, + Verbs: []string{"get"}, + }}, + }, + }, + }}, + }}, + }, + }, + } + + EventuallyCreation(func() error { return k8sClient.Create(ctx, tr) }).Should(Succeed()) + expectTenantResourceFailed(baseNamespace, tr.Name, "cluster-scoped kind rbac.authorization.k8s.io/v1/ClusterRole is not allowed") + expectClusterRoleAbsent(clusterRoleName) + }) + + It("rejects cluster-scoped generator output", func() { + clusterRoleName := "tr-generator-cluster-scoped" + defer func() { + ignoreNotFound(k8sClient.Delete(ctx, &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: clusterRoleName}})) + }() + + tr := newGeneratorConfigMapTenantResource(baseNamespace, "generator-cluster-scoped", fmt.Sprintf(`--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: %s +rules: +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["get"] +`, clusterRoleName)) + + EventuallyCreation(func() error { return k8sClient.Create(ctx, tr) }).Should(Succeed()) + expectTenantResourceFailed(baseNamespace, tr.Name, "cluster-scoped kind rbac.authorization.k8s.io/v1/ClusterRole is not allowed") + expectClusterRoleAbsent(clusterRoleName) + }) + + It("rejects cluster-scoped namespacedItems", func() { + clusterRoleName := "tr-source-cluster-scoped" + source := &rbacv1.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterRoleName, + Labels: map[string]string{ + "replicate": "cluster-scoped", + }, + }, + Rules: []rbacv1.PolicyRule{{ + APIGroups: []string{""}, + Resources: []string{"configmaps"}, + Verbs: []string{"get"}, + }}, + } + + EventuallyCreation(func() error { return k8sClient.Create(ctx, source) }).Should(Succeed()) + defer func() { + ignoreNotFound(k8sClient.Delete(ctx, source)) + }() + + tr := &capsulev1beta2.TenantResource{ + ObjectMeta: metav1.ObjectMeta{ + Name: "namespaceditems-cluster-scoped", + Namespace: baseNamespace, + }, + Spec: capsulev1beta2.TenantResourceSpec{ + TenantResourceCommonSpec: capsulev1beta2.TenantResourceCommonSpec{ + PruningOnDelete: ptr.To(true), + ResyncPeriod: resyncPeriod, + Resources: []capsulev1beta2.ResourceSpec{{ + NamespacedItems: []template.ResourceReference{{ + VersionKind: capruntime.VersionKind{ + APIVersion: rbacv1.SchemeGroupVersion.String(), + Kind: "ClusterRole", + }, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "replicate": "cluster-scoped", + }, + }, + }}, + }}, + }, + }, + } + + EventuallyCreation(func() error { return k8sClient.Create(ctx, tr) }).Should(Succeed()) + expectTenantResourceFailed(baseNamespace, tr.Name, "cluster-scoped kind rbac.authorization.k8s.io/v1/ClusterRole is not allowed") + }) + }) + It("skips applying resources to terminating namespaces and removes them from processedItems", func() { terminatingNamespace := targetNamespaces[2] releaseNamespace := holdNamespaceTerminating(ctx, terminatingNamespace) @@ -1839,6 +1958,12 @@ func expectSecretAbsent(namespace, name string) { }, 5*time.Second, defaultPollInterval).Should(HaveOccurred()) } +func expectClusterRoleAbsent(name string) { + Consistently(func() error { + return k8sClient.Get(context.Background(), types.NamespacedName{Name: name}, &rbacv1.ClusterRole{}) + }, 5*time.Second, defaultPollInterval).Should(HaveOccurred()) +} + func renameFirstTenantResourceRawConfigMap(tr *capsulev1beta2.TenantResource, name string) { tr.Spec.Resources[0].RawItems[0] = capsulev1beta2.RawExtension{ RawExtension: runtime.RawExtension{ diff --git a/internal/controllers/cfg/invalidator/manager.go b/internal/controllers/cfg/invalidator/manager.go index 219c5cc6..4a01a389 100644 --- a/internal/controllers/cfg/invalidator/manager.go +++ b/internal/controllers/cfg/invalidator/manager.go @@ -24,6 +24,7 @@ import ( capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" "github.com/projectcapsule/capsule/internal/cache" "github.com/projectcapsule/capsule/internal/controllers/utils" + "github.com/projectcapsule/capsule/internal/metrics" "github.com/projectcapsule/capsule/pkg/runtime/configuration" "github.com/projectcapsule/capsule/pkg/runtime/predicates" ) @@ -31,12 +32,13 @@ import ( type CacheInvalidator struct { client.Client - reader client.Reader - Rest *rest.Config Log logr.Logger - configName string + reader client.Reader + configName string + metrics *metrics.ConfigRecorder + Configuration configuration.Configuration RegistryCache *cache.RegistryRuleSetCache @@ -63,9 +65,14 @@ func (r *CacheInvalidator) Start(ctx context.Context) error { return nil } -func (r *CacheInvalidator) SetupWithManager(mgr ctrl.Manager, ctrlConfig utils.ControllerOptions) (err error) { +func (r *CacheInvalidator) SetupWithManager( + mgr ctrl.Manager, + ctrlConfig utils.ControllerOptions, + metrics *metrics.ConfigRecorder, +) (err error) { r.configName = ctrlConfig.ConfigurationName r.reader = mgr.GetAPIReader() + r.metrics = metrics err = ctrl.NewControllerManagedBy(mgr). Named("config/caches"). diff --git a/internal/controllers/cfg/status/manager.go b/internal/controllers/cfg/status/manager.go index fecdd778..4fd0a2df 100644 --- a/internal/controllers/cfg/status/manager.go +++ b/internal/controllers/cfg/status/manager.go @@ -27,6 +27,7 @@ import ( capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" "github.com/projectcapsule/capsule/internal/controllers/utils" + "github.com/projectcapsule/capsule/internal/metrics" capmeta "github.com/projectcapsule/capsule/pkg/api/meta" "github.com/projectcapsule/capsule/pkg/api/rbac" "github.com/projectcapsule/capsule/pkg/runtime/configuration" @@ -42,17 +43,22 @@ const tenantEventMarker = "tenant-event" type Manager struct { client.Client - reader client.Reader - Rest *rest.Config + reader client.Reader configName string Log logr.Logger + metrics *metrics.ConfigRecorder } -func (r *Manager) SetupWithManager(mgr ctrl.Manager, ctrlConfig utils.ControllerOptions) (err error) { +func (r *Manager) SetupWithManager( + mgr ctrl.Manager, + ctrlConfig utils.ControllerOptions, + metrics *metrics.ConfigRecorder, +) (err error) { r.configName = ctrlConfig.ConfigurationName r.reader = mgr.GetAPIReader() + r.metrics = metrics return ctrl.NewControllerManagedBy(mgr). Named("capsule/configuration"). @@ -165,6 +171,8 @@ func (r *Manager) Reconcile(ctx context.Context, request reconcile.Request) (res if apierrors.IsNotFound(err) { log.V(5).Info("requested object not found, could have been deleted after reconcile request") + r.metrics.DeleteMetrics(request.Name) + return reconcile.Result{}, nil } @@ -179,6 +187,8 @@ func (r *Manager) Reconcile(ctx context.Context, request reconcile.Request) (res return } + + r.metrics.RecordConditions(instance) }() // Validating the Capsule Configuration options. diff --git a/internal/controllers/customquotas/custom_quota_controller.go b/internal/controllers/customquotas/custom_quota_controller.go index 201e552e..a0bfec91 100644 --- a/internal/controllers/customquotas/custom_quota_controller.go +++ b/internal/controllers/customquotas/custom_quota_controller.go @@ -256,6 +256,7 @@ func (r *customQuotaClaimController) emitMetrics( // Usage Metrics r.metrics.ResourceUsageGauge.WithLabelValues(instance.GetName(), instance.GetNamespace()).Set(float64(instance.Status.Usage.Used.MilliValue()) / 1000) + r.metrics.ResourceUsagePercentageGauge.WithLabelValues(instance.GetName(), instance.GetNamespace()).Set(usagePercentage(instance.Status.Usage.Used, instance.Spec.Limit)) r.metrics.ResourceAvailableGauge.WithLabelValues(instance.GetName(), instance.GetNamespace()).Set(float64(instance.Status.Usage.Available.MilliValue()) / 1000) r.metrics.ResourceLimitGauge.WithLabelValues(instance.GetName(), instance.GetNamespace()).Set(float64(instance.Spec.Limit.MilliValue()) / 1000) diff --git a/internal/controllers/customquotas/global_custom_quota_controller.go b/internal/controllers/customquotas/global_custom_quota_controller.go index e4c3fbf4..cff125e4 100644 --- a/internal/controllers/customquotas/global_custom_quota_controller.go +++ b/internal/controllers/customquotas/global_custom_quota_controller.go @@ -349,6 +349,7 @@ func (r *clusterCustomQuotaClaimController) emitMetrics( // Usage Metrics r.metrics.ResourceUsageGauge.WithLabelValues(instance.GetName()).Set(float64(instance.Status.Usage.Used.MilliValue()) / 1000) + r.metrics.ResourceUsagePercentageGauge.WithLabelValues(instance.GetName()).Set(usagePercentage(instance.Status.Usage.Used, instance.Spec.Limit)) r.metrics.ResourceAvailableGauge.WithLabelValues(instance.GetName()).Set(float64(instance.Status.Usage.Available.MilliValue()) / 1000) r.metrics.ResourceLimitGauge.WithLabelValues(instance.GetName()).Set(float64(instance.Spec.Limit.MilliValue()) / 1000) diff --git a/internal/controllers/customquotas/utils.go b/internal/controllers/customquotas/utils.go index 5cf85e38..ba80da9f 100644 --- a/internal/controllers/customquotas/utils.go +++ b/internal/controllers/customquotas/utils.go @@ -33,6 +33,14 @@ import ( const immediatePendingDeleteRequeue = 500 * time.Millisecond +func usagePercentage(used, limit resource.Quantity) float64 { + if limit.MilliValue() <= 0 { + return 0 + } + + return (float64(used.MilliValue()) / float64(limit.MilliValue())) * 100 +} + type GroupedTarget struct { GVK schema.GroupVersionKind Targets []capsulev1beta2.CustomQuotaStatusTarget diff --git a/internal/controllers/customquotas/utils_test.go b/internal/controllers/customquotas/utils_test.go new file mode 100644 index 00000000..34d63b8b --- /dev/null +++ b/internal/controllers/customquotas/utils_test.go @@ -0,0 +1,51 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package customquotas + +import ( + "testing" + + "k8s.io/apimachinery/pkg/api/resource" +) + +func TestUsagePercentage(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + used string + limit string + want float64 + }{ + { + name: "returns zero for zero limit", + used: "1", + limit: "0", + want: 0, + }, + { + name: "calculates whole quantity percentage", + used: "2", + limit: "8", + want: 25, + }, + { + name: "calculates milli quantity percentage", + used: "250m", + limit: "1", + want: 25, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got := usagePercentage(resource.MustParse(tt.used), resource.MustParse(tt.limit)) + if got != tt.want { + t.Fatalf("expected %v, got %v", tt.want, got) + } + }) + } +} diff --git a/internal/controllers/resources/collect.go b/internal/controllers/resources/collect.go index 698df9bc..55198e9d 100644 --- a/internal/controllers/resources/collect.go +++ b/internal/controllers/resources/collect.go @@ -41,6 +41,7 @@ type Collector struct { } type CollectorOptions struct { + AllowClusterScopedObjects bool AllowCrossNamespaceSelection bool Accumulator processor.Accumulator Iterator CollectorIteratorOptions @@ -216,6 +217,10 @@ func (co *Collector) AddToAccumulation( return err } + if err := co.validateClusterScopedObjectAllowed(opts, obj); err != nil { + return err + } + tntName := "" if tnt != nil { tntName = tnt.GetName() @@ -305,7 +310,7 @@ func (co *Collector) CollectNamespacedItems( } for _, item := range spec.NamespacedItems { - p, err := item.LoadResources(ctx, c, co.mapper, namespace, []labels.Selector{selector}, opts.Iterator.FastContext, false, opts.ValidatorNamespaces) + p, err := item.LoadResources(ctx, c, co.mapper, namespace, []labels.Selector{selector}, opts.Iterator.FastContext, opts.AllowClusterScopedObjects, opts.ValidatorNamespaces) if err != nil { totalError = errors.Join(totalError, err) @@ -378,6 +383,26 @@ func GatherAdditionalMetadata( return labels, annotations } +func (co *Collector) validateClusterScopedObjectAllowed( + opts CollectorOptions, + obj *unstructured.Unstructured, +) error { + if opts.AllowClusterScopedObjects { + return nil + } + + isNamespaced, err := tpl.IsNamespacedGVK(co.mapper, obj.GetAPIVersion(), obj.GetKind()) + if err != nil { + return err + } + + if !isNamespaced { + return fmt.Errorf("cluster-scoped kind %s/%s is not allowed", obj.GetAPIVersion(), obj.GetKind()) + } + + return nil +} + // Handles a single generator item. func (co *Collector) handleGeneratorItem( ctx context.Context, diff --git a/internal/controllers/resources/collect_test.go b/internal/controllers/resources/collect_test.go new file mode 100644 index 00000000..2fbc1b07 --- /dev/null +++ b/internal/controllers/resources/collect_test.go @@ -0,0 +1,95 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package resources + +import ( + "strings" + "testing" + + k8smeta "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/pkg/api/processor" +) + +func TestCollectorAddToAccumulationClusterScopedObjects(t *testing.T) { + t.Parallel() + + mapper := k8smeta.NewDefaultRESTMapper([]schema.GroupVersion{{Version: "v1"}}) + mapper.Add(schema.GroupVersionKind{Version: "v1", Kind: "ConfigMap"}, k8smeta.RESTScopeNamespace) + mapper.Add(schema.GroupVersionKind{Version: "v1", Kind: "Namespace"}, k8smeta.RESTScopeRoot) + + collector := NewCollector(nil, mapper) + + t.Run("allows namespaced object", func(t *testing.T) { + t.Parallel() + + acc := processor.Accumulator{} + obj := newUnstructured("v1", "ConfigMap", "default", "example") + + if err := collector.AddToAccumulation(nil, nil, CollectorOptions{Accumulator: acc}, capsuleResourceSpec(), obj, "test", true); err != nil { + t.Fatalf("expected no error, got %v", err) + } + + if len(acc) != 1 { + t.Fatalf("expected object to be accumulated, got %d items", len(acc)) + } + }) + + t.Run("rejects cluster scoped object by default", func(t *testing.T) { + t.Parallel() + + acc := processor.Accumulator{} + obj := newUnstructured("v1", "Namespace", "", "example") + + err := collector.AddToAccumulation(nil, nil, CollectorOptions{Accumulator: acc}, capsuleResourceSpec(), obj, "test", true) + if err == nil { + t.Fatal("expected error, got nil") + } + + if !strings.Contains(err.Error(), "cluster-scoped kind v1/Namespace is not allowed") { + t.Fatalf("expected cluster scoped error, got %v", err) + } + + if len(acc) != 0 { + t.Fatalf("expected object not to be accumulated, got %d items", len(acc)) + } + }) + + t.Run("allows cluster scoped object when configured", func(t *testing.T) { + t.Parallel() + + acc := processor.Accumulator{} + obj := newUnstructured("v1", "Namespace", "", "example") + + opts := CollectorOptions{ + Accumulator: acc, + AllowClusterScopedObjects: true, + } + + if err := collector.AddToAccumulation(nil, nil, opts, capsuleResourceSpec(), obj, "test", true); err != nil { + t.Fatalf("expected no error, got %v", err) + } + + if len(acc) != 1 { + t.Fatalf("expected object to be accumulated, got %d items", len(acc)) + } + }) +} + +func newUnstructured(apiVersion, kind, namespace, name string) *unstructured.Unstructured { + obj := &unstructured.Unstructured{} + obj.SetAPIVersion(apiVersion) + obj.SetKind(kind) + obj.SetNamespace(namespace) + obj.SetName(name) + + return obj +} + +func capsuleResourceSpec() capsulev1beta2.ResourceSpec { + return capsulev1beta2.ResourceSpec{} +} diff --git a/internal/controllers/resources/global.go b/internal/controllers/resources/global.go index dd407c69..af689857 100644 --- a/internal/controllers/resources/global.go +++ b/internal/controllers/resources/global.go @@ -60,7 +60,8 @@ func (r *globalResourceController) SetupWithManager(mgr ctrl.Manager, ctrlConfig Configuration: r.configuration, GatherClient: mgr.GetAPIReader(), AllowCrossNamespaceSelection: true, - Mapper: mgr.GetRESTMapper(), + + Mapper: mgr.GetRESTMapper(), } r.collector = NewCollector( mgr.GetAPIReader(), @@ -413,6 +414,7 @@ func (r *globalResourceController) gatherResources( opts := CollectorOptions{ Accumulator: acc, AllowCrossNamespaceSelection: true, + AllowClusterScopedObjects: true, } // Collect Available Generated Items diff --git a/internal/controllers/resources/namespaced.go b/internal/controllers/resources/namespaced.go index 5db871b8..1b2c01f6 100644 --- a/internal/controllers/resources/namespaced.go +++ b/internal/controllers/resources/namespaced.go @@ -501,6 +501,7 @@ func (r *namespacedResourceController) gatherResources( opts := CollectorOptions{ Accumulator: acc, AllowCrossNamespaceSelection: false, + AllowClusterScopedObjects: false, ValidatorNamespaces: tpl.NewNamespaceValidator(false, sets.New[string](tnt.Status.Namespaces...)), } diff --git a/internal/controllers/rulestatus/manager.go b/internal/controllers/rulestatus/manager.go index ff6970ea..1d970ea8 100644 --- a/internal/controllers/rulestatus/manager.go +++ b/internal/controllers/rulestatus/manager.go @@ -37,7 +37,7 @@ type Manager struct { reader client.Reader - Metrics *metrics.TenantRecorder + Metrics *metrics.RuleStatusRecorder Log logr.Logger Recorder events.EventRecorder Configuration configuration.Configuration @@ -71,6 +71,8 @@ func (r Manager) Reconcile(ctx context.Context, request ctrl.Request) (result ct if apierrors.IsNotFound(err) { log.V(5).Info("request object not found, could have been deleted after reconcile request") + r.Metrics.DeleteMetrics(request.Name, request.Namespace) + return reconcile.Result{}, nil } @@ -97,6 +99,8 @@ func (r Manager) Reconcile(ctx context.Context, request ctrl.Request) (result ct return } + r.Metrics.RecordConditions(instance) + if e := patchHelper.Patch(ctx, instance); err != nil { if apierrors.IsNotFound(e) || apierrors.HasStatusCause(e, corev1.NamespaceTerminatingCause) { err = nil diff --git a/internal/controllers/tenantowner/manager.go b/internal/controllers/tenantowner/manager.go index 72898fab..4efe046b 100644 --- a/internal/controllers/tenantowner/manager.go +++ b/internal/controllers/tenantowner/manager.go @@ -24,6 +24,7 @@ import ( capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" "github.com/projectcapsule/capsule/internal/controllers/utils" + "github.com/projectcapsule/capsule/internal/metrics" capmeta "github.com/projectcapsule/capsule/pkg/api/meta" "github.com/projectcapsule/capsule/pkg/api/rbac" indexer "github.com/projectcapsule/capsule/pkg/runtime/indexers/tenant" @@ -39,12 +40,17 @@ import ( type TenantOwnerManager struct { client.Client - reader client.Reader - - Log logr.Logger + reader client.Reader + metrics *metrics.TenantOwnerRecorder + Log logr.Logger } -func (r *TenantOwnerManager) SetupWithManager(mgr ctrl.Manager, ctrlConfig utils.ControllerOptions) error { +func (r *TenantOwnerManager) SetupWithManager( + mgr ctrl.Manager, + ctrlConfig utils.ControllerOptions, + metrics *metrics.TenantOwnerRecorder, +) error { + r.metrics = metrics r.reader = mgr.GetAPIReader() return ctrl.NewControllerManagedBy(mgr). @@ -121,6 +127,8 @@ func (r *TenantOwnerManager) Reconcile(ctx context.Context, req ctrl.Request) (r instance := &capsulev1beta2.TenantOwner{} if err = r.Get(ctx, req.NamespacedName, instance); err != nil { if apierrors.IsNotFound(err) { + r.metrics.DeleteMetrics(req.Name) + return reconcile.Result{}, nil } @@ -139,6 +147,8 @@ func (r *TenantOwnerManager) Reconcile(ctx context.Context, req ctrl.Request) (r return reconcile.Result{}, fmt.Errorf("cannot update TenantOwner status: %w", statusErr) } + r.metrics.RecordConditions(instance) + return reconcile.Result{}, reconcileErr } diff --git a/internal/metrics/config_recorder.go b/internal/metrics/config_recorder.go new file mode 100644 index 00000000..00445bda --- /dev/null +++ b/internal/metrics/config_recorder.go @@ -0,0 +1,86 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +//nolint:dupl +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + crtlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/pkg/api/meta" +) + +type ConfigRecorder struct { + resourceConditionGauge *prometheus.GaugeVec +} + +func MustMakeConfigRecorder() *ConfigRecorder { + metricsRecorder := NewConfigRecorder() + crtlmetrics.Registry.MustRegister(metricsRecorder.Collectors()...) + + return metricsRecorder +} + +func NewConfigRecorder() *ConfigRecorder { + return &ConfigRecorder{ + resourceConditionGauge: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: metricsPrefix, + Name: "config_condition", + Help: "The current condition status of a config resource.", + }, + []string{"name", "condition"}, + ), + } +} + +func (r *ConfigRecorder) Collectors() []prometheus.Collector { + return []prometheus.Collector{ + r.resourceConditionGauge, + } +} + +// RecordCondition records the condition as given for the ref. +func (r *ConfigRecorder) RecordConditions(resource *capsulev1beta2.CapsuleConfiguration) { + for _, status := range []string{meta.ReadyCondition, meta.CordonedCondition} { + var value float64 + + cond := resource.Status.Conditions.GetConditionByType(status) + if cond == nil { + r.DeleteConditionMetricByType(resource.GetName(), status) + + continue + } + + if cond.Status == metav1.ConditionTrue { + value = 1 + } + + r.resourceConditionGauge.WithLabelValues(resource.GetName(), status).Set(value) + } +} + +func (r *ConfigRecorder) DeleteConditionMetrics(name string) { + r.resourceConditionGauge.DeletePartialMatch(map[string]string{ + "name": name, + }) +} + +func (r *ConfigRecorder) DeleteConditionMetricByType(name string, condition string) { + r.resourceConditionGauge.DeletePartialMatch(map[string]string{ + "name": name, + "condition": condition, + }) +} + +// DeleteCondition deletes the condition metrics for the ref. +func (r *ConfigRecorder) DeleteMetrics(resourceName string) { + r.resourceConditionGauge.DeletePartialMatch(map[string]string{ + "name": resourceName, + }) + + r.DeleteConditionMetrics(resourceName) +} diff --git a/internal/metrics/customquota_recorder.go b/internal/metrics/customquota_recorder.go index 188ba93a..ad0ed608 100644 --- a/internal/metrics/customquota_recorder.go +++ b/internal/metrics/customquota_recorder.go @@ -9,11 +9,12 @@ import ( ) type CustomQuotaRecorder struct { - ConditionGauge *prometheus.GaugeVec - ResourceUsageGauge *prometheus.GaugeVec - ResourceLimitGauge *prometheus.GaugeVec - ResourceAvailableGauge *prometheus.GaugeVec - ResourceItemUsageGauge *prometheus.GaugeVec + ConditionGauge *prometheus.GaugeVec + ResourceUsageGauge *prometheus.GaugeVec + ResourceUsagePercentageGauge *prometheus.GaugeVec + ResourceLimitGauge *prometheus.GaugeVec + ResourceAvailableGauge *prometheus.GaugeVec + ResourceItemUsageGauge *prometheus.GaugeVec } func MustMakeCustomQuotaRecorder() *CustomQuotaRecorder { @@ -39,6 +40,14 @@ func NewCustomQuotaRecorder() *CustomQuotaRecorder { Help: "Current resource usage for given custom quota", }, []string{"custom_quota", "target_namespace"}, ), + + ResourceUsagePercentageGauge: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: metricsPrefix, + Name: "custom_quota_resource_usage_percentage", + Help: "Current resource usage (%) for given custom quota", + }, []string{"custom_quota", "target_namespace"}, + ), ResourceLimitGauge: prometheus.NewGaugeVec( prometheus.GaugeOpts{ Namespace: metricsPrefix, @@ -67,6 +76,7 @@ func (r *CustomQuotaRecorder) Collectors() []prometheus.Collector { return []prometheus.Collector{ r.ConditionGauge, r.ResourceUsageGauge, + r.ResourceUsagePercentageGauge, r.ResourceLimitGauge, r.ResourceAvailableGauge, r.ResourceItemUsageGauge, @@ -82,6 +92,10 @@ func (r *CustomQuotaRecorder) DeleteAllMetricsForCustomQuota(name string, namesp "custom_quota": name, "target_namespace": namespace, }) + r.ResourceUsagePercentageGauge.DeletePartialMatch(map[string]string{ + "custom_quota": name, + "target_namespace": namespace, + }) r.ResourceLimitGauge.DeletePartialMatch(map[string]string{ "custom_quota": name, "target_namespace": namespace, diff --git a/internal/metrics/global_customquota_recorder.go b/internal/metrics/global_customquota_recorder.go index 48c29c24..460482ff 100644 --- a/internal/metrics/global_customquota_recorder.go +++ b/internal/metrics/global_customquota_recorder.go @@ -9,11 +9,12 @@ import ( ) type GlobalCustomQuotaRecorder struct { - ConditionGauge *prometheus.GaugeVec - ResourceUsageGauge *prometheus.GaugeVec - ResourceLimitGauge *prometheus.GaugeVec - ResourceAvailableGauge *prometheus.GaugeVec - ResourceItemUsageGauge *prometheus.GaugeVec + ConditionGauge *prometheus.GaugeVec + ResourceUsageGauge *prometheus.GaugeVec + ResourceUsagePercentageGauge *prometheus.GaugeVec + ResourceLimitGauge *prometheus.GaugeVec + ResourceAvailableGauge *prometheus.GaugeVec + ResourceItemUsageGauge *prometheus.GaugeVec } func MustMakeGlobalCustomQuotaRecorder() *GlobalCustomQuotaRecorder { @@ -39,6 +40,13 @@ func NewGlobalCustomQuotaRecorder() *GlobalCustomQuotaRecorder { Help: "Current resource usage for given global custom quota", }, []string{"custom_quota"}, ), + ResourceUsagePercentageGauge: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: metricsPrefix, + Name: "global_custom_quota_resource_usage_percentage", + Help: "Current resource usage (%) for given global custom quota", + }, []string{"custom_quota"}, + ), ResourceLimitGauge: prometheus.NewGaugeVec( prometheus.GaugeOpts{ Namespace: metricsPrefix, @@ -66,6 +74,7 @@ func NewGlobalCustomQuotaRecorder() *GlobalCustomQuotaRecorder { func (r *GlobalCustomQuotaRecorder) Collectors() []prometheus.Collector { return []prometheus.Collector{ r.ConditionGauge, + r.ResourceUsagePercentageGauge, r.ResourceUsageGauge, r.ResourceLimitGauge, r.ResourceAvailableGauge, @@ -80,6 +89,10 @@ func (r *GlobalCustomQuotaRecorder) DeleteAllMetricsForGlobalCustomQuota(name st r.ResourceUsageGauge.DeletePartialMatch(map[string]string{ "custom_quota": name, }) + r.ResourceUsagePercentageGauge.DeletePartialMatch(map[string]string{ + "custom_quota": name, + }) + r.ResourceLimitGauge.DeletePartialMatch(map[string]string{ "custom_quota": name, }) diff --git a/internal/metrics/global_tenantresource_recorder.go b/internal/metrics/global_tenantresource_recorder.go index ed105aad..1878c8fc 100644 --- a/internal/metrics/global_tenantresource_recorder.go +++ b/internal/metrics/global_tenantresource_recorder.go @@ -1,6 +1,7 @@ // Copyright 2020-2026 Project Capsule Authors // SPDX-License-Identifier: Apache-2.0 +//nolint:dupl package metrics import ( diff --git a/internal/metrics/rulestatus_recorder.go b/internal/metrics/rulestatus_recorder.go new file mode 100644 index 00000000..7d933d01 --- /dev/null +++ b/internal/metrics/rulestatus_recorder.go @@ -0,0 +1,89 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +//nolint:dupl +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + crtlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/pkg/api/meta" +) + +type RuleStatusRecorder struct { + resourceConditionGauge *prometheus.GaugeVec +} + +func MustMakeRuleStatusRecorder() *RuleStatusRecorder { + metricsRecorder := NewRuleStatusRecorder() + crtlmetrics.Registry.MustRegister(metricsRecorder.Collectors()...) + + return metricsRecorder +} + +func NewRuleStatusRecorder() *RuleStatusRecorder { + return &RuleStatusRecorder{ + resourceConditionGauge: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: metricsPrefix, + Name: "rulestatus_condition", + Help: "The current condition status of a rulestatus resource.", + }, + []string{"name", "target_namespace", "condition"}, + ), + } +} + +func (r *RuleStatusRecorder) Collectors() []prometheus.Collector { + return []prometheus.Collector{ + r.resourceConditionGauge, + } +} + +// RecordCondition records the condition as given for the ref. +func (r *RuleStatusRecorder) RecordConditions(resource *capsulev1beta2.RuleStatus) { + for _, status := range []string{meta.ReadyCondition, meta.CordonedCondition} { + var value float64 + + cond := resource.Status.Conditions.GetConditionByType(status) + if cond == nil { + r.DeleteConditionMetricByType(resource.GetName(), resource.GetNamespace(), status) + + continue + } + + if cond.Status == metav1.ConditionTrue { + value = 1 + } + + r.resourceConditionGauge.WithLabelValues(resource.GetName(), resource.GetNamespace(), status).Set(value) + } +} + +func (r *RuleStatusRecorder) DeleteConditionMetrics(name string, namespace string) { + r.resourceConditionGauge.DeletePartialMatch(map[string]string{ + "name": name, + "target_namespace": namespace, + }) +} + +func (r *RuleStatusRecorder) DeleteConditionMetricByType(name string, namespace string, condition string) { + r.resourceConditionGauge.DeletePartialMatch(map[string]string{ + "name": name, + "target_namespace": namespace, + "condition": condition, + }) +} + +// DeleteCondition deletes the condition metrics for the ref. +func (r *RuleStatusRecorder) DeleteMetrics(resourceName string, resourceNamespace string) { + r.resourceConditionGauge.DeletePartialMatch(map[string]string{ + "name": resourceName, + "target_namespace": resourceNamespace, + }) + + r.DeleteConditionMetrics(resourceName, resourceNamespace) +} diff --git a/internal/metrics/tenantowner_recorder.go b/internal/metrics/tenantowner_recorder.go new file mode 100644 index 00000000..bfb5a904 --- /dev/null +++ b/internal/metrics/tenantowner_recorder.go @@ -0,0 +1,86 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +//nolint:dupl +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + crtlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/pkg/api/meta" +) + +type TenantOwnerRecorder struct { + resourceConditionGauge *prometheus.GaugeVec +} + +func MustMakeTenantOwnerRecorder() *TenantOwnerRecorder { + metricsRecorder := NewTenantOwnerRecorder() + crtlmetrics.Registry.MustRegister(metricsRecorder.Collectors()...) + + return metricsRecorder +} + +func NewTenantOwnerRecorder() *TenantOwnerRecorder { + return &TenantOwnerRecorder{ + resourceConditionGauge: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: metricsPrefix, + Name: "tenantowner_condition", + Help: "The current condition status of a tenantowner resource.", + }, + []string{"name", "condition"}, + ), + } +} + +func (r *TenantOwnerRecorder) Collectors() []prometheus.Collector { + return []prometheus.Collector{ + r.resourceConditionGauge, + } +} + +// RecordCondition records the condition as given for the ref. +func (r *TenantOwnerRecorder) RecordConditions(resource *capsulev1beta2.TenantOwner) { + for _, status := range []string{meta.ReadyCondition, meta.CordonedCondition} { + var value float64 + + cond := resource.Status.Conditions.GetConditionByType(status) + if cond == nil { + r.DeleteConditionMetricByType(resource.GetName(), status) + + continue + } + + if cond.Status == metav1.ConditionTrue { + value = 1 + } + + r.resourceConditionGauge.WithLabelValues(resource.GetName(), status).Set(value) + } +} + +func (r *TenantOwnerRecorder) DeleteConditionMetrics(name string) { + r.resourceConditionGauge.DeletePartialMatch(map[string]string{ + "name": name, + }) +} + +func (r *TenantOwnerRecorder) DeleteConditionMetricByType(name string, condition string) { + r.resourceConditionGauge.DeletePartialMatch(map[string]string{ + "name": name, + "condition": condition, + }) +} + +// DeleteCondition deletes the condition metrics for the ref. +func (r *TenantOwnerRecorder) DeleteMetrics(resourceName string) { + r.resourceConditionGauge.DeletePartialMatch(map[string]string{ + "name": resourceName, + }) + + r.DeleteConditionMetrics(resourceName) +} diff --git a/internal/metrics/tenantresource_recorder.go b/internal/metrics/tenantresource_recorder.go index efe34764..1c9b4a4c 100644 --- a/internal/metrics/tenantresource_recorder.go +++ b/internal/metrics/tenantresource_recorder.go @@ -1,6 +1,7 @@ // Copyright 2020-2026 Project Capsule Authors // SPDX-License-Identifier: Apache-2.0 +//nolint:dupl package metrics import ( diff --git a/pkg/template/reference.go b/pkg/template/reference.go index 899640e8..f6b74391 100644 --- a/pkg/template/reference.go +++ b/pkg/template/reference.go @@ -128,21 +128,27 @@ func (t ResourceReference) LoadResources( func (t ResourceReference) IsNamespacedGVK( restMapper k8smeta.RESTMapper, ) (bool, error) { - gv, err := schema.ParseGroupVersion(t.APIVersion) + return IsNamespacedGVK(restMapper, t.APIVersion, t.Kind) +} + +func IsNamespacedGVK( + restMapper k8smeta.RESTMapper, + apiVersion string, + kind string, +) (bool, error) { + gv, err := schema.ParseGroupVersion(apiVersion) if err != nil { - return false, fmt.Errorf("invalid apiVersion %q: %w", t.APIVersion, err) + return false, fmt.Errorf("invalid apiVersion %q: %w", apiVersion, err) } - gvk := gv.WithKind(t.Kind) + gvk := gv.WithKind(kind) mapping, err := restMapper.RESTMapping(gvk.GroupKind(), gvk.Version) if err != nil { return false, fmt.Errorf("failed to resolve GVK %s: %w", gvk.String(), err) } - isNamespaced := mapping.Scope.Name() == k8smeta.RESTScopeNameNamespace - - return isNamespaced, nil + return mapping.Scope.Name() == k8smeta.RESTScopeNameNamespace, nil } func (t ResourceReference) loadResources(